1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * Copyright (c) 2021 The FreeBSD Foundation 7 * 8 * Portions of this software were developed by Mark Johnston under sponsorship 9 * from the FreeBSD Foundation. 10 * 11 * This code is part of the KMSAN subsystem of the NetBSD kernel. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #define SAN_RUNTIME 36 37 #include <sys/cdefs.h> 38 #if 0 39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bio.h> 45 #include <sys/buf.h> 46 #include <sys/conf.h> 47 #include <sys/kdb.h> 48 #include <sys/kernel.h> 49 #include <sys/linker.h> 50 #include <sys/malloc.h> 51 #include <sys/mbuf.h> 52 #include <sys/memdesc.h> 53 #include <sys/msan.h> 54 #include <sys/proc.h> 55 #include <sys/stack.h> 56 #include <sys/sysctl.h> 57 #include <sys/uio.h> 58 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 62 #include <machine/msan.h> 63 #include <machine/stdarg.h> 64 65 void kmsan_init_arg(size_t); 66 void kmsan_init_ret(size_t); 67 68 /* -------------------------------------------------------------------------- */ 69 70 /* 71 * Part of the compiler ABI. 72 */ 73 74 typedef struct { 75 uint8_t *shad; 76 msan_orig_t *orig; 77 } msan_meta_t; 78 79 #define MSAN_PARAM_SIZE 800 80 #define MSAN_RETVAL_SIZE 800 81 typedef struct { 82 uint8_t param_shadow[MSAN_PARAM_SIZE]; 83 uint8_t retval_shadow[MSAN_RETVAL_SIZE]; 84 uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; 85 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 86 uint64_t va_arg_overflow_size; 87 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; 88 msan_orig_t retval_origin; 89 } msan_tls_t; 90 91 /* -------------------------------------------------------------------------- */ 92 93 #define MSAN_NCONTEXT 4 94 #define MSAN_ORIG_MASK (~0x3) 95 96 typedef struct kmsan_td { 97 size_t ctx; 98 msan_tls_t tls[MSAN_NCONTEXT]; 99 } msan_td_t; 100 101 static msan_tls_t dummy_tls; 102 103 /* 104 * Use separate dummy regions for loads and stores: stores may mark the region 105 * as uninitialized, and that can trigger false positives. 106 */ 107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 110 static msan_td_t msan_thread0; 111 static bool kmsan_reporting = false; 112 113 /* 114 * Avoid clobbering any thread-local state before we panic. 115 */ 116 #define kmsan_panic(f, ...) do { \ 117 kmsan_disabled = true; \ 118 panic(f, __VA_ARGS__); \ 119 } while (0) 120 121 #define REPORT(f, ...) do { \ 122 if (panic_on_violation) { \ 123 kmsan_panic(f, __VA_ARGS__); \ 124 } else { \ 125 struct stack st; \ 126 \ 127 stack_save(&st); \ 128 printf(f "\n", __VA_ARGS__); \ 129 stack_print_ddb(&st); \ 130 } \ 131 } while (0) 132 133 FEATURE(kmsan, "Kernel memory sanitizer"); 134 135 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 136 "KMSAN options"); 137 138 static bool panic_on_violation = 1; 139 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, 140 &panic_on_violation, 0, 141 "Panic if an invalid access is detected"); 142 143 static bool kmsan_disabled __read_mostly = true; 144 #define kmsan_enabled (!kmsan_disabled) 145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 146 &kmsan_disabled, 0, "KMSAN is disabled"); 147 148 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); 149 150 /* -------------------------------------------------------------------------- */ 151 152 static inline const char * 153 kmsan_orig_name(int type) 154 { 155 switch (type) { 156 case KMSAN_TYPE_STACK: 157 return ("stack"); 158 case KMSAN_TYPE_KMEM: 159 return ("kmem"); 160 case KMSAN_TYPE_MALLOC: 161 return ("malloc"); 162 case KMSAN_TYPE_UMA: 163 return ("UMA"); 164 default: 165 return ("unknown"); 166 } 167 } 168 169 static void 170 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off, 171 const char *hook) 172 { 173 const char *typename; 174 char *var, *fn; 175 uintptr_t ptr; 176 long foff; 177 char buf[128]; 178 int type; 179 180 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 181 return; 182 183 kmsan_reporting = true; 184 __compiler_membar(); 185 186 if (*orig == 0) { 187 REPORT("MSan: Uninitialized memory in %s, offset %zu", 188 hook, off); 189 goto out; 190 } 191 192 kmsan_md_orig_decode(*orig, &type, &ptr); 193 typename = kmsan_orig_name(type); 194 195 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 196 sizeof(buf), &foff) == 0) { 197 REPORT("MSan: Uninitialized %s memory in %s, " 198 "offset %zu/%zu, addr %p, from %s+%#lx", 199 typename, hook, off, size, addr, buf, foff); 200 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 201 /* 202 * The format of the string is: "----var@function". Parse it to 203 * display a nice warning. 204 */ 205 var = (char *)ptr + 4; 206 strlcpy(buf, var, sizeof(buf)); 207 var = buf; 208 fn = strchr(buf, '@'); 209 *fn++ = '\0'; 210 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, " 211 "variable '%s' from %s", typename, hook, off, var, fn); 212 } else { 213 REPORT("MSan: Uninitialized %s memory in %s, " 214 "offset %zu/%zu, addr %p, PC %p", 215 typename, hook, off, size, addr, (void *)ptr); 216 } 217 218 out: 219 __compiler_membar(); 220 kmsan_reporting = false; 221 } 222 223 static void 224 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 225 { 226 const char *typename; 227 char *var, *fn; 228 uintptr_t ptr; 229 char buf[128]; 230 long foff; 231 int type; 232 233 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 234 return; 235 236 kmsan_reporting = true; 237 __compiler_membar(); 238 239 if (orig == 0) { 240 REPORT("MSan: uninitialized variable in %p", (void *)pc); 241 goto out; 242 } 243 244 kmsan_md_orig_decode(orig, &type, &ptr); 245 typename = kmsan_orig_name(type); 246 247 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 248 sizeof(buf), &foff) == 0) { 249 REPORT("MSan: Uninitialized %s memory from %s+%#lx", 250 typename, buf, foff); 251 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 252 /* 253 * The format of the string is: "----var@function". Parse it to 254 * display a nice warning. 255 */ 256 var = (char *)ptr + 4; 257 strlcpy(buf, var, sizeof(buf)); 258 var = buf; 259 fn = strchr(buf, '@'); 260 *fn++ = '\0'; 261 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn); 262 } else { 263 REPORT("MSan: Uninitialized %s memory, origin %x", 264 typename, orig); 265 } 266 267 out: 268 __compiler_membar(); 269 kmsan_reporting = false; 270 } 271 272 /* -------------------------------------------------------------------------- */ 273 274 static inline msan_meta_t 275 kmsan_meta_get(const void *addr, size_t size, const bool write) 276 { 277 msan_meta_t ret; 278 279 if (__predict_false(!kmsan_enabled)) { 280 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 281 ret.orig = (msan_orig_t *)msan_dummy_orig; 282 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { 283 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 284 ret.orig = (msan_orig_t *)msan_dummy_orig; 285 } else { 286 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); 287 ret.orig = 288 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 289 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & 290 MSAN_ORIG_MASK); 291 } 292 293 return (ret); 294 } 295 296 static inline void 297 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) 298 { 299 msan_orig_t *orig; 300 size_t i; 301 302 if (__predict_false(!kmsan_enabled)) 303 return; 304 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) 305 return; 306 307 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 308 size += ((uintptr_t)orig & (sizeof(*orig) - 1)); 309 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 310 311 for (i = 0; i < size; i += 4) { 312 orig[i / 4] = o; 313 } 314 } 315 316 static inline void 317 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) 318 { 319 uint8_t *shad; 320 321 if (__predict_false(!kmsan_enabled)) 322 return; 323 if (__predict_false(kmsan_md_unsupported(addr))) 324 return; 325 326 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 327 __builtin_memset(shad, c, size); 328 } 329 330 static inline void 331 kmsan_meta_copy(void *dst, const void *src, size_t size) 332 { 333 uint8_t *orig_src, *orig_dst; 334 uint8_t *shad_src, *shad_dst; 335 msan_orig_t *_src, *_dst; 336 size_t i; 337 338 if (__predict_false(!kmsan_enabled)) 339 return; 340 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) 341 return; 342 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { 343 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); 344 return; 345 } 346 347 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); 348 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); 349 __builtin_memmove(shad_dst, shad_src, size); 350 351 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); 352 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); 353 for (i = 0; i < size; i++) { 354 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); 355 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); 356 *_dst = *_src; 357 orig_src++; 358 orig_dst++; 359 } 360 } 361 362 static inline void 363 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) 364 { 365 msan_orig_t *orig; 366 uint8_t *shad; 367 size_t i; 368 369 if (__predict_false(!kmsan_enabled)) 370 return; 371 if (__predict_false(kmsan_md_unsupported(addr))) 372 return; 373 374 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 375 for (i = 0; i < size; i++) { 376 if (__predict_true(shad[i] == 0)) 377 continue; 378 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)&shad[i]); 379 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 380 kmsan_report_hook((const char *)addr + i, orig, size, i, hook); 381 break; 382 } 383 } 384 385 void 386 kmsan_init_arg(size_t n) 387 { 388 msan_td_t *mtd; 389 uint8_t *arg; 390 391 if (__predict_false(!kmsan_enabled)) 392 return; 393 if (__predict_false(curthread == NULL)) 394 return; 395 mtd = curthread->td_kmsan; 396 arg = mtd->tls[mtd->ctx].param_shadow; 397 __builtin_memset(arg, 0, n); 398 } 399 400 void 401 kmsan_init_ret(size_t n) 402 { 403 msan_td_t *mtd; 404 uint8_t *arg; 405 406 if (__predict_false(!kmsan_enabled)) 407 return; 408 if (__predict_false(curthread == NULL)) 409 return; 410 mtd = curthread->td_kmsan; 411 arg = mtd->tls[mtd->ctx].retval_shadow; 412 __builtin_memset(arg, 0, n); 413 } 414 415 static void 416 kmsan_check_arg(size_t size, const char *hook) 417 { 418 msan_orig_t *orig; 419 msan_td_t *mtd; 420 uint8_t *arg; 421 size_t ctx, i; 422 423 if (__predict_false(!kmsan_enabled)) 424 return; 425 if (__predict_false(curthread == NULL)) 426 return; 427 mtd = curthread->td_kmsan; 428 ctx = mtd->ctx; 429 arg = mtd->tls[ctx].param_shadow; 430 431 for (i = 0; i < size; i++) { 432 if (__predict_true(arg[i] == 0)) 433 continue; 434 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)]; 435 kmsan_report_hook((const char *)arg + i, orig, size, i, hook); 436 break; 437 } 438 } 439 440 void 441 kmsan_thread_alloc(struct thread *td) 442 { 443 msan_td_t *mtd; 444 445 if (!kmsan_enabled) 446 return; 447 448 mtd = td->td_kmsan; 449 if (mtd == NULL) { 450 /* We might be recycling a thread. */ 451 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + 452 sizeof(int)); 453 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); 454 } 455 kmsan_memset(mtd, 0, sizeof(*mtd)); 456 mtd->ctx = 0; 457 458 if (td->td_kstack != 0) 459 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), 460 KMSAN_STATE_UNINIT); 461 462 td->td_kmsan = mtd; 463 } 464 465 void 466 kmsan_thread_free(struct thread *td) 467 { 468 msan_td_t *mtd; 469 470 if (!kmsan_enabled) 471 return; 472 if (__predict_false(td == curthread)) 473 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); 474 475 mtd = td->td_kmsan; 476 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); 477 free(mtd, M_KMSAN); 478 td->td_kmsan = NULL; 479 } 480 481 void kmsan_intr_enter(void); 482 void kmsan_intr_leave(void); 483 484 void 485 kmsan_intr_enter(void) 486 { 487 msan_td_t *mtd; 488 489 if (__predict_false(!kmsan_enabled)) 490 return; 491 492 mtd = curthread->td_kmsan; 493 mtd->ctx++; 494 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) 495 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 496 } 497 498 void 499 kmsan_intr_leave(void) 500 { 501 msan_td_t *mtd; 502 503 if (__predict_false(!kmsan_enabled)) 504 return; 505 506 mtd = curthread->td_kmsan; 507 if (__predict_false(mtd->ctx == 0)) 508 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 509 mtd->ctx--; 510 } 511 512 /* -------------------------------------------------------------------------- */ 513 514 void 515 kmsan_shadow_map(vm_offset_t addr, size_t size) 516 { 517 size_t npages, i; 518 vm_offset_t va; 519 520 MPASS(addr % PAGE_SIZE == 0); 521 MPASS(size % PAGE_SIZE == 0); 522 523 if (!kmsan_enabled) 524 return; 525 526 npages = atop(size); 527 528 va = kmsan_md_addr_to_shad(addr); 529 for (i = 0; i < npages; i++) { 530 pmap_san_enter(va + ptoa(i)); 531 } 532 533 va = kmsan_md_addr_to_orig(addr); 534 for (i = 0; i < npages; i++) { 535 pmap_san_enter(va + ptoa(i)); 536 } 537 } 538 539 void 540 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) 541 { 542 msan_orig_t orig; 543 544 orig = kmsan_md_orig_encode(type, pc); 545 kmsan_origin_fill(addr, orig, size); 546 } 547 548 void 549 kmsan_mark(const void *addr, size_t size, uint8_t c) 550 { 551 kmsan_shadow_fill((uintptr_t)addr, c, size); 552 } 553 554 void 555 kmsan_mark_bio(const struct bio *bp, uint8_t c) 556 { 557 kmsan_mark(bp->bio_data, bp->bio_length, c); 558 } 559 560 void 561 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) 562 { 563 do { 564 if ((m->m_flags & M_EXTPG) == 0) 565 kmsan_mark(m->m_data, m->m_len, c); 566 m = m->m_next; 567 } while (m != NULL); 568 } 569 570 void 571 kmsan_check(const void *p, size_t sz, const char *descr) 572 { 573 kmsan_shadow_check((uintptr_t)p, sz, descr); 574 } 575 576 void 577 kmsan_check_bio(const struct bio *bp, const char *descr) 578 { 579 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); 580 } 581 582 void 583 kmsan_check_mbuf(const struct mbuf *m, const char *descr) 584 { 585 do { 586 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); 587 } while ((m = m->m_next) != NULL); 588 } 589 590 void 591 kmsan_init(void) 592 { 593 int disabled; 594 595 disabled = 0; 596 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); 597 if (disabled) 598 return; 599 600 /* Initialize the TLS for curthread. */ 601 msan_thread0.ctx = 0; 602 thread0.td_kmsan = &msan_thread0; 603 604 /* Now officially enabled. */ 605 kmsan_disabled = false; 606 } 607 608 /* -------------------------------------------------------------------------- */ 609 610 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 611 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 612 613 msan_meta_t 614 __msan_metadata_ptr_for_load_n(void *addr, size_t size) 615 { 616 return (kmsan_meta_get(addr, size, false)); 617 } 618 619 msan_meta_t 620 __msan_metadata_ptr_for_store_n(void *addr, size_t size) 621 { 622 return (kmsan_meta_get(addr, size, true)); 623 } 624 625 #define MSAN_META_FUNC(size) \ 626 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 627 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 628 { \ 629 return (kmsan_meta_get(addr, size, false)); \ 630 } \ 631 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 632 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 633 { \ 634 return (kmsan_meta_get(addr, size, true)); \ 635 } 636 637 MSAN_META_FUNC(1) 638 MSAN_META_FUNC(2) 639 MSAN_META_FUNC(4) 640 MSAN_META_FUNC(8) 641 642 void __msan_instrument_asm_store(const void *, size_t); 643 msan_orig_t __msan_chain_origin(msan_orig_t); 644 void __msan_poison(const void *, size_t); 645 void __msan_unpoison(const void *, size_t); 646 void __msan_poison_alloca(const void *, uint64_t, const char *); 647 void __msan_unpoison_alloca(const void *, uint64_t); 648 void __msan_warning(msan_orig_t); 649 msan_tls_t *__msan_get_context_state(void); 650 651 void 652 __msan_instrument_asm_store(const void *addr, size_t size) 653 { 654 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 655 } 656 657 msan_orig_t 658 __msan_chain_origin(msan_orig_t origin) 659 { 660 return (origin); 661 } 662 663 void 664 __msan_poison(const void *addr, size_t size) 665 { 666 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 667 } 668 669 void 670 __msan_unpoison(const void *addr, size_t size) 671 { 672 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 673 } 674 675 void 676 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr) 677 { 678 msan_orig_t orig; 679 680 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 681 kmsan_origin_fill(addr, orig, size); 682 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 683 } 684 685 void 686 __msan_unpoison_alloca(const void *addr, uint64_t size) 687 { 688 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 689 } 690 691 void 692 __msan_warning(msan_orig_t origin) 693 { 694 if (__predict_false(!kmsan_enabled)) 695 return; 696 kmsan_report_inline(origin, KMSAN_RET_ADDR); 697 } 698 699 msan_tls_t * 700 __msan_get_context_state(void) 701 { 702 msan_td_t *mtd; 703 704 /* 705 * When APs are started, they execute some C code before curthread is 706 * set. We have to handle that here. 707 */ 708 if (__predict_false(!kmsan_enabled || curthread == NULL)) 709 return (&dummy_tls); 710 mtd = curthread->td_kmsan; 711 return (&mtd->tls[mtd->ctx]); 712 } 713 714 /* -------------------------------------------------------------------------- */ 715 716 /* 717 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 718 * initialized areas properly. 719 */ 720 721 void * 722 kmsan_memcpy(void *dst, const void *src, size_t len) 723 { 724 /* No kmsan_check_arg, because inlined. */ 725 kmsan_init_ret(sizeof(void *)); 726 if (__predict_true(len != 0)) { 727 kmsan_meta_copy(dst, src, len); 728 } 729 return (__builtin_memcpy(dst, src, len)); 730 } 731 732 int 733 kmsan_memcmp(const void *b1, const void *b2, size_t len) 734 { 735 const uint8_t *_b1 = b1, *_b2 = b2; 736 size_t i; 737 738 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 739 "memcmp():args"); 740 kmsan_init_ret(sizeof(int)); 741 742 for (i = 0; i < len; i++) { 743 if (*_b1 != *_b2) { 744 kmsan_shadow_check((uintptr_t)b1, i + 1, 745 "memcmp():arg1"); 746 kmsan_shadow_check((uintptr_t)b2, i + 1, 747 "memcmp():arg2"); 748 return (*_b1 - *_b2); 749 } 750 _b1++, _b2++; 751 } 752 753 return (0); 754 } 755 756 void * 757 kmsan_memset(void *dst, int c, size_t len) 758 { 759 /* No kmsan_check_arg, because inlined. */ 760 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); 761 kmsan_init_ret(sizeof(void *)); 762 return (__builtin_memset(dst, c, len)); 763 } 764 765 void * 766 kmsan_memmove(void *dst, const void *src, size_t len) 767 { 768 /* No kmsan_check_arg, because inlined. */ 769 if (__predict_true(len != 0)) { 770 kmsan_meta_copy(dst, src, len); 771 } 772 kmsan_init_ret(sizeof(void *)); 773 return (__builtin_memmove(dst, src, len)); 774 } 775 776 __strong_reference(kmsan_memcpy, __msan_memcpy); 777 __strong_reference(kmsan_memset, __msan_memset); 778 __strong_reference(kmsan_memmove, __msan_memmove); 779 780 char * 781 kmsan_strcpy(char *dst, const char *src) 782 { 783 const char *_src = src; 784 char *_dst = dst; 785 size_t len = 0; 786 787 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 788 789 while (1) { 790 len++; 791 *dst = *src; 792 if (*src == '\0') 793 break; 794 src++, dst++; 795 } 796 797 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); 798 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); 799 kmsan_init_ret(sizeof(char *)); 800 return (_dst); 801 } 802 803 int 804 kmsan_strcmp(const char *s1, const char *s2) 805 { 806 const char *_s1 = s1, *_s2 = s2; 807 size_t len = 0; 808 809 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 810 kmsan_init_ret(sizeof(int)); 811 812 while (1) { 813 len++; 814 if (*s1 != *s2) 815 break; 816 if (*s1 == '\0') { 817 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 818 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 819 return (0); 820 } 821 s1++, s2++; 822 } 823 824 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 825 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 826 827 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 828 } 829 830 size_t 831 kmsan_strlen(const char *str) 832 { 833 const char *s; 834 835 kmsan_check_arg(sizeof(str), "strlen():args"); 836 837 s = str; 838 while (1) { 839 if (*s == '\0') 840 break; 841 s++; 842 } 843 844 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); 845 kmsan_init_ret(sizeof(size_t)); 846 return (s - str); 847 } 848 849 int kmsan_copyin(const void *, void *, size_t); 850 int kmsan_copyout(const void *, void *, size_t); 851 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 852 853 int 854 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 855 { 856 int ret; 857 858 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 859 "copyin():args"); 860 ret = copyin(uaddr, kaddr, len); 861 if (ret == 0) 862 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); 863 kmsan_init_ret(sizeof(int)); 864 return (ret); 865 } 866 867 int 868 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 869 { 870 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 871 "copyout():args"); 872 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); 873 kmsan_init_ret(sizeof(int)); 874 return (copyout(kaddr, uaddr, len)); 875 } 876 877 int 878 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 879 { 880 size_t _done; 881 int ret; 882 883 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 884 sizeof(len) + sizeof(done), "copyinstr():args"); 885 ret = copyinstr(uaddr, kaddr, len, &_done); 886 if (ret == 0) 887 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); 888 if (done != NULL) { 889 *done = _done; 890 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); 891 } 892 kmsan_init_ret(sizeof(int)); 893 return (ret); 894 } 895 896 /* -------------------------------------------------------------------------- */ 897 898 int 899 kmsan_fubyte(volatile const void *base) 900 { 901 int ret; 902 903 kmsan_check_arg(sizeof(base), "fubyte(): args"); 904 ret = fubyte(base); 905 kmsan_init_ret(sizeof(int)); 906 return (ret); 907 } 908 909 int 910 kmsan_fuword16(volatile const void *base) 911 { 912 int ret; 913 914 kmsan_check_arg(sizeof(base), "fuword16(): args"); 915 ret = fuword16(base); 916 kmsan_init_ret(sizeof(int)); 917 return (ret); 918 } 919 920 int 921 kmsan_fueword(volatile const void *base, long *val) 922 { 923 int ret; 924 925 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); 926 ret = fueword(base, val); 927 if (ret == 0) 928 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 929 sizeof(*val)); 930 kmsan_init_ret(sizeof(int)); 931 return (ret); 932 } 933 934 int 935 kmsan_fueword32(volatile const void *base, int32_t *val) 936 { 937 int ret; 938 939 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); 940 ret = fueword32(base, val); 941 if (ret == 0) 942 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 943 sizeof(*val)); 944 kmsan_init_ret(sizeof(int)); 945 return (ret); 946 } 947 948 int 949 kmsan_fueword64(volatile const void *base, int64_t *val) 950 { 951 int ret; 952 953 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); 954 ret = fueword64(base, val); 955 if (ret == 0) 956 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 957 sizeof(*val)); 958 kmsan_init_ret(sizeof(int)); 959 return (ret); 960 } 961 962 int 963 kmsan_subyte(volatile void *base, int byte) 964 { 965 int ret; 966 967 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); 968 ret = subyte(base, byte); 969 kmsan_init_ret(sizeof(int)); 970 return (ret); 971 } 972 973 int 974 kmsan_suword(volatile void *base, long word) 975 { 976 int ret; 977 978 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); 979 ret = suword(base, word); 980 kmsan_init_ret(sizeof(int)); 981 return (ret); 982 } 983 984 int 985 kmsan_suword16(volatile void *base, int word) 986 { 987 int ret; 988 989 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); 990 ret = suword16(base, word); 991 kmsan_init_ret(sizeof(int)); 992 return (ret); 993 } 994 995 int 996 kmsan_suword32(volatile void *base, int32_t word) 997 { 998 int ret; 999 1000 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); 1001 ret = suword32(base, word); 1002 kmsan_init_ret(sizeof(int)); 1003 return (ret); 1004 } 1005 1006 int 1007 kmsan_suword64(volatile void *base, int64_t word) 1008 { 1009 int ret; 1010 1011 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); 1012 ret = suword64(base, word); 1013 kmsan_init_ret(sizeof(int)); 1014 return (ret); 1015 } 1016 1017 int 1018 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, 1019 uint32_t newval) 1020 { 1021 int ret; 1022 1023 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1024 sizeof(newval), "casueword32(): args"); 1025 ret = casueword32(base, oldval, oldvalp, newval); 1026 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1027 sizeof(*oldvalp)); 1028 kmsan_init_ret(sizeof(int)); 1029 return (ret); 1030 } 1031 1032 int 1033 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, 1034 u_long newval) 1035 { 1036 int ret; 1037 1038 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1039 sizeof(newval), "casueword32(): args"); 1040 ret = casueword(base, oldval, oldvalp, newval); 1041 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1042 sizeof(*oldvalp)); 1043 kmsan_init_ret(sizeof(int)); 1044 return (ret); 1045 } 1046 1047 /* -------------------------------------------------------------------------- */ 1048 1049 #include <machine/atomic.h> 1050 #include <sys/atomic_san.h> 1051 1052 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1053 void kmsan_atomic_add_##name(volatile type *ptr, type val) \ 1054 { \ 1055 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1056 "atomic_add_" #name "():args"); \ 1057 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1058 "atomic_add_" #name "():ptr"); \ 1059 atomic_add_##name(ptr, val); \ 1060 } 1061 1062 #define MSAN_ATOMIC_FUNC_ADD(name, type) \ 1063 _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1064 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 1065 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) 1066 1067 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1068 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ 1069 { \ 1070 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1071 "atomic_subtract_" #name "():args"); \ 1072 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1073 "atomic_subtract_" #name "():ptr"); \ 1074 atomic_subtract_##name(ptr, val); \ 1075 } 1076 1077 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1078 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1079 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 1080 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 1081 1082 #define _MSAN_ATOMIC_FUNC_SET(name, type) \ 1083 void kmsan_atomic_set_##name(volatile type *ptr, type val) \ 1084 { \ 1085 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1086 "atomic_set_" #name "():args"); \ 1087 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1088 "atomic_set_" #name "():ptr"); \ 1089 atomic_set_##name(ptr, val); \ 1090 } 1091 1092 #define MSAN_ATOMIC_FUNC_SET(name, type) \ 1093 _MSAN_ATOMIC_FUNC_SET(name, type) \ 1094 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 1095 _MSAN_ATOMIC_FUNC_SET(rel_##name, type) 1096 1097 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1098 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ 1099 { \ 1100 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1101 "atomic_clear_" #name "():args"); \ 1102 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1103 "atomic_clear_" #name "():ptr"); \ 1104 atomic_clear_##name(ptr, val); \ 1105 } 1106 1107 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1108 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1109 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 1110 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 1111 1112 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 1113 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 1114 { \ 1115 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1116 "atomic_fetchadd_" #name "():args"); \ 1117 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1118 "atomic_fetchadd_" #name "():ptr"); \ 1119 kmsan_init_ret(sizeof(type)); \ 1120 return (atomic_fetchadd_##name(ptr, val)); \ 1121 } 1122 1123 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 1124 type kmsan_atomic_readandclear_##name(volatile type *ptr) \ 1125 { \ 1126 kmsan_check_arg(sizeof(ptr), \ 1127 "atomic_readandclear_" #name "():args"); \ 1128 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1129 "atomic_readandclear_" #name "():ptr"); \ 1130 kmsan_init_ret(sizeof(type)); \ 1131 return (atomic_readandclear_##name(ptr)); \ 1132 } 1133 1134 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 1135 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ 1136 { \ 1137 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1138 "atomic_testandclear_" #name "():args"); \ 1139 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1140 "atomic_testandclear_" #name "():ptr"); \ 1141 kmsan_init_ret(sizeof(int)); \ 1142 return (atomic_testandclear_##name(ptr, v)); \ 1143 } 1144 1145 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 1146 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ 1147 { \ 1148 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1149 "atomic_testandset_" #name "():args"); \ 1150 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1151 "atomic_testandset_" #name "():ptr"); \ 1152 kmsan_init_ret(sizeof(int)); \ 1153 return (atomic_testandset_##name(ptr, v)); \ 1154 } 1155 1156 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \ 1157 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ 1158 { \ 1159 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1160 "atomic_swap_" #name "():args"); \ 1161 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1162 "atomic_swap_" #name "():ptr"); \ 1163 kmsan_init_ret(sizeof(type)); \ 1164 return (atomic_swap_##name(ptr, val)); \ 1165 } 1166 1167 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1168 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ 1169 type nval) \ 1170 { \ 1171 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1172 sizeof(nval), "atomic_cmpset_" #name "():args"); \ 1173 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1174 "atomic_cmpset_" #name "():ptr"); \ 1175 kmsan_init_ret(sizeof(int)); \ 1176 return (atomic_cmpset_##name(ptr, oval, nval)); \ 1177 } 1178 1179 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1180 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1181 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 1182 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 1183 1184 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1185 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ 1186 type nval) \ 1187 { \ 1188 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1189 sizeof(nval), "atomic_fcmpset_" #name "():args"); \ 1190 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1191 "atomic_fcmpset_" #name "():ptr"); \ 1192 kmsan_init_ret(sizeof(int)); \ 1193 return (atomic_fcmpset_##name(ptr, oval, nval)); \ 1194 } 1195 1196 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1197 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1198 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 1199 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 1200 1201 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 1202 void kmsan_atomic_thread_fence_##name(void) \ 1203 { \ 1204 atomic_thread_fence_##name(); \ 1205 } 1206 1207 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1208 type kmsan_atomic_load_##name(volatile type *ptr) \ 1209 { \ 1210 kmsan_check_arg(sizeof(ptr), \ 1211 "atomic_load_" #name "():args"); \ 1212 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1213 "atomic_load_" #name "():ptr"); \ 1214 kmsan_init_ret(sizeof(type)); \ 1215 return (atomic_load_##name(ptr)); \ 1216 } 1217 1218 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1219 _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1220 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) 1221 1222 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1223 void kmsan_atomic_store_##name(volatile type *ptr, type val) \ 1224 { \ 1225 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1226 "atomic_store_" #name "():args"); \ 1227 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ 1228 sizeof(type)); \ 1229 atomic_store_##name(ptr, val); \ 1230 } 1231 1232 #define MSAN_ATOMIC_FUNC_STORE(name, type) \ 1233 _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1234 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) 1235 1236 MSAN_ATOMIC_FUNC_ADD(8, uint8_t); 1237 MSAN_ATOMIC_FUNC_ADD(16, uint16_t); 1238 MSAN_ATOMIC_FUNC_ADD(32, uint32_t); 1239 MSAN_ATOMIC_FUNC_ADD(64, uint64_t); 1240 MSAN_ATOMIC_FUNC_ADD(int, u_int); 1241 MSAN_ATOMIC_FUNC_ADD(long, u_long); 1242 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); 1243 1244 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); 1245 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); 1246 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); 1247 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); 1248 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); 1249 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); 1250 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); 1251 1252 MSAN_ATOMIC_FUNC_SET(8, uint8_t); 1253 MSAN_ATOMIC_FUNC_SET(16, uint16_t); 1254 MSAN_ATOMIC_FUNC_SET(32, uint32_t); 1255 MSAN_ATOMIC_FUNC_SET(64, uint64_t); 1256 MSAN_ATOMIC_FUNC_SET(int, u_int); 1257 MSAN_ATOMIC_FUNC_SET(long, u_long); 1258 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); 1259 1260 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); 1261 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); 1262 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); 1263 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); 1264 MSAN_ATOMIC_FUNC_CLEAR(int, u_int); 1265 MSAN_ATOMIC_FUNC_CLEAR(long, u_long); 1266 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); 1267 1268 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); 1269 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); 1270 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); 1271 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); 1272 1273 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); 1274 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); 1275 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); 1276 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); 1277 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); 1278 1279 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); 1280 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); 1281 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); 1282 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); 1283 1284 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); 1285 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); 1286 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); 1287 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); 1288 1289 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); 1290 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); 1291 MSAN_ATOMIC_FUNC_SWAP(int, u_int); 1292 MSAN_ATOMIC_FUNC_SWAP(long, u_long); 1293 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); 1294 1295 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); 1296 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); 1297 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); 1298 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); 1299 MSAN_ATOMIC_FUNC_CMPSET(int, u_int); 1300 MSAN_ATOMIC_FUNC_CMPSET(long, u_long); 1301 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); 1302 1303 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); 1304 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); 1305 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); 1306 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); 1307 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); 1308 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); 1309 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); 1310 1311 _MSAN_ATOMIC_FUNC_LOAD(bool, bool); 1312 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); 1313 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); 1314 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); 1315 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); 1316 MSAN_ATOMIC_FUNC_LOAD(char, u_char); 1317 MSAN_ATOMIC_FUNC_LOAD(short, u_short); 1318 MSAN_ATOMIC_FUNC_LOAD(int, u_int); 1319 MSAN_ATOMIC_FUNC_LOAD(long, u_long); 1320 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); 1321 1322 _MSAN_ATOMIC_FUNC_STORE(bool, bool); 1323 MSAN_ATOMIC_FUNC_STORE(8, uint8_t); 1324 MSAN_ATOMIC_FUNC_STORE(16, uint16_t); 1325 MSAN_ATOMIC_FUNC_STORE(32, uint32_t); 1326 MSAN_ATOMIC_FUNC_STORE(64, uint64_t); 1327 MSAN_ATOMIC_FUNC_STORE(char, u_char); 1328 MSAN_ATOMIC_FUNC_STORE(short, u_short); 1329 MSAN_ATOMIC_FUNC_STORE(int, u_int); 1330 MSAN_ATOMIC_FUNC_STORE(long, u_long); 1331 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); 1332 1333 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); 1334 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); 1335 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); 1336 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); 1337 1338 void 1339 kmsan_atomic_interrupt_fence(void) 1340 { 1341 atomic_interrupt_fence(); 1342 } 1343 1344 /* -------------------------------------------------------------------------- */ 1345 1346 #include <sys/bus.h> 1347 #include <machine/bus.h> 1348 #include <sys/bus_san.h> 1349 1350 int 1351 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 1352 int flags, bus_space_handle_t *handlep) 1353 { 1354 return (bus_space_map(tag, hnd, size, flags, handlep)); 1355 } 1356 1357 void 1358 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 1359 bus_size_t size) 1360 { 1361 bus_space_unmap(tag, hnd, size); 1362 } 1363 1364 int 1365 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 1366 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 1367 { 1368 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 1369 } 1370 1371 void 1372 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 1373 bus_size_t size) 1374 { 1375 bus_space_free(tag, hnd, size); 1376 } 1377 1378 void 1379 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 1380 bus_size_t offset, bus_size_t size, int flags) 1381 { 1382 bus_space_barrier(tag, hnd, offset, size, flags); 1383 } 1384 1385 /* XXXMJ x86-specific */ 1386 #define MSAN_BUS_READ_FUNC(func, width, type) \ 1387 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 1388 bus_space_handle_t hnd, bus_size_t offset) \ 1389 { \ 1390 type ret; \ 1391 if ((tag) != X86_BUS_SPACE_IO) \ 1392 kmsan_shadow_fill((uintptr_t)(hnd + offset), \ 1393 KMSAN_STATE_INITED, (width)); \ 1394 ret = bus_space_read##func##_##width(tag, hnd, offset); \ 1395 kmsan_init_ret(sizeof(type)); \ 1396 return (ret); \ 1397 } \ 1398 1399 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ 1400 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 1401 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 1402 bus_size_t count) \ 1403 { \ 1404 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ 1405 (width) * count); \ 1406 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 1407 count); \ 1408 } 1409 1410 MSAN_BUS_READ_FUNC(, 1, uint8_t) 1411 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 1412 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 1413 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 1414 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 1415 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 1416 1417 MSAN_BUS_READ_FUNC(, 2, uint16_t) 1418 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 1419 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 1420 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 1421 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 1422 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 1423 1424 MSAN_BUS_READ_FUNC(, 4, uint32_t) 1425 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 1426 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 1427 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 1428 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 1429 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 1430 1431 MSAN_BUS_READ_FUNC(, 8, uint64_t) 1432 1433 #define MSAN_BUS_WRITE_FUNC(func, width, type) \ 1434 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 1435 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1436 { \ 1437 bus_space_write##func##_##width(tag, hnd, offset, value);\ 1438 } \ 1439 1440 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 1441 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ 1442 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 1443 bus_size_t count) \ 1444 { \ 1445 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 1446 "bus_space_write()"); \ 1447 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 1448 count); \ 1449 } 1450 1451 MSAN_BUS_WRITE_FUNC(, 1, uint8_t) 1452 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 1453 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 1454 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 1455 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 1456 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 1457 1458 MSAN_BUS_WRITE_FUNC(, 2, uint16_t) 1459 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 1460 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 1461 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 1462 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 1463 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 1464 1465 MSAN_BUS_WRITE_FUNC(, 4, uint32_t) 1466 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 1467 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 1468 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 1469 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 1470 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 1471 1472 MSAN_BUS_WRITE_FUNC(, 8, uint64_t) 1473 1474 #define MSAN_BUS_SET_FUNC(func, width, type) \ 1475 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 1476 bus_space_handle_t hnd, bus_size_t offset, type value, \ 1477 bus_size_t count) \ 1478 { \ 1479 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 1480 count); \ 1481 } 1482 1483 MSAN_BUS_SET_FUNC(multi, 1, uint8_t) 1484 MSAN_BUS_SET_FUNC(region, 1, uint8_t) 1485 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 1486 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 1487 1488 MSAN_BUS_SET_FUNC(multi, 2, uint16_t) 1489 MSAN_BUS_SET_FUNC(region, 2, uint16_t) 1490 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 1491 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 1492 1493 MSAN_BUS_SET_FUNC(multi, 4, uint32_t) 1494 MSAN_BUS_SET_FUNC(region, 4, uint32_t) 1495 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 1496 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 1497 1498 /* -------------------------------------------------------------------------- */ 1499 1500 void 1501 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) 1502 { 1503 /* 1504 * Some drivers, e.g., nvme, use the same code path for loading device 1505 * read and write requests, and will thus specify both flags. In this 1506 * case we should not do any checking since it will generally lead to 1507 * false positives. 1508 */ 1509 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == 1510 BUS_DMASYNC_PREWRITE) { 1511 switch (desc->md_type) { 1512 case MEMDESC_VADDR: 1513 kmsan_check(desc->u.md_vaddr, desc->md_len, 1514 "dmasync"); 1515 break; 1516 case MEMDESC_MBUF: 1517 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); 1518 break; 1519 case 0: 1520 break; 1521 default: 1522 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1523 desc->md_type); 1524 } 1525 } 1526 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1527 switch (desc->md_type) { 1528 case MEMDESC_VADDR: 1529 kmsan_mark(desc->u.md_vaddr, desc->md_len, 1530 KMSAN_STATE_INITED); 1531 break; 1532 case MEMDESC_MBUF: 1533 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); 1534 break; 1535 case 0: 1536 break; 1537 default: 1538 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1539 desc->md_type); 1540 } 1541 } 1542 } 1543