1 /* $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KASAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #define SAN_RUNTIME 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 #if 0 36 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $"); 37 #endif 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/asan.h> 42 #include <sys/kernel.h> 43 #include <sys/stack.h> 44 #include <sys/sysctl.h> 45 46 #include <machine/asan.h> 47 48 /* ASAN constants. Part of the compiler ABI. */ 49 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE - 1) 50 #define KASAN_ALLOCA_SCALE_SIZE 32 51 52 /* ASAN ABI version. */ 53 #if defined(__clang__) && (__clang_major__ - 0 >= 6) 54 #define ASAN_ABI_VERSION 8 55 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) 56 #define ASAN_ABI_VERSION 8 57 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) 58 #define ASAN_ABI_VERSION 6 59 #else 60 #error "Unsupported compiler version" 61 #endif 62 63 #define __RET_ADDR (unsigned long)__builtin_return_address(0) 64 65 /* Global variable descriptor. Part of the compiler ABI. */ 66 struct __asan_global_source_location { 67 const char *filename; 68 int line_no; 69 int column_no; 70 }; 71 72 struct __asan_global { 73 const void *beg; /* address of the global variable */ 74 size_t size; /* size of the global variable */ 75 size_t size_with_redzone; /* size with the redzone */ 76 const void *name; /* name of the variable */ 77 const void *module_name; /* name of the module where the var is declared */ 78 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ 79 struct __asan_global_source_location *location; 80 #if ASAN_ABI_VERSION >= 7 81 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ 82 #endif 83 }; 84 85 FEATURE(kasan, "Kernel address sanitizer"); 86 87 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 88 "KASAN options"); 89 90 static int panic_on_violation = 1; 91 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN, 92 &panic_on_violation, 0, 93 "Panic if an invalid access is detected"); 94 95 static bool kasan_enabled __read_mostly = false; 96 97 /* -------------------------------------------------------------------------- */ 98 99 void 100 kasan_shadow_map(vm_offset_t addr, size_t size) 101 { 102 size_t sz, npages, i; 103 vm_offset_t sva, eva; 104 105 KASSERT(addr % KASAN_SHADOW_SCALE == 0, 106 ("%s: invalid address %#lx", __func__, addr)); 107 108 sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE; 109 110 sva = kasan_md_addr_to_shad(addr); 111 eva = kasan_md_addr_to_shad(addr) + sz; 112 113 sva = rounddown(sva, PAGE_SIZE); 114 eva = roundup(eva, PAGE_SIZE); 115 116 npages = (eva - sva) / PAGE_SIZE; 117 118 KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS, 119 ("%s: invalid address range %#lx-%#lx", __func__, sva, eva)); 120 121 for (i = 0; i < npages; i++) 122 pmap_kasan_enter(sva + ptoa(i)); 123 } 124 125 void 126 kasan_init(void) 127 { 128 int disabled; 129 130 disabled = 0; 131 TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled); 132 if (disabled) 133 return; 134 135 /* MD initialization. */ 136 kasan_md_init(); 137 138 /* Now officially enabled. */ 139 kasan_enabled = true; 140 } 141 142 static inline const char * 143 kasan_code_name(uint8_t code) 144 { 145 switch (code) { 146 case KASAN_GENERIC_REDZONE: 147 return "GenericRedZone"; 148 case KASAN_MALLOC_REDZONE: 149 return "MallocRedZone"; 150 case KASAN_KMEM_REDZONE: 151 return "KmemRedZone"; 152 case KASAN_UMA_FREED: 153 return "UMAUseAfterFree"; 154 case KASAN_KSTACK_FREED: 155 return "KernelStack"; 156 case KASAN_EXEC_ARGS_FREED: 157 return "ExecKVA"; 158 case 1 ... 7: 159 return "RedZonePartial"; 160 case KASAN_STACK_LEFT: 161 return "StackLeft"; 162 case KASAN_STACK_MID: 163 return "StackMiddle"; 164 case KASAN_STACK_RIGHT: 165 return "StackRight"; 166 case KASAN_USE_AFTER_RET: 167 return "UseAfterRet"; 168 case KASAN_USE_AFTER_SCOPE: 169 return "UseAfterScope"; 170 default: 171 return "Unknown"; 172 } 173 } 174 175 #define REPORT(f, ...) do { \ 176 if (panic_on_violation) { \ 177 kasan_enabled = false; \ 178 panic(f, __VA_ARGS__); \ 179 } else { \ 180 struct stack st; \ 181 \ 182 stack_save(&st); \ 183 printf(f "\n", __VA_ARGS__); \ 184 stack_print_ddb(&st); \ 185 } \ 186 } while (0) 187 188 static void 189 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, 190 uint8_t code) 191 { 192 REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)", 193 size, (write ? "write" : "read"), addr, kasan_code_name(code), 194 code); 195 } 196 197 static __always_inline void 198 kasan_shadow_1byte_markvalid(unsigned long addr) 199 { 200 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); 201 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 202 203 *byte = last; 204 } 205 206 static __always_inline void 207 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) 208 { 209 size_t i; 210 211 for (i = 0; i < size; i++) { 212 kasan_shadow_1byte_markvalid((unsigned long)addr + i); 213 } 214 } 215 216 static __always_inline void 217 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) 218 { 219 void *shad; 220 221 if (__predict_false(size == 0)) 222 return; 223 if (__predict_false(kasan_md_unsupported((vm_offset_t)addr))) 224 return; 225 226 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, 227 ("%s: invalid address %p", __func__, addr)); 228 KASSERT(size % KASAN_SHADOW_SCALE == 0, 229 ("%s: invalid size %zu", __func__, size)); 230 231 shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr); 232 size = size >> KASAN_SHADOW_SCALE_SHIFT; 233 234 __builtin_memset(shad, code, size); 235 } 236 237 /* 238 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, 239 * and the rest as invalid. There are generally two use cases: 240 * 241 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks 242 * the redzone at the end of the buffer as invalid. If the entire is to be 243 * marked invalid, origsize will be 0. 244 * 245 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. 246 */ 247 void 248 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code) 249 { 250 size_t i, n, redz; 251 int8_t *shad; 252 253 if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS && 254 (vm_offset_t)addr < DMAP_MAX_ADDRESS) 255 return; 256 257 KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS && 258 (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS, 259 ("%s: invalid address %p", __func__, addr)); 260 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, 261 ("%s: invalid address %p", __func__, addr)); 262 redz = redzsize - roundup(size, KASAN_SHADOW_SCALE); 263 KASSERT(redz % KASAN_SHADOW_SCALE == 0, 264 ("%s: invalid size %zu", __func__, redz)); 265 shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr); 266 267 /* Chunks of 8 bytes, valid. */ 268 n = size / KASAN_SHADOW_SCALE; 269 for (i = 0; i < n; i++) { 270 *shad++ = 0; 271 } 272 273 /* Possibly one chunk, mid. */ 274 if ((size & KASAN_SHADOW_MASK) != 0) { 275 *shad++ = (size & KASAN_SHADOW_MASK); 276 } 277 278 /* Chunks of 8 bytes, invalid. */ 279 n = redz / KASAN_SHADOW_SCALE; 280 for (i = 0; i < n; i++) { 281 *shad++ = code; 282 } 283 } 284 285 /* -------------------------------------------------------------------------- */ 286 287 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ 288 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ 289 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) 290 291 static __always_inline bool 292 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) 293 { 294 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); 295 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 296 297 if (__predict_true(*byte == 0 || last <= *byte)) { 298 return (true); 299 } 300 *code = *byte; 301 return (false); 302 } 303 304 static __always_inline bool 305 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) 306 { 307 int8_t *byte, last; 308 309 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { 310 return (kasan_shadow_1byte_isvalid(addr, code) && 311 kasan_shadow_1byte_isvalid(addr+1, code)); 312 } 313 314 byte = (int8_t *)kasan_md_addr_to_shad(addr); 315 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; 316 317 if (__predict_true(*byte == 0 || last <= *byte)) { 318 return (true); 319 } 320 *code = *byte; 321 return (false); 322 } 323 324 static __always_inline bool 325 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) 326 { 327 int8_t *byte, last; 328 329 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { 330 return (kasan_shadow_2byte_isvalid(addr, code) && 331 kasan_shadow_2byte_isvalid(addr+2, code)); 332 } 333 334 byte = (int8_t *)kasan_md_addr_to_shad(addr); 335 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; 336 337 if (__predict_true(*byte == 0 || last <= *byte)) { 338 return (true); 339 } 340 *code = *byte; 341 return (false); 342 } 343 344 static __always_inline bool 345 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) 346 { 347 int8_t *byte, last; 348 349 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { 350 return (kasan_shadow_4byte_isvalid(addr, code) && 351 kasan_shadow_4byte_isvalid(addr+4, code)); 352 } 353 354 byte = (int8_t *)kasan_md_addr_to_shad(addr); 355 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; 356 357 if (__predict_true(*byte == 0 || last <= *byte)) { 358 return (true); 359 } 360 *code = *byte; 361 return (false); 362 } 363 364 static __always_inline bool 365 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) 366 { 367 size_t i; 368 369 for (i = 0; i < size; i++) { 370 if (!kasan_shadow_1byte_isvalid(addr+i, code)) 371 return (false); 372 } 373 374 return (true); 375 } 376 377 static __always_inline void 378 kasan_shadow_check(unsigned long addr, size_t size, bool write, 379 unsigned long retaddr) 380 { 381 uint8_t code; 382 bool valid; 383 384 if (__predict_false(!kasan_enabled)) 385 return; 386 if (__predict_false(size == 0)) 387 return; 388 if (__predict_false(kasan_md_unsupported(addr))) 389 return; 390 if (__predict_false(panicstr != NULL)) 391 return; 392 393 if (__builtin_constant_p(size)) { 394 switch (size) { 395 case 1: 396 valid = kasan_shadow_1byte_isvalid(addr, &code); 397 break; 398 case 2: 399 valid = kasan_shadow_2byte_isvalid(addr, &code); 400 break; 401 case 4: 402 valid = kasan_shadow_4byte_isvalid(addr, &code); 403 break; 404 case 8: 405 valid = kasan_shadow_8byte_isvalid(addr, &code); 406 break; 407 default: 408 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 409 break; 410 } 411 } else { 412 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 413 } 414 415 if (__predict_false(!valid)) { 416 kasan_report(addr, size, write, retaddr, code); 417 } 418 } 419 420 /* -------------------------------------------------------------------------- */ 421 422 void * 423 kasan_memcpy(void *dst, const void *src, size_t len) 424 { 425 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 426 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 427 return (__builtin_memcpy(dst, src, len)); 428 } 429 430 int 431 kasan_memcmp(const void *b1, const void *b2, size_t len) 432 { 433 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); 434 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); 435 return (__builtin_memcmp(b1, b2, len)); 436 } 437 438 void * 439 kasan_memset(void *b, int c, size_t len) 440 { 441 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); 442 return (__builtin_memset(b, c, len)); 443 } 444 445 void * 446 kasan_memmove(void *dst, const void *src, size_t len) 447 { 448 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 449 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 450 return (__builtin_memmove(dst, src, len)); 451 } 452 453 size_t 454 kasan_strlen(const char *str) 455 { 456 const char *s; 457 458 s = str; 459 while (1) { 460 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); 461 if (*s == '\0') 462 break; 463 s++; 464 } 465 466 return (s - str); 467 } 468 469 char * 470 kasan_strcpy(char *dst, const char *src) 471 { 472 char *save = dst; 473 474 while (1) { 475 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); 476 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); 477 *dst = *src; 478 if (*src == '\0') 479 break; 480 src++, dst++; 481 } 482 483 return save; 484 } 485 486 int 487 kasan_strcmp(const char *s1, const char *s2) 488 { 489 while (1) { 490 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); 491 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); 492 if (*s1 != *s2) 493 break; 494 if (*s1 == '\0') 495 return 0; 496 s1++, s2++; 497 } 498 499 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 500 } 501 502 int 503 kasan_copyin(const void *uaddr, void *kaddr, size_t len) 504 { 505 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 506 return (copyin(uaddr, kaddr, len)); 507 } 508 509 int 510 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 511 { 512 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 513 return (copyinstr(uaddr, kaddr, len, done)); 514 } 515 516 int 517 kasan_copyout(const void *kaddr, void *uaddr, size_t len) 518 { 519 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); 520 return (copyout(kaddr, uaddr, len)); 521 } 522 523 /* -------------------------------------------------------------------------- */ 524 525 #include <machine/atomic.h> 526 #define ATOMIC_SAN_PREFIX kasan 527 #include <sys/atomic_san.h> 528 529 #define _ASAN_ATOMIC_FUNC_ADD(name, type) \ 530 void kasan_atomic_add_##name(volatile type *ptr, type val) \ 531 { \ 532 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 533 __RET_ADDR); \ 534 atomic_add_##name(ptr, val); \ 535 } 536 537 #define ASAN_ATOMIC_FUNC_ADD(name, type) \ 538 _ASAN_ATOMIC_FUNC_ADD(name, type) \ 539 _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 540 _ASAN_ATOMIC_FUNC_ADD(rel_##name, type) 541 542 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 543 void kasan_atomic_subtract_##name(volatile type *ptr, type val) \ 544 { \ 545 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 546 __RET_ADDR); \ 547 atomic_subtract_##name(ptr, val); \ 548 } 549 550 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 551 _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 552 _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 553 _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 554 555 #define _ASAN_ATOMIC_FUNC_SET(name, type) \ 556 void kasan_atomic_set_##name(volatile type *ptr, type val) \ 557 { \ 558 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 559 __RET_ADDR); \ 560 atomic_set_##name(ptr, val); \ 561 } 562 563 #define ASAN_ATOMIC_FUNC_SET(name, type) \ 564 _ASAN_ATOMIC_FUNC_SET(name, type) \ 565 _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \ 566 _ASAN_ATOMIC_FUNC_SET(rel_##name, type) 567 568 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ 569 void kasan_atomic_clear_##name(volatile type *ptr, type val) \ 570 { \ 571 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 572 __RET_ADDR); \ 573 atomic_clear_##name(ptr, val); \ 574 } 575 576 #define ASAN_ATOMIC_FUNC_CLEAR(name, type) \ 577 _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ 578 _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 579 _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 580 581 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \ 582 type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 583 { \ 584 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 585 __RET_ADDR); \ 586 return (atomic_fetchadd_##name(ptr, val)); \ 587 } 588 589 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 590 type kasan_atomic_readandclear_##name(volatile type *ptr) \ 591 { \ 592 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 593 __RET_ADDR); \ 594 return (atomic_readandclear_##name(ptr)); \ 595 } 596 597 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 598 int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ 599 { \ 600 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 601 __RET_ADDR); \ 602 return (atomic_testandclear_##name(ptr, v)); \ 603 } 604 605 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 606 int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \ 607 { \ 608 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 609 __RET_ADDR); \ 610 return (atomic_testandset_##name(ptr, v)); \ 611 } 612 613 #define ASAN_ATOMIC_FUNC_SWAP(name, type) \ 614 type kasan_atomic_swap_##name(volatile type *ptr, type val) \ 615 { \ 616 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 617 __RET_ADDR); \ 618 return (atomic_swap_##name(ptr, val)); \ 619 } 620 621 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ 622 int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \ 623 type nval) \ 624 { \ 625 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 626 __RET_ADDR); \ 627 return (atomic_cmpset_##name(ptr, oval, nval)); \ 628 } 629 630 #define ASAN_ATOMIC_FUNC_CMPSET(name, type) \ 631 _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ 632 _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 633 _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 634 635 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ 636 int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ 637 type nval) \ 638 { \ 639 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 640 __RET_ADDR); \ 641 return (atomic_fcmpset_##name(ptr, oval, nval)); \ 642 } 643 644 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ 645 _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ 646 _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 647 _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 648 649 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 650 void kasan_atomic_thread_fence_##name(void) \ 651 { \ 652 atomic_thread_fence_##name(); \ 653 } 654 655 #define _ASAN_ATOMIC_FUNC_LOAD(name, type) \ 656 type kasan_atomic_load_##name(volatile type *ptr) \ 657 { \ 658 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 659 __RET_ADDR); \ 660 return (atomic_load_##name(ptr)); \ 661 } 662 663 #define ASAN_ATOMIC_FUNC_LOAD(name, type) \ 664 _ASAN_ATOMIC_FUNC_LOAD(name, type) \ 665 _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type) 666 667 #define _ASAN_ATOMIC_FUNC_STORE(name, type) \ 668 void kasan_atomic_store_##name(volatile type *ptr, type val) \ 669 { \ 670 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 671 __RET_ADDR); \ 672 atomic_store_##name(ptr, val); \ 673 } 674 675 #define ASAN_ATOMIC_FUNC_STORE(name, type) \ 676 _ASAN_ATOMIC_FUNC_STORE(name, type) \ 677 _ASAN_ATOMIC_FUNC_STORE(rel_##name, type) 678 679 ASAN_ATOMIC_FUNC_ADD(8, uint8_t); 680 ASAN_ATOMIC_FUNC_ADD(16, uint16_t); 681 ASAN_ATOMIC_FUNC_ADD(32, uint32_t); 682 ASAN_ATOMIC_FUNC_ADD(64, uint64_t); 683 ASAN_ATOMIC_FUNC_ADD(int, u_int); 684 ASAN_ATOMIC_FUNC_ADD(long, u_long); 685 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); 686 687 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); 688 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); 689 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); 690 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); 691 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int); 692 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long); 693 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); 694 695 ASAN_ATOMIC_FUNC_SET(8, uint8_t); 696 ASAN_ATOMIC_FUNC_SET(16, uint16_t); 697 ASAN_ATOMIC_FUNC_SET(32, uint32_t); 698 ASAN_ATOMIC_FUNC_SET(64, uint64_t); 699 ASAN_ATOMIC_FUNC_SET(int, u_int); 700 ASAN_ATOMIC_FUNC_SET(long, u_long); 701 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t); 702 703 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t); 704 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t); 705 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t); 706 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t); 707 ASAN_ATOMIC_FUNC_CLEAR(int, u_int); 708 ASAN_ATOMIC_FUNC_CLEAR(long, u_long); 709 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); 710 711 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); 712 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); 713 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int); 714 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long); 715 716 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); 717 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); 718 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); 719 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); 720 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); 721 722 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); 723 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); 724 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); 725 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); 726 727 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); 728 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); 729 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int); 730 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long); 731 732 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t); 733 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t); 734 ASAN_ATOMIC_FUNC_SWAP(int, u_int); 735 ASAN_ATOMIC_FUNC_SWAP(long, u_long); 736 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); 737 738 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t); 739 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t); 740 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t); 741 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t); 742 ASAN_ATOMIC_FUNC_CMPSET(int, u_int); 743 ASAN_ATOMIC_FUNC_CMPSET(long, u_long); 744 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); 745 746 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); 747 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); 748 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); 749 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); 750 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int); 751 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long); 752 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); 753 754 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t); 755 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t); 756 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t); 757 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t); 758 ASAN_ATOMIC_FUNC_LOAD(char, u_char); 759 ASAN_ATOMIC_FUNC_LOAD(short, u_short); 760 ASAN_ATOMIC_FUNC_LOAD(int, u_int); 761 ASAN_ATOMIC_FUNC_LOAD(long, u_long); 762 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); 763 764 ASAN_ATOMIC_FUNC_STORE(8, uint8_t); 765 ASAN_ATOMIC_FUNC_STORE(16, uint16_t); 766 ASAN_ATOMIC_FUNC_STORE(32, uint32_t); 767 ASAN_ATOMIC_FUNC_STORE(64, uint64_t); 768 ASAN_ATOMIC_FUNC_STORE(char, u_char); 769 ASAN_ATOMIC_FUNC_STORE(short, u_short); 770 ASAN_ATOMIC_FUNC_STORE(int, u_int); 771 ASAN_ATOMIC_FUNC_STORE(long, u_long); 772 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); 773 774 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq); 775 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel); 776 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); 777 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); 778 779 void 780 kasan_atomic_interrupt_fence(void) 781 { 782 } 783 784 /* -------------------------------------------------------------------------- */ 785 786 #include <sys/bus.h> 787 #include <machine/bus.h> 788 #define BUS_SAN_PREFIX kasan 789 #include <sys/bus_san.h> 790 791 int 792 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 793 int flags, bus_space_handle_t *handlep) 794 { 795 return (bus_space_map(tag, hnd, size, flags, handlep)); 796 } 797 798 void 799 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 800 bus_size_t size) 801 { 802 bus_space_unmap(tag, hnd, size); 803 } 804 805 int 806 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 807 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 808 { 809 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 810 } 811 812 void 813 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 814 bus_size_t size) 815 { 816 bus_space_free(tag, hnd, size); 817 } 818 819 void 820 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 821 bus_size_t offset, bus_size_t size, int flags) 822 { 823 bus_space_barrier(tag, hnd, offset, size, flags); 824 } 825 826 #define ASAN_BUS_READ_FUNC(func, width, type) \ 827 type kasan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 828 bus_space_handle_t hnd, bus_size_t offset) \ 829 { \ 830 return (bus_space_read##func##_##width(tag, hnd, \ 831 offset)); \ 832 } \ 833 834 #define ASAN_BUS_READ_PTR_FUNC(func, width, type) \ 835 void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 836 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 837 bus_size_t count) \ 838 { \ 839 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 840 false, __RET_ADDR); \ 841 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 842 count); \ 843 } 844 845 ASAN_BUS_READ_FUNC(, 1, uint8_t) 846 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t) 847 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 848 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 849 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 850 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 851 852 ASAN_BUS_READ_FUNC(, 2, uint16_t) 853 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t) 854 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 855 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 856 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 857 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 858 859 ASAN_BUS_READ_FUNC(, 4, uint32_t) 860 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t) 861 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 862 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 863 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 864 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 865 866 ASAN_BUS_READ_FUNC(, 8, uint64_t) 867 868 #define ASAN_BUS_WRITE_FUNC(func, width, type) \ 869 void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 870 bus_space_handle_t hnd, bus_size_t offset, type value) \ 871 { \ 872 bus_space_write##func##_##width(tag, hnd, offset, value);\ 873 } \ 874 875 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 876 void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ 877 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 878 bus_size_t count) \ 879 { \ 880 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 881 true, __RET_ADDR); \ 882 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 883 count); \ 884 } 885 886 ASAN_BUS_WRITE_FUNC(, 1, uint8_t) 887 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 888 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 889 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 890 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 891 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 892 893 ASAN_BUS_WRITE_FUNC(, 2, uint16_t) 894 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 895 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 896 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 897 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 898 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 899 900 ASAN_BUS_WRITE_FUNC(, 4, uint32_t) 901 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 902 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 903 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 904 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 905 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 906 907 ASAN_BUS_WRITE_FUNC(, 8, uint64_t) 908 909 #define ASAN_BUS_SET_FUNC(func, width, type) \ 910 void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 911 bus_space_handle_t hnd, bus_size_t offset, type value, \ 912 bus_size_t count) \ 913 { \ 914 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 915 count); \ 916 } 917 918 ASAN_BUS_SET_FUNC(multi, 1, uint8_t) 919 ASAN_BUS_SET_FUNC(region, 1, uint8_t) 920 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 921 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 922 923 ASAN_BUS_SET_FUNC(multi, 2, uint16_t) 924 ASAN_BUS_SET_FUNC(region, 2, uint16_t) 925 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 926 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 927 928 ASAN_BUS_SET_FUNC(multi, 4, uint32_t) 929 ASAN_BUS_SET_FUNC(region, 4, uint32_t) 930 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 931 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 932 933 /* -------------------------------------------------------------------------- */ 934 935 void __asan_register_globals(struct __asan_global *, size_t); 936 void __asan_unregister_globals(struct __asan_global *, size_t); 937 938 void 939 __asan_register_globals(struct __asan_global *globals, size_t n) 940 { 941 size_t i; 942 943 for (i = 0; i < n; i++) { 944 kasan_mark(globals[i].beg, globals[i].size, 945 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); 946 } 947 } 948 949 void 950 __asan_unregister_globals(struct __asan_global *globals, size_t n) 951 { 952 size_t i; 953 954 for (i = 0; i < n; i++) { 955 kasan_mark(globals[i].beg, globals[i].size_with_redzone, 956 globals[i].size_with_redzone, 0); 957 } 958 } 959 960 #define ASAN_LOAD_STORE(size) \ 961 void __asan_load##size(unsigned long); \ 962 void __asan_load##size(unsigned long addr) \ 963 { \ 964 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 965 } \ 966 void __asan_load##size##_noabort(unsigned long); \ 967 void __asan_load##size##_noabort(unsigned long addr) \ 968 { \ 969 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 970 } \ 971 void __asan_store##size(unsigned long); \ 972 void __asan_store##size(unsigned long addr) \ 973 { \ 974 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 975 } \ 976 void __asan_store##size##_noabort(unsigned long); \ 977 void __asan_store##size##_noabort(unsigned long addr) \ 978 { \ 979 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 980 } 981 982 ASAN_LOAD_STORE(1); 983 ASAN_LOAD_STORE(2); 984 ASAN_LOAD_STORE(4); 985 ASAN_LOAD_STORE(8); 986 ASAN_LOAD_STORE(16); 987 988 void __asan_loadN(unsigned long, size_t); 989 void __asan_loadN_noabort(unsigned long, size_t); 990 void __asan_storeN(unsigned long, size_t); 991 void __asan_storeN_noabort(unsigned long, size_t); 992 void __asan_handle_no_return(void); 993 994 void 995 __asan_loadN(unsigned long addr, size_t size) 996 { 997 kasan_shadow_check(addr, size, false, __RET_ADDR); 998 } 999 1000 void 1001 __asan_loadN_noabort(unsigned long addr, size_t size) 1002 { 1003 kasan_shadow_check(addr, size, false, __RET_ADDR); 1004 } 1005 1006 void 1007 __asan_storeN(unsigned long addr, size_t size) 1008 { 1009 kasan_shadow_check(addr, size, true, __RET_ADDR); 1010 } 1011 1012 void 1013 __asan_storeN_noabort(unsigned long addr, size_t size) 1014 { 1015 kasan_shadow_check(addr, size, true, __RET_ADDR); 1016 } 1017 1018 void 1019 __asan_handle_no_return(void) 1020 { 1021 /* nothing */ 1022 } 1023 1024 #define ASAN_SET_SHADOW(byte) \ 1025 void __asan_set_shadow_##byte(void *, size_t); \ 1026 void __asan_set_shadow_##byte(void *addr, size_t size) \ 1027 { \ 1028 __builtin_memset((void *)addr, 0x##byte, size); \ 1029 } 1030 1031 ASAN_SET_SHADOW(00); 1032 ASAN_SET_SHADOW(f1); 1033 ASAN_SET_SHADOW(f2); 1034 ASAN_SET_SHADOW(f3); 1035 ASAN_SET_SHADOW(f5); 1036 ASAN_SET_SHADOW(f8); 1037 1038 void __asan_poison_stack_memory(const void *, size_t); 1039 void __asan_unpoison_stack_memory(const void *, size_t); 1040 1041 void 1042 __asan_poison_stack_memory(const void *addr, size_t size) 1043 { 1044 size = roundup(size, KASAN_SHADOW_SCALE); 1045 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); 1046 } 1047 1048 void 1049 __asan_unpoison_stack_memory(const void *addr, size_t size) 1050 { 1051 kasan_shadow_Nbyte_markvalid(addr, size); 1052 } 1053 1054 void __asan_alloca_poison(const void *, size_t); 1055 void __asan_allocas_unpoison(const void *, const void *); 1056 1057 void 1058 __asan_alloca_poison(const void *addr, size_t size) 1059 { 1060 const void *l, *r; 1061 1062 KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0, 1063 ("%s: invalid address %p", __func__, addr)); 1064 1065 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; 1066 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); 1067 1068 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); 1069 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), 1070 KASAN_STACK_MID); 1071 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); 1072 } 1073 1074 void 1075 __asan_allocas_unpoison(const void *stkbegin, const void *stkend) 1076 { 1077 size_t size; 1078 1079 if (__predict_false(!stkbegin)) 1080 return; 1081 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) 1082 return; 1083 size = (uintptr_t)stkend - (uintptr_t)stkbegin; 1084 1085 kasan_shadow_Nbyte_fill(stkbegin, size, 0); 1086 } 1087 1088 void __asan_poison_memory_region(const void *addr, size_t size); 1089 void __asan_unpoison_memory_region(const void *addr, size_t size); 1090 1091 void 1092 __asan_poison_memory_region(const void *addr, size_t size) 1093 { 1094 } 1095 1096 void 1097 __asan_unpoison_memory_region(const void *addr, size_t size) 1098 { 1099 } 1100