1 /* $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KASAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #define SAN_RUNTIME 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 #if 0 36 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $"); 37 #endif 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/asan.h> 42 #include <sys/kernel.h> 43 #include <sys/stack.h> 44 #include <sys/sysctl.h> 45 46 #include <machine/asan.h> 47 48 /* ASAN constants. Part of the compiler ABI. */ 49 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE - 1) 50 #define KASAN_ALLOCA_SCALE_SIZE 32 51 52 /* ASAN ABI version. */ 53 #if defined(__clang__) && (__clang_major__ - 0 >= 6) 54 #define ASAN_ABI_VERSION 8 55 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) 56 #define ASAN_ABI_VERSION 8 57 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) 58 #define ASAN_ABI_VERSION 6 59 #else 60 #error "Unsupported compiler version" 61 #endif 62 63 #define __RET_ADDR (unsigned long)__builtin_return_address(0) 64 65 /* Global variable descriptor. Part of the compiler ABI. */ 66 struct __asan_global_source_location { 67 const char *filename; 68 int line_no; 69 int column_no; 70 }; 71 72 struct __asan_global { 73 const void *beg; /* address of the global variable */ 74 size_t size; /* size of the global variable */ 75 size_t size_with_redzone; /* size with the redzone */ 76 const void *name; /* name of the variable */ 77 const void *module_name; /* name of the module where the var is declared */ 78 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ 79 struct __asan_global_source_location *location; 80 #if ASAN_ABI_VERSION >= 7 81 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ 82 #endif 83 }; 84 85 FEATURE(kasan, "Kernel address sanitizer"); 86 87 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 88 "KASAN options"); 89 90 static int panic_on_violation = 1; 91 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN, 92 &panic_on_violation, 0, 93 "Panic if an invalid access is detected"); 94 95 static bool kasan_enabled __read_mostly = false; 96 97 /* -------------------------------------------------------------------------- */ 98 99 void 100 kasan_shadow_map(vm_offset_t addr, size_t size) 101 { 102 size_t sz, npages, i; 103 vm_offset_t sva, eva; 104 105 KASSERT(addr % KASAN_SHADOW_SCALE == 0, 106 ("%s: invalid address %#lx", __func__, addr)); 107 108 sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE; 109 110 sva = kasan_md_addr_to_shad(addr); 111 eva = kasan_md_addr_to_shad(addr) + sz; 112 113 sva = rounddown(sva, PAGE_SIZE); 114 eva = roundup(eva, PAGE_SIZE); 115 116 npages = (eva - sva) / PAGE_SIZE; 117 118 KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS, 119 ("%s: invalid address range %#lx-%#lx", __func__, sva, eva)); 120 121 for (i = 0; i < npages; i++) 122 pmap_kasan_enter(sva + ptoa(i)); 123 } 124 125 void 126 kasan_init(void) 127 { 128 int disabled; 129 130 disabled = 0; 131 TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled); 132 if (disabled) 133 return; 134 135 /* MD initialization. */ 136 kasan_md_init(); 137 138 /* Now officially enabled. */ 139 kasan_enabled = true; 140 } 141 142 static inline const char * 143 kasan_code_name(uint8_t code) 144 { 145 switch (code) { 146 case KASAN_GENERIC_REDZONE: 147 return "GenericRedZone"; 148 case KASAN_MALLOC_REDZONE: 149 return "MallocRedZone"; 150 case KASAN_KMEM_REDZONE: 151 return "KmemRedZone"; 152 case KASAN_UMA_FREED: 153 return "UMAUseAfterFree"; 154 case KASAN_KSTACK_FREED: 155 return "KernelStack"; 156 case KASAN_EXEC_ARGS_FREED: 157 return "ExecKVA"; 158 case 1 ... 7: 159 return "RedZonePartial"; 160 case KASAN_STACK_LEFT: 161 return "StackLeft"; 162 case KASAN_STACK_MID: 163 return "StackMiddle"; 164 case KASAN_STACK_RIGHT: 165 return "StackRight"; 166 case KASAN_USE_AFTER_RET: 167 return "UseAfterRet"; 168 case KASAN_USE_AFTER_SCOPE: 169 return "UseAfterScope"; 170 default: 171 return "Unknown"; 172 } 173 } 174 175 #define REPORT(f, ...) do { \ 176 if (panic_on_violation) { \ 177 panic(f, __VA_ARGS__); \ 178 } else { \ 179 struct stack st; \ 180 \ 181 stack_save(&st); \ 182 printf(f "\n", __VA_ARGS__); \ 183 stack_print_ddb(&st); \ 184 } \ 185 } while (0) 186 187 static void 188 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, 189 uint8_t code) 190 { 191 REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)", 192 size, (write ? "write" : "read"), addr, kasan_code_name(code), 193 code); 194 } 195 196 static __always_inline void 197 kasan_shadow_1byte_markvalid(unsigned long addr) 198 { 199 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); 200 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 201 202 *byte = last; 203 } 204 205 static __always_inline void 206 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) 207 { 208 size_t i; 209 210 for (i = 0; i < size; i++) { 211 kasan_shadow_1byte_markvalid((unsigned long)addr + i); 212 } 213 } 214 215 static __always_inline void 216 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) 217 { 218 void *shad; 219 220 if (__predict_false(size == 0)) 221 return; 222 if (__predict_false(kasan_md_unsupported((vm_offset_t)addr))) 223 return; 224 225 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, 226 ("%s: invalid address %p", __func__, addr)); 227 KASSERT(size % KASAN_SHADOW_SCALE == 0, 228 ("%s: invalid size %zu", __func__, size)); 229 230 shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr); 231 size = size >> KASAN_SHADOW_SCALE_SHIFT; 232 233 __builtin_memset(shad, code, size); 234 } 235 236 /* 237 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, 238 * and the rest as invalid. There are generally two use cases: 239 * 240 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks 241 * the redzone at the end of the buffer as invalid. If the entire is to be 242 * marked invalid, origsize will be 0. 243 * 244 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. 245 */ 246 void 247 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code) 248 { 249 size_t i, n, redz; 250 int8_t *shad; 251 252 if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS && 253 (vm_offset_t)addr < DMAP_MAX_ADDRESS) 254 return; 255 256 KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS && 257 (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS, 258 ("%s: invalid address %p", __func__, addr)); 259 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, 260 ("%s: invalid address %p", __func__, addr)); 261 redz = redzsize - roundup(size, KASAN_SHADOW_SCALE); 262 KASSERT(redz % KASAN_SHADOW_SCALE == 0, 263 ("%s: invalid size %zu", __func__, redz)); 264 shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr); 265 266 /* Chunks of 8 bytes, valid. */ 267 n = size / KASAN_SHADOW_SCALE; 268 for (i = 0; i < n; i++) { 269 *shad++ = 0; 270 } 271 272 /* Possibly one chunk, mid. */ 273 if ((size & KASAN_SHADOW_MASK) != 0) { 274 *shad++ = (size & KASAN_SHADOW_MASK); 275 } 276 277 /* Chunks of 8 bytes, invalid. */ 278 n = redz / KASAN_SHADOW_SCALE; 279 for (i = 0; i < n; i++) { 280 *shad++ = code; 281 } 282 } 283 284 /* -------------------------------------------------------------------------- */ 285 286 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ 287 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ 288 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) 289 290 static __always_inline bool 291 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) 292 { 293 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); 294 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 295 296 if (__predict_true(*byte == 0 || last <= *byte)) { 297 return (true); 298 } 299 *code = *byte; 300 return (false); 301 } 302 303 static __always_inline bool 304 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) 305 { 306 int8_t *byte, last; 307 308 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { 309 return (kasan_shadow_1byte_isvalid(addr, code) && 310 kasan_shadow_1byte_isvalid(addr+1, code)); 311 } 312 313 byte = (int8_t *)kasan_md_addr_to_shad(addr); 314 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; 315 316 if (__predict_true(*byte == 0 || last <= *byte)) { 317 return (true); 318 } 319 *code = *byte; 320 return (false); 321 } 322 323 static __always_inline bool 324 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) 325 { 326 int8_t *byte, last; 327 328 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { 329 return (kasan_shadow_2byte_isvalid(addr, code) && 330 kasan_shadow_2byte_isvalid(addr+2, code)); 331 } 332 333 byte = (int8_t *)kasan_md_addr_to_shad(addr); 334 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; 335 336 if (__predict_true(*byte == 0 || last <= *byte)) { 337 return (true); 338 } 339 *code = *byte; 340 return (false); 341 } 342 343 static __always_inline bool 344 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) 345 { 346 int8_t *byte, last; 347 348 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { 349 return (kasan_shadow_4byte_isvalid(addr, code) && 350 kasan_shadow_4byte_isvalid(addr+4, code)); 351 } 352 353 byte = (int8_t *)kasan_md_addr_to_shad(addr); 354 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; 355 356 if (__predict_true(*byte == 0 || last <= *byte)) { 357 return (true); 358 } 359 *code = *byte; 360 return (false); 361 } 362 363 static __always_inline bool 364 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) 365 { 366 size_t i; 367 368 for (i = 0; i < size; i++) { 369 if (!kasan_shadow_1byte_isvalid(addr+i, code)) 370 return (false); 371 } 372 373 return (true); 374 } 375 376 static __always_inline void 377 kasan_shadow_check(unsigned long addr, size_t size, bool write, 378 unsigned long retaddr) 379 { 380 uint8_t code; 381 bool valid; 382 383 if (__predict_false(!kasan_enabled)) 384 return; 385 if (__predict_false(size == 0)) 386 return; 387 if (__predict_false(kasan_md_unsupported(addr))) 388 return; 389 if (__predict_false(panicstr != NULL)) 390 return; 391 392 if (__builtin_constant_p(size)) { 393 switch (size) { 394 case 1: 395 valid = kasan_shadow_1byte_isvalid(addr, &code); 396 break; 397 case 2: 398 valid = kasan_shadow_2byte_isvalid(addr, &code); 399 break; 400 case 4: 401 valid = kasan_shadow_4byte_isvalid(addr, &code); 402 break; 403 case 8: 404 valid = kasan_shadow_8byte_isvalid(addr, &code); 405 break; 406 default: 407 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 408 break; 409 } 410 } else { 411 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 412 } 413 414 if (__predict_false(!valid)) { 415 kasan_report(addr, size, write, retaddr, code); 416 } 417 } 418 419 /* -------------------------------------------------------------------------- */ 420 421 void * 422 kasan_memcpy(void *dst, const void *src, size_t len) 423 { 424 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 425 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 426 return (__builtin_memcpy(dst, src, len)); 427 } 428 429 int 430 kasan_memcmp(const void *b1, const void *b2, size_t len) 431 { 432 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); 433 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); 434 return (__builtin_memcmp(b1, b2, len)); 435 } 436 437 void * 438 kasan_memset(void *b, int c, size_t len) 439 { 440 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); 441 return (__builtin_memset(b, c, len)); 442 } 443 444 void * 445 kasan_memmove(void *dst, const void *src, size_t len) 446 { 447 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 448 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 449 return (__builtin_memmove(dst, src, len)); 450 } 451 452 size_t 453 kasan_strlen(const char *str) 454 { 455 const char *s; 456 457 s = str; 458 while (1) { 459 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); 460 if (*s == '\0') 461 break; 462 s++; 463 } 464 465 return (s - str); 466 } 467 468 char * 469 kasan_strcpy(char *dst, const char *src) 470 { 471 char *save = dst; 472 473 while (1) { 474 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); 475 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); 476 *dst = *src; 477 if (*src == '\0') 478 break; 479 src++, dst++; 480 } 481 482 return save; 483 } 484 485 int 486 kasan_strcmp(const char *s1, const char *s2) 487 { 488 while (1) { 489 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); 490 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); 491 if (*s1 != *s2) 492 break; 493 if (*s1 == '\0') 494 return 0; 495 s1++, s2++; 496 } 497 498 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 499 } 500 501 int 502 kasan_copyin(const void *uaddr, void *kaddr, size_t len) 503 { 504 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 505 return (copyin(uaddr, kaddr, len)); 506 } 507 508 int 509 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 510 { 511 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 512 return (copyinstr(uaddr, kaddr, len, done)); 513 } 514 515 int 516 kasan_copyout(const void *kaddr, void *uaddr, size_t len) 517 { 518 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); 519 return (copyout(kaddr, uaddr, len)); 520 } 521 522 /* -------------------------------------------------------------------------- */ 523 524 #include <machine/atomic.h> 525 #define ATOMIC_SAN_PREFIX kasan 526 #include <sys/atomic_san.h> 527 528 #define _ASAN_ATOMIC_FUNC_ADD(name, type) \ 529 void kasan_atomic_add_##name(volatile type *ptr, type val) \ 530 { \ 531 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 532 __RET_ADDR); \ 533 atomic_add_##name(ptr, val); \ 534 } 535 536 #define ASAN_ATOMIC_FUNC_ADD(name, type) \ 537 _ASAN_ATOMIC_FUNC_ADD(name, type) \ 538 _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 539 _ASAN_ATOMIC_FUNC_ADD(rel_##name, type) 540 541 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 542 void kasan_atomic_subtract_##name(volatile type *ptr, type val) \ 543 { \ 544 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 545 __RET_ADDR); \ 546 atomic_subtract_##name(ptr, val); \ 547 } 548 549 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 550 _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 551 _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 552 _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 553 554 #define _ASAN_ATOMIC_FUNC_SET(name, type) \ 555 void kasan_atomic_set_##name(volatile type *ptr, type val) \ 556 { \ 557 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 558 __RET_ADDR); \ 559 atomic_set_##name(ptr, val); \ 560 } 561 562 #define ASAN_ATOMIC_FUNC_SET(name, type) \ 563 _ASAN_ATOMIC_FUNC_SET(name, type) \ 564 _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \ 565 _ASAN_ATOMIC_FUNC_SET(rel_##name, type) 566 567 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ 568 void kasan_atomic_clear_##name(volatile type *ptr, type val) \ 569 { \ 570 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 571 __RET_ADDR); \ 572 atomic_clear_##name(ptr, val); \ 573 } 574 575 #define ASAN_ATOMIC_FUNC_CLEAR(name, type) \ 576 _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ 577 _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 578 _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 579 580 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \ 581 type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 582 { \ 583 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 584 __RET_ADDR); \ 585 return (atomic_fetchadd_##name(ptr, val)); \ 586 } 587 588 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 589 type kasan_atomic_readandclear_##name(volatile type *ptr) \ 590 { \ 591 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 592 __RET_ADDR); \ 593 return (atomic_readandclear_##name(ptr)); \ 594 } 595 596 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 597 int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ 598 { \ 599 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 600 __RET_ADDR); \ 601 return (atomic_testandclear_##name(ptr, v)); \ 602 } 603 604 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 605 int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \ 606 { \ 607 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 608 __RET_ADDR); \ 609 return (atomic_testandset_##name(ptr, v)); \ 610 } 611 612 #define ASAN_ATOMIC_FUNC_SWAP(name, type) \ 613 type kasan_atomic_swap_##name(volatile type *ptr, type val) \ 614 { \ 615 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 616 __RET_ADDR); \ 617 return (atomic_swap_##name(ptr, val)); \ 618 } 619 620 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ 621 int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \ 622 type nval) \ 623 { \ 624 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 625 __RET_ADDR); \ 626 return (atomic_cmpset_##name(ptr, oval, nval)); \ 627 } 628 629 #define ASAN_ATOMIC_FUNC_CMPSET(name, type) \ 630 _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ 631 _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 632 _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 633 634 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ 635 int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ 636 type nval) \ 637 { \ 638 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 639 __RET_ADDR); \ 640 return (atomic_fcmpset_##name(ptr, oval, nval)); \ 641 } 642 643 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ 644 _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ 645 _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 646 _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 647 648 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 649 void kasan_atomic_thread_fence_##name(void) \ 650 { \ 651 atomic_thread_fence_##name(); \ 652 } 653 654 #define _ASAN_ATOMIC_FUNC_LOAD(name, type) \ 655 type kasan_atomic_load_##name(volatile type *ptr) \ 656 { \ 657 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 658 __RET_ADDR); \ 659 return (atomic_load_##name(ptr)); \ 660 } 661 662 #define ASAN_ATOMIC_FUNC_LOAD(name, type) \ 663 _ASAN_ATOMIC_FUNC_LOAD(name, type) \ 664 _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type) 665 666 #define _ASAN_ATOMIC_FUNC_STORE(name, type) \ 667 void kasan_atomic_store_##name(volatile type *ptr, type val) \ 668 { \ 669 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ 670 __RET_ADDR); \ 671 atomic_store_##name(ptr, val); \ 672 } 673 674 #define ASAN_ATOMIC_FUNC_STORE(name, type) \ 675 _ASAN_ATOMIC_FUNC_STORE(name, type) \ 676 _ASAN_ATOMIC_FUNC_STORE(rel_##name, type) 677 678 ASAN_ATOMIC_FUNC_ADD(8, uint8_t); 679 ASAN_ATOMIC_FUNC_ADD(16, uint16_t); 680 ASAN_ATOMIC_FUNC_ADD(32, uint32_t); 681 ASAN_ATOMIC_FUNC_ADD(64, uint64_t); 682 ASAN_ATOMIC_FUNC_ADD(int, u_int); 683 ASAN_ATOMIC_FUNC_ADD(long, u_long); 684 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); 685 686 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); 687 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); 688 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); 689 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); 690 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int); 691 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long); 692 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); 693 694 ASAN_ATOMIC_FUNC_SET(8, uint8_t); 695 ASAN_ATOMIC_FUNC_SET(16, uint16_t); 696 ASAN_ATOMIC_FUNC_SET(32, uint32_t); 697 ASAN_ATOMIC_FUNC_SET(64, uint64_t); 698 ASAN_ATOMIC_FUNC_SET(int, u_int); 699 ASAN_ATOMIC_FUNC_SET(long, u_long); 700 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t); 701 702 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t); 703 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t); 704 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t); 705 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t); 706 ASAN_ATOMIC_FUNC_CLEAR(int, u_int); 707 ASAN_ATOMIC_FUNC_CLEAR(long, u_long); 708 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); 709 710 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); 711 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); 712 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int); 713 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long); 714 715 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); 716 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); 717 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); 718 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); 719 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); 720 721 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); 722 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); 723 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); 724 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); 725 726 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); 727 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); 728 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int); 729 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long); 730 731 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t); 732 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t); 733 ASAN_ATOMIC_FUNC_SWAP(int, u_int); 734 ASAN_ATOMIC_FUNC_SWAP(long, u_long); 735 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); 736 737 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t); 738 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t); 739 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t); 740 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t); 741 ASAN_ATOMIC_FUNC_CMPSET(int, u_int); 742 ASAN_ATOMIC_FUNC_CMPSET(long, u_long); 743 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); 744 745 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); 746 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); 747 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); 748 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); 749 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int); 750 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long); 751 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); 752 753 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t); 754 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t); 755 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t); 756 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t); 757 ASAN_ATOMIC_FUNC_LOAD(char, u_char); 758 ASAN_ATOMIC_FUNC_LOAD(short, u_short); 759 ASAN_ATOMIC_FUNC_LOAD(int, u_int); 760 ASAN_ATOMIC_FUNC_LOAD(long, u_long); 761 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); 762 763 ASAN_ATOMIC_FUNC_STORE(8, uint8_t); 764 ASAN_ATOMIC_FUNC_STORE(16, uint16_t); 765 ASAN_ATOMIC_FUNC_STORE(32, uint32_t); 766 ASAN_ATOMIC_FUNC_STORE(64, uint64_t); 767 ASAN_ATOMIC_FUNC_STORE(char, u_char); 768 ASAN_ATOMIC_FUNC_STORE(short, u_short); 769 ASAN_ATOMIC_FUNC_STORE(int, u_int); 770 ASAN_ATOMIC_FUNC_STORE(long, u_long); 771 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); 772 773 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq); 774 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel); 775 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); 776 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); 777 778 void 779 kasan_atomic_interrupt_fence(void) 780 { 781 } 782 783 /* -------------------------------------------------------------------------- */ 784 785 #include <sys/bus.h> 786 #include <machine/bus.h> 787 #define BUS_SAN_PREFIX kasan 788 #include <sys/bus_san.h> 789 790 int 791 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 792 int flags, bus_space_handle_t *handlep) 793 { 794 return (bus_space_map(tag, hnd, size, flags, handlep)); 795 } 796 797 void 798 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 799 bus_size_t size) 800 { 801 bus_space_unmap(tag, hnd, size); 802 } 803 804 int 805 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 806 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 807 { 808 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 809 } 810 811 void 812 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 813 bus_size_t size) 814 { 815 bus_space_free(tag, hnd, size); 816 } 817 818 void 819 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 820 bus_size_t offset, bus_size_t size, int flags) 821 { 822 bus_space_barrier(tag, hnd, offset, size, flags); 823 } 824 825 #define ASAN_BUS_READ_FUNC(func, width, type) \ 826 type kasan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 827 bus_space_handle_t hnd, bus_size_t offset) \ 828 { \ 829 return (bus_space_read##func##_##width(tag, hnd, \ 830 offset)); \ 831 } \ 832 833 #define ASAN_BUS_READ_PTR_FUNC(func, width, type) \ 834 void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 835 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 836 bus_size_t count) \ 837 { \ 838 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 839 false, __RET_ADDR); \ 840 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 841 count); \ 842 } 843 844 ASAN_BUS_READ_FUNC(, 1, uint8_t) 845 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t) 846 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 847 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 848 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 849 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 850 851 ASAN_BUS_READ_FUNC(, 2, uint16_t) 852 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t) 853 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 854 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 855 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 856 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 857 858 ASAN_BUS_READ_FUNC(, 4, uint32_t) 859 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t) 860 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 861 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 862 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 863 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 864 865 ASAN_BUS_READ_FUNC(, 8, uint64_t) 866 867 #define ASAN_BUS_WRITE_FUNC(func, width, type) \ 868 void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 869 bus_space_handle_t hnd, bus_size_t offset, type value) \ 870 { \ 871 bus_space_write##func##_##width(tag, hnd, offset, value);\ 872 } \ 873 874 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 875 void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ 876 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 877 bus_size_t count) \ 878 { \ 879 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 880 true, __RET_ADDR); \ 881 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 882 count); \ 883 } 884 885 ASAN_BUS_WRITE_FUNC(, 1, uint8_t) 886 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 887 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 888 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 889 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 890 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 891 892 ASAN_BUS_WRITE_FUNC(, 2, uint16_t) 893 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 894 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 895 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 896 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 897 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 898 899 ASAN_BUS_WRITE_FUNC(, 4, uint32_t) 900 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 901 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 902 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 903 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 904 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 905 906 ASAN_BUS_WRITE_FUNC(, 8, uint64_t) 907 908 #define ASAN_BUS_SET_FUNC(func, width, type) \ 909 void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 910 bus_space_handle_t hnd, bus_size_t offset, type value, \ 911 bus_size_t count) \ 912 { \ 913 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 914 count); \ 915 } 916 917 ASAN_BUS_SET_FUNC(multi, 1, uint8_t) 918 ASAN_BUS_SET_FUNC(region, 1, uint8_t) 919 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 920 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 921 922 ASAN_BUS_SET_FUNC(multi, 2, uint16_t) 923 ASAN_BUS_SET_FUNC(region, 2, uint16_t) 924 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 925 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 926 927 ASAN_BUS_SET_FUNC(multi, 4, uint32_t) 928 ASAN_BUS_SET_FUNC(region, 4, uint32_t) 929 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 930 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 931 932 /* -------------------------------------------------------------------------- */ 933 934 void __asan_register_globals(struct __asan_global *, size_t); 935 void __asan_unregister_globals(struct __asan_global *, size_t); 936 937 void 938 __asan_register_globals(struct __asan_global *globals, size_t n) 939 { 940 size_t i; 941 942 for (i = 0; i < n; i++) { 943 kasan_mark(globals[i].beg, globals[i].size, 944 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); 945 } 946 } 947 948 void 949 __asan_unregister_globals(struct __asan_global *globals, size_t n) 950 { 951 /* never called */ 952 } 953 954 #define ASAN_LOAD_STORE(size) \ 955 void __asan_load##size(unsigned long); \ 956 void __asan_load##size(unsigned long addr) \ 957 { \ 958 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 959 } \ 960 void __asan_load##size##_noabort(unsigned long); \ 961 void __asan_load##size##_noabort(unsigned long addr) \ 962 { \ 963 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 964 } \ 965 void __asan_store##size(unsigned long); \ 966 void __asan_store##size(unsigned long addr) \ 967 { \ 968 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 969 } \ 970 void __asan_store##size##_noabort(unsigned long); \ 971 void __asan_store##size##_noabort(unsigned long addr) \ 972 { \ 973 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 974 } 975 976 ASAN_LOAD_STORE(1); 977 ASAN_LOAD_STORE(2); 978 ASAN_LOAD_STORE(4); 979 ASAN_LOAD_STORE(8); 980 ASAN_LOAD_STORE(16); 981 982 void __asan_loadN(unsigned long, size_t); 983 void __asan_loadN_noabort(unsigned long, size_t); 984 void __asan_storeN(unsigned long, size_t); 985 void __asan_storeN_noabort(unsigned long, size_t); 986 void __asan_handle_no_return(void); 987 988 void 989 __asan_loadN(unsigned long addr, size_t size) 990 { 991 kasan_shadow_check(addr, size, false, __RET_ADDR); 992 } 993 994 void 995 __asan_loadN_noabort(unsigned long addr, size_t size) 996 { 997 kasan_shadow_check(addr, size, false, __RET_ADDR); 998 } 999 1000 void 1001 __asan_storeN(unsigned long addr, size_t size) 1002 { 1003 kasan_shadow_check(addr, size, true, __RET_ADDR); 1004 } 1005 1006 void 1007 __asan_storeN_noabort(unsigned long addr, size_t size) 1008 { 1009 kasan_shadow_check(addr, size, true, __RET_ADDR); 1010 } 1011 1012 void 1013 __asan_handle_no_return(void) 1014 { 1015 /* nothing */ 1016 } 1017 1018 #define ASAN_SET_SHADOW(byte) \ 1019 void __asan_set_shadow_##byte(void *, size_t); \ 1020 void __asan_set_shadow_##byte(void *addr, size_t size) \ 1021 { \ 1022 __builtin_memset((void *)addr, 0x##byte, size); \ 1023 } 1024 1025 ASAN_SET_SHADOW(00); 1026 ASAN_SET_SHADOW(f1); 1027 ASAN_SET_SHADOW(f2); 1028 ASAN_SET_SHADOW(f3); 1029 ASAN_SET_SHADOW(f5); 1030 ASAN_SET_SHADOW(f8); 1031 1032 void __asan_poison_stack_memory(const void *, size_t); 1033 void __asan_unpoison_stack_memory(const void *, size_t); 1034 1035 void 1036 __asan_poison_stack_memory(const void *addr, size_t size) 1037 { 1038 size = roundup(size, KASAN_SHADOW_SCALE); 1039 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); 1040 } 1041 1042 void 1043 __asan_unpoison_stack_memory(const void *addr, size_t size) 1044 { 1045 kasan_shadow_Nbyte_markvalid(addr, size); 1046 } 1047 1048 void __asan_alloca_poison(const void *, size_t); 1049 void __asan_allocas_unpoison(const void *, const void *); 1050 1051 void 1052 __asan_alloca_poison(const void *addr, size_t size) 1053 { 1054 const void *l, *r; 1055 1056 KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0, 1057 ("%s: invalid address %p", __func__, addr)); 1058 1059 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; 1060 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); 1061 1062 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); 1063 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), 1064 KASAN_STACK_MID); 1065 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); 1066 } 1067 1068 void 1069 __asan_allocas_unpoison(const void *stkbegin, const void *stkend) 1070 { 1071 size_t size; 1072 1073 if (__predict_false(!stkbegin)) 1074 return; 1075 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) 1076 return; 1077 size = (uintptr_t)stkend - (uintptr_t)stkbegin; 1078 1079 kasan_shadow_Nbyte_fill(stkbegin, size, 0); 1080 } 1081 1082 void __asan_poison_memory_region(const void *addr, size_t size); 1083 void __asan_unpoison_memory_region(const void *addr, size_t size); 1084 1085 void 1086 __asan_poison_memory_region(const void *addr, size_t size) 1087 { 1088 } 1089 1090 void 1091 __asan_unpoison_memory_region(const void *addr, size_t size) 1092 { 1093 } 1094