1 /* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * Copyright (c) 2019 Andrew Turner 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Maxime Villard. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #define SAN_RUNTIME 34 35 #include "opt_ddb.h" 36 37 #include <sys/cdefs.h> 38 #include <sys/param.h> 39 #include <sys/kernel.h> 40 #include <sys/bus.h> 41 #include <sys/conf.h> 42 #include <sys/cpu.h> 43 #include <sys/csan.h> 44 #include <sys/proc.h> 45 #include <sys/smp.h> 46 #include <sys/systm.h> 47 48 #include <ddb/ddb.h> 49 #include <ddb/db_sym.h> 50 51 #ifdef KCSAN_PANIC 52 #define REPORT panic 53 #else 54 #define REPORT printf 55 #endif 56 57 typedef struct { 58 uintptr_t addr; 59 uint32_t size; 60 bool write:1; 61 bool atomic:1; 62 uintptr_t pc; 63 } csan_cell_t; 64 65 typedef struct { 66 bool inited; 67 uint32_t cnt; 68 csan_cell_t cell; 69 } csan_cpu_t; 70 71 static csan_cpu_t kcsan_cpus[MAXCPU]; 72 static bool kcsan_enabled __read_mostly; 73 74 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 75 76 #define KCSAN_NACCESSES 1024 77 #define KCSAN_DELAY 10 /* 10 microseconds */ 78 79 /* -------------------------------------------------------------------------- */ 80 81 /* The MD code. */ 82 #include <machine/csan.h> 83 84 /* -------------------------------------------------------------------------- */ 85 86 static void 87 kcsan_enable(void *dummy __unused) 88 { 89 90 printf("Enabling KCSCAN, expect reduced performance.\n"); 91 kcsan_enabled = true; 92 } 93 SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); 94 95 void 96 kcsan_cpu_init(u_int cpu) 97 { 98 kcsan_cpus[cpu].inited = true; 99 } 100 101 /* -------------------------------------------------------------------------- */ 102 103 static inline void 104 kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) 105 { 106 const char *newsym, *oldsym; 107 #ifdef DDB 108 c_db_sym_t sym; 109 db_expr_t offset; 110 111 sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); 112 db_symbol_values(sym, &newsym, NULL); 113 114 sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); 115 db_symbol_values(sym, &oldsym, NULL); 116 #else 117 newsym = ""; 118 oldsym = ""; 119 #endif 120 REPORT("CSan: Racy Access " 121 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " 122 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", 123 newcpu, 124 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), 125 (void *)new->addr, new->size, (void *)new->pc, newsym, 126 oldcpu, 127 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), 128 (void *)old->addr, old->size, (void *)old->pc, oldsym); 129 kcsan_md_unwind(); 130 } 131 132 static inline bool 133 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) 134 { 135 if (new->write && !new->atomic) 136 return false; 137 if (old->write && !old->atomic) 138 return false; 139 return true; 140 } 141 142 static inline void 143 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) 144 { 145 csan_cell_t old, new; 146 csan_cpu_t *cpu; 147 uint64_t intr; 148 size_t i; 149 150 if (__predict_false(!kcsan_enabled)) 151 return; 152 if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr))) 153 return; 154 if (KERNEL_PANICKED()) 155 return; 156 157 new.addr = addr; 158 new.size = size; 159 new.write = write; 160 new.atomic = atomic; 161 new.pc = pc; 162 163 CPU_FOREACH(i) { 164 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); 165 166 if (old.addr + old.size <= new.addr) 167 continue; 168 if (new.addr + new.size <= old.addr) 169 continue; 170 if (__predict_true(!old.write && !new.write)) 171 continue; 172 if (__predict_true(kcsan_access_is_atomic(&new, &old))) 173 continue; 174 175 kcsan_report(&new, PCPU_GET(cpuid), &old, i); 176 break; 177 } 178 179 if (__predict_false(!kcsan_md_is_avail())) 180 return; 181 182 kcsan_md_disable_intrs(&intr); 183 184 cpu = &kcsan_cpus[PCPU_GET(cpuid)]; 185 if (__predict_false(!cpu->inited)) 186 goto out; 187 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; 188 if (__predict_true(cpu->cnt != 0)) 189 goto out; 190 191 __builtin_memcpy(&cpu->cell, &new, sizeof(new)); 192 kcsan_md_delay(KCSAN_DELAY); 193 __builtin_memset(&cpu->cell, 0, sizeof(new)); 194 195 out: 196 kcsan_md_enable_intrs(&intr); 197 } 198 199 #define CSAN_READ(size) \ 200 void __tsan_read##size(uintptr_t); \ 201 void __tsan_read##size(uintptr_t addr) \ 202 { \ 203 kcsan_access(addr, size, false, false, __RET_ADDR); \ 204 } \ 205 void __tsan_unaligned_read##size(uintptr_t); \ 206 void __tsan_unaligned_read##size(uintptr_t addr) \ 207 { \ 208 kcsan_access(addr, size, false, false, __RET_ADDR); \ 209 } 210 211 CSAN_READ(1) 212 CSAN_READ(2) 213 CSAN_READ(4) 214 CSAN_READ(8) 215 CSAN_READ(16) 216 217 #define CSAN_WRITE(size) \ 218 void __tsan_write##size(uintptr_t); \ 219 void __tsan_write##size(uintptr_t addr) \ 220 { \ 221 kcsan_access(addr, size, true, false, __RET_ADDR); \ 222 } \ 223 void __tsan_unaligned_write##size(uintptr_t); \ 224 void __tsan_unaligned_write##size(uintptr_t addr) \ 225 { \ 226 kcsan_access(addr, size, true, false, __RET_ADDR); \ 227 } 228 229 CSAN_WRITE(1) 230 CSAN_WRITE(2) 231 CSAN_WRITE(4) 232 CSAN_WRITE(8) 233 CSAN_WRITE(16) 234 235 void __tsan_read_range(uintptr_t, size_t); 236 void __tsan_write_range(uintptr_t, size_t); 237 238 void 239 __tsan_read_range(uintptr_t addr, size_t size) 240 { 241 kcsan_access(addr, size, false, false, __RET_ADDR); 242 } 243 244 void 245 __tsan_write_range(uintptr_t addr, size_t size) 246 { 247 kcsan_access(addr, size, true, false, __RET_ADDR); 248 } 249 250 void __tsan_init(void); 251 void __tsan_func_entry(void *); 252 void __tsan_func_exit(void); 253 254 void 255 __tsan_init(void) 256 { 257 } 258 259 void 260 __tsan_func_entry(void *call_pc) 261 { 262 } 263 264 void 265 __tsan_func_exit(void) 266 { 267 } 268 269 /* -------------------------------------------------------------------------- */ 270 271 void * 272 kcsan_memcpy(void *dst, const void *src, size_t len) 273 { 274 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 275 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 276 return __builtin_memcpy(dst, src, len); 277 } 278 279 int 280 kcsan_memcmp(const void *b1, const void *b2, size_t len) 281 { 282 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); 283 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); 284 return __builtin_memcmp(b1, b2, len); 285 } 286 287 void * 288 kcsan_memset(void *b, int c, size_t len) 289 { 290 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); 291 return __builtin_memset(b, c, len); 292 } 293 294 void * 295 kcsan_memmove(void *dst, const void *src, size_t len) 296 { 297 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 298 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 299 return __builtin_memmove(dst, src, len); 300 } 301 302 __strong_reference(kcsan_memcpy, __tsan_memcpy); 303 __strong_reference(kcsan_memset, __tsan_memset); 304 __strong_reference(kcsan_memmove, __tsan_memmove); 305 306 char * 307 kcsan_strcpy(char *dst, const char *src) 308 { 309 char *save = dst; 310 311 while (1) { 312 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); 313 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); 314 *dst = *src; 315 if (*src == '\0') 316 break; 317 src++, dst++; 318 } 319 320 return save; 321 } 322 323 int 324 kcsan_strcmp(const char *s1, const char *s2) 325 { 326 while (1) { 327 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); 328 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); 329 if (*s1 != *s2) 330 break; 331 if (*s1 == '\0') 332 return 0; 333 s1++, s2++; 334 } 335 336 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 337 } 338 339 size_t 340 kcsan_strlen(const char *str) 341 { 342 const char *s; 343 344 s = str; 345 while (1) { 346 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); 347 if (*s == '\0') 348 break; 349 s++; 350 } 351 352 return (s - str); 353 } 354 355 int 356 kcsan_copyin(const void *uaddr, void *kaddr, size_t len) 357 { 358 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 359 return copyin(uaddr, kaddr, len); 360 } 361 362 int 363 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 364 { 365 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 366 return copyinstr(uaddr, kaddr, len, done); 367 } 368 369 int 370 kcsan_copyout(const void *kaddr, void *uaddr, size_t len) 371 { 372 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 373 return copyout(kaddr, uaddr, len); 374 } 375 376 /* -------------------------------------------------------------------------- */ 377 378 #include <machine/atomic.h> 379 #include <sys/atomic_san.h> 380 381 #define _CSAN_ATOMIC_FUNC_ADD(name, type) \ 382 void kcsan_atomic_add_##name(volatile type *ptr, type val) \ 383 { \ 384 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 385 __RET_ADDR); \ 386 atomic_add_##name(ptr, val); \ 387 } 388 389 #define CSAN_ATOMIC_FUNC_ADD(name, type) \ 390 _CSAN_ATOMIC_FUNC_ADD(name, type) \ 391 _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 392 _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) 393 394 #define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ 395 void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ 396 { \ 397 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 398 __RET_ADDR); \ 399 atomic_clear_##name(ptr, val); \ 400 } 401 402 #define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ 403 _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ 404 _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 405 _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 406 407 #define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ 408 int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ 409 type val2) \ 410 { \ 411 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 412 __RET_ADDR); \ 413 return (atomic_cmpset_##name(ptr, val1, val2)); \ 414 } 415 416 #define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ 417 _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ 418 _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 419 _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 420 421 #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 422 int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ 423 type val2) \ 424 { \ 425 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 426 __RET_ADDR); \ 427 return (atomic_fcmpset_##name(ptr, val1, val2)); \ 428 } 429 430 #define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 431 _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 432 _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 433 _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 434 435 #define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 436 type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 437 { \ 438 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 439 __RET_ADDR); \ 440 return (atomic_fetchadd_##name(ptr, val)); \ 441 } 442 443 #define _CSAN_ATOMIC_FUNC_LOAD(name, type) \ 444 type kcsan_atomic_load_##name(volatile type *ptr) \ 445 { \ 446 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \ 447 __RET_ADDR); \ 448 return (atomic_load_##name(ptr)); \ 449 } 450 451 #define CSAN_ATOMIC_FUNC_LOAD(name, type) \ 452 _CSAN_ATOMIC_FUNC_LOAD(name, type) \ 453 _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \ 454 455 #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 456 type kcsan_atomic_readandclear_##name(volatile type *ptr) \ 457 { \ 458 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 459 __RET_ADDR); \ 460 return (atomic_readandclear_##name(ptr)); \ 461 } 462 463 #define _CSAN_ATOMIC_FUNC_SET(name, type) \ 464 void kcsan_atomic_set_##name(volatile type *ptr, type val) \ 465 { \ 466 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 467 __RET_ADDR); \ 468 atomic_set_##name(ptr, val); \ 469 } 470 471 #define CSAN_ATOMIC_FUNC_SET(name, type) \ 472 _CSAN_ATOMIC_FUNC_SET(name, type) \ 473 _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 474 _CSAN_ATOMIC_FUNC_SET(rel_##name, type) 475 476 #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 477 void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \ 478 { \ 479 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 480 __RET_ADDR); \ 481 atomic_subtract_##name(ptr, val); \ 482 } 483 484 #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 485 _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 486 _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 487 _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 488 489 #define _CSAN_ATOMIC_FUNC_STORE(name, type) \ 490 void kcsan_atomic_store_##name(volatile type *ptr, type val) \ 491 { \ 492 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 493 __RET_ADDR); \ 494 atomic_store_##name(ptr, val); \ 495 } 496 497 #define CSAN_ATOMIC_FUNC_STORE(name, type) \ 498 _CSAN_ATOMIC_FUNC_STORE(name, type) \ 499 _CSAN_ATOMIC_FUNC_STORE(rel_##name, type) 500 501 #define CSAN_ATOMIC_FUNC_SWAP(name, type) \ 502 type kcsan_atomic_swap_##name(volatile type *ptr, type val) \ 503 { \ 504 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 505 __RET_ADDR); \ 506 return(atomic_swap_##name(ptr, val)); \ 507 } 508 509 #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 510 int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \ 511 { \ 512 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 513 __RET_ADDR); \ 514 return(atomic_testandclear_##name(ptr, val)); \ 515 } 516 517 #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 518 int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \ 519 { \ 520 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 521 __RET_ADDR); \ 522 return (atomic_testandset_##name(ptr, val)); \ 523 } 524 525 _CSAN_ATOMIC_FUNC_LOAD(bool, bool) 526 _CSAN_ATOMIC_FUNC_STORE(bool, bool) 527 528 CSAN_ATOMIC_FUNC_ADD(8, uint8_t) 529 CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) 530 CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) 531 CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) 532 CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) 533 CSAN_ATOMIC_FUNC_SET(8, uint8_t) 534 CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) 535 _CSAN_ATOMIC_FUNC_STORE(8, uint8_t) 536 #if 0 537 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t) 538 CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t) 539 CSAN_ATOMIC_FUNC_SWAP(8, uint8_t) 540 CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t) 541 CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t) 542 #endif 543 544 CSAN_ATOMIC_FUNC_ADD(16, uint16_t) 545 CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) 546 CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) 547 CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) 548 CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) 549 CSAN_ATOMIC_FUNC_SET(16, uint16_t) 550 CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) 551 _CSAN_ATOMIC_FUNC_STORE(16, uint16_t) 552 #if 0 553 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t) 554 CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t) 555 CSAN_ATOMIC_FUNC_SWAP(16, uint16_t) 556 CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t) 557 CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t) 558 #endif 559 560 CSAN_ATOMIC_FUNC_ADD(32, uint32_t) 561 CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t) 562 CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t) 563 CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t) 564 CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t) 565 CSAN_ATOMIC_FUNC_LOAD(32, uint32_t) 566 CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t) 567 CSAN_ATOMIC_FUNC_SET(32, uint32_t) 568 CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t) 569 CSAN_ATOMIC_FUNC_STORE(32, uint32_t) 570 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t) 571 CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t) 572 CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t) 573 574 CSAN_ATOMIC_FUNC_ADD(64, uint64_t) 575 CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t) 576 CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t) 577 CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t) 578 CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t) 579 CSAN_ATOMIC_FUNC_LOAD(64, uint64_t) 580 CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t) 581 CSAN_ATOMIC_FUNC_SET(64, uint64_t) 582 CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t) 583 CSAN_ATOMIC_FUNC_STORE(64, uint64_t) 584 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t) 585 CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t) 586 CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t) 587 588 CSAN_ATOMIC_FUNC_ADD(char, uint8_t) 589 CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t) 590 CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t) 591 CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t) 592 CSAN_ATOMIC_FUNC_LOAD(char, uint8_t) 593 CSAN_ATOMIC_FUNC_SET(char, uint8_t) 594 CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t) 595 _CSAN_ATOMIC_FUNC_STORE(char, uint8_t) 596 #if 0 597 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t) 598 CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t) 599 CSAN_ATOMIC_FUNC_SWAP(char, uint8_t) 600 CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t) 601 CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t) 602 #endif 603 604 CSAN_ATOMIC_FUNC_ADD(short, uint16_t) 605 CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t) 606 CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t) 607 CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t) 608 CSAN_ATOMIC_FUNC_LOAD(short, uint16_t) 609 CSAN_ATOMIC_FUNC_SET(short, uint16_t) 610 CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t) 611 _CSAN_ATOMIC_FUNC_STORE(short, uint16_t) 612 #if 0 613 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t) 614 CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t) 615 CSAN_ATOMIC_FUNC_SWAP(short, uint16_t) 616 CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t) 617 CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t) 618 #endif 619 620 CSAN_ATOMIC_FUNC_ADD(int, u_int) 621 CSAN_ATOMIC_FUNC_CLEAR(int, u_int) 622 CSAN_ATOMIC_FUNC_CMPSET(int, u_int) 623 CSAN_ATOMIC_FUNC_FCMPSET(int, u_int) 624 CSAN_ATOMIC_FUNC_FETCHADD(int, u_int) 625 CSAN_ATOMIC_FUNC_LOAD(int, u_int) 626 CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int) 627 CSAN_ATOMIC_FUNC_SET(int, u_int) 628 CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int) 629 CSAN_ATOMIC_FUNC_STORE(int, u_int) 630 CSAN_ATOMIC_FUNC_SWAP(int, u_int) 631 CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int) 632 CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int) 633 634 CSAN_ATOMIC_FUNC_ADD(long, u_long) 635 CSAN_ATOMIC_FUNC_CLEAR(long, u_long) 636 CSAN_ATOMIC_FUNC_CMPSET(long, u_long) 637 CSAN_ATOMIC_FUNC_FCMPSET(long, u_long) 638 CSAN_ATOMIC_FUNC_FETCHADD(long, u_long) 639 CSAN_ATOMIC_FUNC_LOAD(long, u_long) 640 CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long) 641 CSAN_ATOMIC_FUNC_SET(long, u_long) 642 CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long) 643 CSAN_ATOMIC_FUNC_STORE(long, u_long) 644 CSAN_ATOMIC_FUNC_SWAP(long, u_long) 645 CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) 646 CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) 647 CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long) 648 649 CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) 650 CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t) 651 CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t) 652 CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t) 653 #if !defined(__amd64__) 654 CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t) 655 #endif 656 CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t) 657 CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t) 658 CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t) 659 CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t) 660 CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t) 661 CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t) 662 #if 0 663 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t) 664 CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t) 665 #endif 666 667 #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 668 void kcsan_atomic_thread_fence_##name(void) \ 669 { \ 670 atomic_thread_fence_##name(); \ 671 } 672 673 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq) 674 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel) 675 CSAN_ATOMIC_FUNC_THREAD_FENCE(rel) 676 CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst) 677 678 void 679 kcsan_atomic_interrupt_fence(void) 680 { 681 atomic_interrupt_fence(); 682 } 683 684 /* -------------------------------------------------------------------------- */ 685 686 #include <sys/bus.h> 687 #include <machine/bus.h> 688 #include <sys/bus_san.h> 689 690 int 691 kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 692 int flags, bus_space_handle_t *handlep) 693 { 694 695 return (bus_space_map(tag, hnd, size, flags, handlep)); 696 } 697 698 void 699 kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 700 bus_size_t size) 701 { 702 703 bus_space_unmap(tag, hnd, size); 704 } 705 706 int 707 kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 708 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 709 { 710 711 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 712 } 713 714 #if !defined(__amd64__) 715 int 716 kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, 717 bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, 718 bus_size_t boundary, int flags, bus_addr_t *addrp, 719 bus_space_handle_t *handlep) 720 { 721 722 return (bus_space_alloc(tag, reg_start, reg_end, size, alignment, 723 boundary, flags, addrp, handlep)); 724 } 725 #endif 726 727 void 728 kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 729 bus_size_t size) 730 { 731 732 bus_space_free(tag, hnd, size); 733 } 734 735 void 736 kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 737 bus_size_t offset, bus_size_t size, int flags) 738 { 739 740 bus_space_barrier(tag, hnd, offset, size, flags); 741 } 742 743 #define CSAN_BUS_READ_FUNC(func, width, type) \ 744 type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 745 bus_space_handle_t hnd, bus_size_t offset) \ 746 { \ 747 return (bus_space_read##func##_##width(tag, hnd, \ 748 offset)); \ 749 } \ 750 751 #define CSAN_BUS_READ_PTR_FUNC(func, width, type) \ 752 void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 753 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 754 bus_size_t count) \ 755 { \ 756 kcsan_access((uintptr_t)buf, sizeof(type) * count, \ 757 false, false, __RET_ADDR); \ 758 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 759 count); \ 760 } 761 762 CSAN_BUS_READ_FUNC(, 1, uint8_t) 763 CSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 764 CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 765 CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 766 CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 767 CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 768 769 CSAN_BUS_READ_FUNC(, 2, uint16_t) 770 CSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 771 CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 772 CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 773 CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 774 CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 775 776 CSAN_BUS_READ_FUNC(, 4, uint32_t) 777 CSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 778 CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 779 CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 780 CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 781 CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 782 783 CSAN_BUS_READ_FUNC(, 8, uint64_t) 784 #if defined(__aarch64__) 785 CSAN_BUS_READ_FUNC(_stream, 8, uint64_t) 786 CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) 787 CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) 788 CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) 789 CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) 790 #endif 791 792 #define CSAN_BUS_WRITE_FUNC(func, width, type) \ 793 void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 794 bus_space_handle_t hnd, bus_size_t offset, type value) \ 795 { \ 796 bus_space_write##func##_##width(tag, hnd, offset, value); \ 797 } \ 798 799 #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 800 void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \ 801 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 802 bus_size_t count) \ 803 { \ 804 kcsan_access((uintptr_t)buf, sizeof(type) * count, \ 805 true, false, __RET_ADDR); \ 806 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 807 count); \ 808 } 809 810 CSAN_BUS_WRITE_FUNC(, 1, uint8_t) 811 CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 812 CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 813 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 814 CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 815 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 816 817 CSAN_BUS_WRITE_FUNC(, 2, uint16_t) 818 CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 819 CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 820 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 821 CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 822 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 823 824 CSAN_BUS_WRITE_FUNC(, 4, uint32_t) 825 CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 826 CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 827 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 828 CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 829 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 830 831 CSAN_BUS_WRITE_FUNC(, 8, uint64_t) 832 #if defined(__aarch64__) 833 CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t) 834 CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t) 835 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t) 836 CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t) 837 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t) 838 #endif 839 840 #define CSAN_BUS_SET_FUNC(func, width, type) \ 841 void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 842 bus_space_handle_t hnd, bus_size_t offset, type value, \ 843 bus_size_t count) \ 844 { \ 845 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 846 count); \ 847 } 848 849 CSAN_BUS_SET_FUNC(multi, 1, uint8_t) 850 CSAN_BUS_SET_FUNC(region, 1, uint8_t) 851 #if !defined(__aarch64__) 852 CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 853 CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 854 #endif 855 856 CSAN_BUS_SET_FUNC(multi, 2, uint16_t) 857 CSAN_BUS_SET_FUNC(region, 2, uint16_t) 858 #if !defined(__aarch64__) 859 CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 860 CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 861 #endif 862 863 CSAN_BUS_SET_FUNC(multi, 4, uint32_t) 864 CSAN_BUS_SET_FUNC(region, 4, uint32_t) 865 #if !defined(__aarch64__) 866 CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 867 CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 868 #endif 869 870 #if !defined(__amd64__) 871 CSAN_BUS_SET_FUNC(multi, 8, uint64_t) 872 CSAN_BUS_SET_FUNC(region, 8, uint64_t) 873 #if !defined(__aarch64__) 874 CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t) 875 CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t) 876 #endif 877 #endif 878 879 #define CSAN_BUS_PEEK_FUNC(width, type) \ 880 int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \ 881 bus_space_handle_t hnd, bus_size_t offset, type *value) \ 882 { \ 883 kcsan_access((uintptr_t)value, sizeof(type), true, false, \ 884 __RET_ADDR); \ 885 return (bus_space_peek_##width(tag, hnd, offset, value)); \ 886 } 887 888 CSAN_BUS_PEEK_FUNC(1, uint8_t) 889 CSAN_BUS_PEEK_FUNC(2, uint16_t) 890 CSAN_BUS_PEEK_FUNC(4, uint32_t) 891 #if !defined(__i386__) 892 CSAN_BUS_PEEK_FUNC(8, uint64_t) 893 #endif 894 895 #define CSAN_BUS_POKE_FUNC(width, type) \ 896 int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \ 897 bus_space_handle_t hnd, bus_size_t offset, type value) \ 898 { \ 899 return (bus_space_poke_##width(tag, hnd, offset, value)); \ 900 } 901 902 CSAN_BUS_POKE_FUNC(1, uint8_t) 903 CSAN_BUS_POKE_FUNC(2, uint16_t) 904 CSAN_BUS_POKE_FUNC(4, uint32_t) 905 #if !defined(__i386__) 906 CSAN_BUS_POKE_FUNC(8, uint64_t) 907 #endif 908