1 /* $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $ */
2
3 /*
4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 *
7 * This code is part of the KASAN subsystem of the NetBSD kernel.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #define SAN_RUNTIME
32
33 #include <sys/cdefs.h>
34 #if 0
35 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
36 #endif
37
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/asan.h>
41 #include <sys/kernel.h>
42 #include <sys/proc.h>
43 #include <sys/stack.h>
44 #include <sys/sysctl.h>
45
46 #include <machine/asan.h>
47 #include <machine/bus.h>
48
49 /* ASAN constants. Part of the compiler ABI. */
50 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE - 1)
51 #define KASAN_ALLOCA_SCALE_SIZE 32
52
53 /* ASAN ABI version. */
54 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
55 #define ASAN_ABI_VERSION 8
56 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
57 #define ASAN_ABI_VERSION 8
58 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
59 #define ASAN_ABI_VERSION 6
60 #else
61 #error "Unsupported compiler version"
62 #endif
63
64 #define __RET_ADDR (unsigned long)__builtin_return_address(0)
65
66 /* Global variable descriptor. Part of the compiler ABI. */
67 struct __asan_global_source_location {
68 const char *filename;
69 int line_no;
70 int column_no;
71 };
72
73 struct __asan_global {
74 const void *beg; /* address of the global variable */
75 size_t size; /* size of the global variable */
76 size_t size_with_redzone; /* size with the redzone */
77 const void *name; /* name of the variable */
78 const void *module_name; /* name of the module where the var is declared */
79 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
80 struct __asan_global_source_location *location;
81 #if ASAN_ABI_VERSION >= 7
82 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */
83 #endif
84 };
85
86 FEATURE(kasan, "Kernel address sanitizer");
87
88 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
89 "KASAN options");
90
91 static int panic_on_violation = 1;
92 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
93 &panic_on_violation, 0,
94 "Panic if an invalid access is detected");
95
96 #define kasan_enabled (!kasan_disabled)
97 static bool kasan_disabled __read_mostly = true;
98 SYSCTL_BOOL(_debug_kasan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
99 &kasan_disabled, 0, "KASAN is disabled");
100
101 /* -------------------------------------------------------------------------- */
102
103 void
kasan_shadow_map(vm_offset_t addr,size_t size)104 kasan_shadow_map(vm_offset_t addr, size_t size)
105 {
106 size_t sz, npages, i;
107 vm_offset_t sva, eva;
108
109 KASSERT(addr % KASAN_SHADOW_SCALE == 0,
110 ("%s: invalid address %#lx", __func__, addr));
111
112 sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
113
114 sva = kasan_md_addr_to_shad(addr);
115 eva = kasan_md_addr_to_shad(addr) + sz;
116
117 sva = rounddown(sva, PAGE_SIZE);
118 eva = roundup(eva, PAGE_SIZE);
119
120 npages = (eva - sva) / PAGE_SIZE;
121
122 KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
123 ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
124
125 for (i = 0; i < npages; i++)
126 pmap_san_enter(sva + ptoa(i));
127 }
128
129 void
kasan_init(void)130 kasan_init(void)
131 {
132 int disabled;
133
134 disabled = 0;
135 TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
136 if (disabled)
137 return;
138
139 /* MD initialization. */
140 kasan_md_init();
141
142 /* Now officially enabled. */
143 kasan_disabled = false;
144 }
145
146 void
kasan_init_early(vm_offset_t stack,size_t size)147 kasan_init_early(vm_offset_t stack, size_t size)
148 {
149 kasan_md_init_early(stack, size);
150 }
151
152 static inline const char *
kasan_code_name(uint8_t code)153 kasan_code_name(uint8_t code)
154 {
155 switch (code) {
156 case KASAN_GENERIC_REDZONE:
157 return "GenericRedZone";
158 case KASAN_MALLOC_REDZONE:
159 return "MallocRedZone";
160 case KASAN_KMEM_REDZONE:
161 return "KmemRedZone";
162 case KASAN_UMA_FREED:
163 return "UMAUseAfterFree";
164 case KASAN_KSTACK_FREED:
165 return "KernelStack";
166 case KASAN_EXEC_ARGS_FREED:
167 return "ExecKVA";
168 case 1 ... 7:
169 return "RedZonePartial";
170 case KASAN_STACK_LEFT:
171 return "StackLeft";
172 case KASAN_STACK_MID:
173 return "StackMiddle";
174 case KASAN_STACK_RIGHT:
175 return "StackRight";
176 case KASAN_USE_AFTER_RET:
177 return "UseAfterRet";
178 case KASAN_USE_AFTER_SCOPE:
179 return "UseAfterScope";
180 default:
181 return "Unknown";
182 }
183 }
184
185 #define REPORT(f, ...) do { \
186 if (panic_on_violation) { \
187 kasan_disabled = true; \
188 panic(f, __VA_ARGS__); \
189 } else { \
190 struct stack st; \
191 \
192 stack_save(&st); \
193 printf(f "\n", __VA_ARGS__); \
194 stack_print_ddb(&st); \
195 } \
196 } while (0)
197
198 static void
kasan_report(unsigned long addr,size_t size,bool write,unsigned long pc,uint8_t code)199 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
200 uint8_t code)
201 {
202 REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
203 size, (write ? "write" : "read"), addr, kasan_code_name(code),
204 code);
205 }
206
207 static __always_inline void
kasan_shadow_1byte_markvalid(unsigned long addr)208 kasan_shadow_1byte_markvalid(unsigned long addr)
209 {
210 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
211 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
212
213 *byte = last;
214 }
215
216 static __always_inline void
kasan_shadow_Nbyte_markvalid(const void * addr,size_t size)217 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
218 {
219 size_t i;
220
221 for (i = 0; i < size; i++) {
222 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
223 }
224 }
225
226 static __always_inline void
kasan_shadow_Nbyte_fill(const void * addr,size_t size,uint8_t code)227 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
228 {
229 void *shad;
230
231 if (__predict_false(size == 0))
232 return;
233 if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
234 return;
235
236 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
237 ("%s: invalid address %p", __func__, addr));
238 KASSERT(size % KASAN_SHADOW_SCALE == 0,
239 ("%s: invalid size %zu", __func__, size));
240
241 shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
242 size = size >> KASAN_SHADOW_SCALE_SHIFT;
243
244 __builtin_memset(shad, code, size);
245 }
246
247 /*
248 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
249 * and the rest as invalid. There are generally two use cases:
250 *
251 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
252 * the redzone at the end of the buffer as invalid. If the entire is to be
253 * marked invalid, origsize will be 0.
254 *
255 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
256 */
257 void
kasan_mark(const void * addr,size_t size,size_t redzsize,uint8_t code)258 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
259 {
260 size_t i, n, redz;
261 int8_t *shad;
262
263 if (__predict_false(!kasan_enabled))
264 return;
265
266 if (kasan_md_unsupported((vm_offset_t)addr))
267 return;
268
269 KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
270 (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
271 ("%s: invalid address %p", __func__, addr));
272 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
273 ("%s: invalid address %p", __func__, addr));
274 redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
275 KASSERT(redz % KASAN_SHADOW_SCALE == 0,
276 ("%s: invalid size %zu", __func__, redz));
277 shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
278
279 /* Chunks of 8 bytes, valid. */
280 n = size / KASAN_SHADOW_SCALE;
281 for (i = 0; i < n; i++) {
282 *shad++ = 0;
283 }
284
285 /* Possibly one chunk, mid. */
286 if ((size & KASAN_SHADOW_MASK) != 0) {
287 *shad++ = (size & KASAN_SHADOW_MASK);
288 }
289
290 /* Chunks of 8 bytes, invalid. */
291 n = redz / KASAN_SHADOW_SCALE;
292 for (i = 0; i < n; i++) {
293 *shad++ = code;
294 }
295 }
296
297 void
kasan_thread_alloc(struct thread * td)298 kasan_thread_alloc(struct thread *td)
299 {
300 if (td->td_kstack != 0) {
301 kasan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
302 ptoa(td->td_kstack_pages), 0);
303 }
304 }
305
306 /* -------------------------------------------------------------------------- */
307
308 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \
309 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \
310 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
311
312 static __always_inline bool
kasan_shadow_1byte_isvalid(unsigned long addr,uint8_t * code)313 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
314 {
315 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
316 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
317
318 if (__predict_true(*byte == 0 || last <= *byte)) {
319 return (true);
320 }
321 *code = *byte;
322 return (false);
323 }
324
325 static __always_inline bool
kasan_shadow_2byte_isvalid(unsigned long addr,uint8_t * code)326 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
327 {
328 int8_t *byte, last;
329
330 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
331 return (kasan_shadow_1byte_isvalid(addr, code) &&
332 kasan_shadow_1byte_isvalid(addr+1, code));
333 }
334
335 byte = (int8_t *)kasan_md_addr_to_shad(addr);
336 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
337
338 if (__predict_true(*byte == 0 || last <= *byte)) {
339 return (true);
340 }
341 *code = *byte;
342 return (false);
343 }
344
345 static __always_inline bool
kasan_shadow_4byte_isvalid(unsigned long addr,uint8_t * code)346 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
347 {
348 int8_t *byte, last;
349
350 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
351 return (kasan_shadow_2byte_isvalid(addr, code) &&
352 kasan_shadow_2byte_isvalid(addr+2, code));
353 }
354
355 byte = (int8_t *)kasan_md_addr_to_shad(addr);
356 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
357
358 if (__predict_true(*byte == 0 || last <= *byte)) {
359 return (true);
360 }
361 *code = *byte;
362 return (false);
363 }
364
365 static __always_inline bool
kasan_shadow_8byte_isvalid(unsigned long addr,uint8_t * code)366 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
367 {
368 int8_t *byte, last;
369
370 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
371 return (kasan_shadow_4byte_isvalid(addr, code) &&
372 kasan_shadow_4byte_isvalid(addr+4, code));
373 }
374
375 byte = (int8_t *)kasan_md_addr_to_shad(addr);
376 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
377
378 if (__predict_true(*byte == 0 || last <= *byte)) {
379 return (true);
380 }
381 *code = *byte;
382 return (false);
383 }
384
385 static __always_inline bool
kasan_shadow_Nbyte_isvalid(unsigned long addr,size_t size,uint8_t * code)386 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
387 {
388 size_t i;
389
390 for (i = 0; i < size; i++) {
391 if (!kasan_shadow_1byte_isvalid(addr+i, code))
392 return (false);
393 }
394
395 return (true);
396 }
397
398 static __always_inline void
kasan_shadow_check(unsigned long addr,size_t size,bool write,unsigned long retaddr)399 kasan_shadow_check(unsigned long addr, size_t size, bool write,
400 unsigned long retaddr)
401 {
402 uint8_t code;
403 bool valid;
404
405 if (__predict_false(!kasan_enabled))
406 return;
407 if (__predict_false(curthread != NULL &&
408 (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0))
409 return;
410 if (__predict_false(size == 0))
411 return;
412 if (__predict_false(kasan_md_unsupported(addr)))
413 return;
414 if (KERNEL_PANICKED())
415 return;
416
417 if (__builtin_constant_p(size)) {
418 switch (size) {
419 case 1:
420 valid = kasan_shadow_1byte_isvalid(addr, &code);
421 break;
422 case 2:
423 valid = kasan_shadow_2byte_isvalid(addr, &code);
424 break;
425 case 4:
426 valid = kasan_shadow_4byte_isvalid(addr, &code);
427 break;
428 case 8:
429 valid = kasan_shadow_8byte_isvalid(addr, &code);
430 break;
431 default:
432 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
433 break;
434 }
435 } else {
436 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
437 }
438
439 if (__predict_false(!valid)) {
440 kasan_report(addr, size, write, retaddr, code);
441 }
442 }
443
444 /* -------------------------------------------------------------------------- */
445
446 void *
kasan_memcpy(void * dst,const void * src,size_t len)447 kasan_memcpy(void *dst, const void *src, size_t len)
448 {
449 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
450 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
451 return (__builtin_memcpy(dst, src, len));
452 }
453
454 int
kasan_memcmp(const void * b1,const void * b2,size_t len)455 kasan_memcmp(const void *b1, const void *b2, size_t len)
456 {
457 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
458 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
459 return (__builtin_memcmp(b1, b2, len));
460 }
461
462 void *
kasan_memset(void * b,int c,size_t len)463 kasan_memset(void *b, int c, size_t len)
464 {
465 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
466 return (__builtin_memset(b, c, len));
467 }
468
469 void *
kasan_memmove(void * dst,const void * src,size_t len)470 kasan_memmove(void *dst, const void *src, size_t len)
471 {
472 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
473 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
474 return (__builtin_memmove(dst, src, len));
475 }
476
477 size_t
kasan_strlen(const char * str)478 kasan_strlen(const char *str)
479 {
480 const char *s;
481
482 s = str;
483 while (1) {
484 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
485 if (*s == '\0')
486 break;
487 s++;
488 }
489
490 return (s - str);
491 }
492
493 char *
kasan_strcpy(char * dst,const char * src)494 kasan_strcpy(char *dst, const char *src)
495 {
496 char *save = dst;
497
498 while (1) {
499 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
500 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
501 *dst = *src;
502 if (*src == '\0')
503 break;
504 src++, dst++;
505 }
506
507 return save;
508 }
509
510 int
kasan_strcmp(const char * s1,const char * s2)511 kasan_strcmp(const char *s1, const char *s2)
512 {
513 while (1) {
514 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
515 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
516 if (*s1 != *s2)
517 break;
518 if (*s1 == '\0')
519 return 0;
520 s1++, s2++;
521 }
522
523 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
524 }
525
526 int
kasan_copyin(const void * uaddr,void * kaddr,size_t len)527 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
528 {
529 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
530 return (copyin(uaddr, kaddr, len));
531 }
532
533 int
kasan_copyinstr(const void * uaddr,void * kaddr,size_t len,size_t * done)534 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
535 {
536 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
537 return (copyinstr(uaddr, kaddr, len, done));
538 }
539
540 int
kasan_copyout(const void * kaddr,void * uaddr,size_t len)541 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
542 {
543 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
544 return (copyout(kaddr, uaddr, len));
545 }
546
547 /* -------------------------------------------------------------------------- */
548
549 int
kasan_fubyte(volatile const void * base)550 kasan_fubyte(volatile const void *base)
551 {
552 return (fubyte(base));
553 }
554
555 int
kasan_fuword16(volatile const void * base)556 kasan_fuword16(volatile const void *base)
557 {
558 return (fuword16(base));
559 }
560
561 int
kasan_fueword(volatile const void * base,long * val)562 kasan_fueword(volatile const void *base, long *val)
563 {
564 kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
565 return (fueword(base, val));
566 }
567
568 int
kasan_fueword32(volatile const void * base,int32_t * val)569 kasan_fueword32(volatile const void *base, int32_t *val)
570 {
571 kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
572 return (fueword32(base, val));
573 }
574
575 int
kasan_fueword64(volatile const void * base,int64_t * val)576 kasan_fueword64(volatile const void *base, int64_t *val)
577 {
578 kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
579 return (fueword64(base, val));
580 }
581
582 int
kasan_subyte(volatile void * base,int byte)583 kasan_subyte(volatile void *base, int byte)
584 {
585 return (subyte(base, byte));
586 }
587
588 int
kasan_suword(volatile void * base,long word)589 kasan_suword(volatile void *base, long word)
590 {
591 return (suword(base, word));
592 }
593
594 int
kasan_suword16(volatile void * base,int word)595 kasan_suword16(volatile void *base, int word)
596 {
597 return (suword16(base, word));
598 }
599
600 int
kasan_suword32(volatile void * base,int32_t word)601 kasan_suword32(volatile void *base, int32_t word)
602 {
603 return (suword32(base, word));
604 }
605
606 int
kasan_suword64(volatile void * base,int64_t word)607 kasan_suword64(volatile void *base, int64_t word)
608 {
609 return (suword64(base, word));
610 }
611
612 int
kasan_casueword32(volatile uint32_t * base,uint32_t oldval,uint32_t * oldvalp,uint32_t newval)613 kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
614 uint32_t newval)
615 {
616 kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
617 __RET_ADDR);
618 return (casueword32(base, oldval, oldvalp, newval));
619 }
620
621 int
kasan_casueword(volatile u_long * base,u_long oldval,u_long * oldvalp,u_long newval)622 kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
623 u_long newval)
624 {
625 kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
626 __RET_ADDR);
627 return (casueword(base, oldval, oldvalp, newval));
628 }
629
630 /* -------------------------------------------------------------------------- */
631
632 #include <machine/atomic.h>
633 #include <sys/atomic_san.h>
634
635 #define _ASAN_ATOMIC_FUNC_ADD(name, type) \
636 void kasan_atomic_add_##name(volatile type *ptr, type val) \
637 { \
638 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
639 __RET_ADDR); \
640 atomic_add_##name(ptr, val); \
641 }
642
643 #define ASAN_ATOMIC_FUNC_ADD(name, type) \
644 _ASAN_ATOMIC_FUNC_ADD(name, type) \
645 _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \
646 _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
647
648 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
649 void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
650 { \
651 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
652 __RET_ADDR); \
653 atomic_subtract_##name(ptr, val); \
654 }
655
656 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
657 _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
658 _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
659 _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
660
661 #define _ASAN_ATOMIC_FUNC_SET(name, type) \
662 void kasan_atomic_set_##name(volatile type *ptr, type val) \
663 { \
664 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
665 __RET_ADDR); \
666 atomic_set_##name(ptr, val); \
667 }
668
669 #define ASAN_ATOMIC_FUNC_SET(name, type) \
670 _ASAN_ATOMIC_FUNC_SET(name, type) \
671 _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \
672 _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
673
674 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
675 void kasan_atomic_clear_##name(volatile type *ptr, type val) \
676 { \
677 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
678 __RET_ADDR); \
679 atomic_clear_##name(ptr, val); \
680 }
681
682 #define ASAN_ATOMIC_FUNC_CLEAR(name, type) \
683 _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
684 _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
685 _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
686
687 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \
688 type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
689 { \
690 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
691 __RET_ADDR); \
692 return (atomic_fetchadd_##name(ptr, val)); \
693 }
694
695 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
696 type kasan_atomic_readandclear_##name(volatile type *ptr) \
697 { \
698 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
699 __RET_ADDR); \
700 return (atomic_readandclear_##name(ptr)); \
701 }
702
703 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
704 int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
705 { \
706 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
707 __RET_ADDR); \
708 return (atomic_testandclear_##name(ptr, v)); \
709 }
710
711 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \
712 int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
713 { \
714 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
715 __RET_ADDR); \
716 return (atomic_testandset_##name(ptr, v)); \
717 }
718
719 #define ASAN_ATOMIC_FUNC_SWAP(name, type) \
720 type kasan_atomic_swap_##name(volatile type *ptr, type val) \
721 { \
722 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
723 __RET_ADDR); \
724 return (atomic_swap_##name(ptr, val)); \
725 }
726
727 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
728 int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \
729 type nval) \
730 { \
731 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
732 __RET_ADDR); \
733 return (atomic_cmpset_##name(ptr, oval, nval)); \
734 }
735
736 #define ASAN_ATOMIC_FUNC_CMPSET(name, type) \
737 _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
738 _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
739 _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
740
741 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
742 int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
743 type nval) \
744 { \
745 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
746 __RET_ADDR); \
747 return (atomic_fcmpset_##name(ptr, oval, nval)); \
748 }
749
750 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
751 _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
752 _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
753 _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
754
755 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \
756 void kasan_atomic_thread_fence_##name(void) \
757 { \
758 atomic_thread_fence_##name(); \
759 }
760
761 #define _ASAN_ATOMIC_FUNC_LOAD(name, type) \
762 type kasan_atomic_load_##name(const volatile type *ptr) \
763 { \
764 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
765 __RET_ADDR); \
766 return (atomic_load_##name(ptr)); \
767 }
768
769 #define ASAN_ATOMIC_FUNC_LOAD(name, type) \
770 _ASAN_ATOMIC_FUNC_LOAD(name, type) \
771 _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
772
773 #define _ASAN_ATOMIC_FUNC_STORE(name, type) \
774 void kasan_atomic_store_##name(volatile type *ptr, type val) \
775 { \
776 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
777 __RET_ADDR); \
778 atomic_store_##name(ptr, val); \
779 }
780
781 #define ASAN_ATOMIC_FUNC_STORE(name, type) \
782 _ASAN_ATOMIC_FUNC_STORE(name, type) \
783 _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
784
785 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
786 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
787 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
788 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
789 ASAN_ATOMIC_FUNC_ADD(int, u_int);
790 ASAN_ATOMIC_FUNC_ADD(long, u_long);
791 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
792
793 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
794 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
795 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
796 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
797 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
798 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
799 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
800
801 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
802 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
803 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
804 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
805 ASAN_ATOMIC_FUNC_SET(int, u_int);
806 ASAN_ATOMIC_FUNC_SET(long, u_long);
807 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
808
809 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
810 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
811 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
812 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
813 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
814 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
815 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
816
817 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
818 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
819 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
820 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
821
822 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
823 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
824 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
825 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
826 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
827
828 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
829 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
830 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
831 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
832 ASAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t);
833
834 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
835 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
836 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
837 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
838 ASAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long);
839 ASAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t);
840
841 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
842 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
843 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
844 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
845 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
846
847 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
848 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
849 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
850 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
851 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
852 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
853 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
854
855 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
856 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
857 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
858 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
859 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
860 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
861 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
862
863 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
864 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
865 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
866 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
867 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
868 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
869 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
870 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
871 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
872 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
873
874 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
875 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
876 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
877 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
878 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
879 ASAN_ATOMIC_FUNC_STORE(char, u_char);
880 ASAN_ATOMIC_FUNC_STORE(short, u_short);
881 ASAN_ATOMIC_FUNC_STORE(int, u_int);
882 ASAN_ATOMIC_FUNC_STORE(long, u_long);
883 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
884
885 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
886 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
887 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
888 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
889
890 void
kasan_atomic_interrupt_fence(void)891 kasan_atomic_interrupt_fence(void)
892 {
893 }
894
895 /* -------------------------------------------------------------------------- */
896
897 #include <sys/bus.h>
898 #include <machine/bus.h>
899 #include <sys/bus_san.h>
900
901 int
kasan_bus_space_map(bus_space_tag_t tag,bus_addr_t hnd,bus_size_t size,int flags,bus_space_handle_t * handlep)902 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
903 int flags, bus_space_handle_t *handlep)
904 {
905 return (bus_space_map(tag, hnd, size, flags, handlep));
906 }
907
908 void
kasan_bus_space_unmap(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)909 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
910 bus_size_t size)
911 {
912 bus_space_unmap(tag, hnd, size);
913 }
914
915 int
kasan_bus_space_subregion(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,bus_space_handle_t * handlep)916 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
917 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
918 {
919 return (bus_space_subregion(tag, hnd, offset, size, handlep));
920 }
921
922 void
kasan_bus_space_free(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)923 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
924 bus_size_t size)
925 {
926 bus_space_free(tag, hnd, size);
927 }
928
929 void
kasan_bus_space_barrier(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,int flags)930 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
931 bus_size_t offset, bus_size_t size, int flags)
932 {
933 bus_space_barrier(tag, hnd, offset, size, flags);
934 }
935
936 #define ASAN_BUS_READ_FUNC(func, width, type) \
937 type kasan_bus_space_read##func##_##width(bus_space_tag_t tag, \
938 bus_space_handle_t hnd, bus_size_t offset) \
939 { \
940 return (bus_space_read##func##_##width(tag, hnd, \
941 offset)); \
942 } \
943
944 #define ASAN_BUS_READ_PTR_FUNC(func, width, type) \
945 void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
946 bus_space_handle_t hnd, bus_size_t size, type *buf, \
947 bus_size_t count) \
948 { \
949 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
950 false, __RET_ADDR); \
951 bus_space_read_##func##_##width(tag, hnd, size, buf, \
952 count); \
953 }
954
955 ASAN_BUS_READ_FUNC(, 1, uint8_t)
956 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
957 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
958 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
959 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
960 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
961
962 ASAN_BUS_READ_FUNC(, 2, uint16_t)
963 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
964 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
965 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
966 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
967 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
968
969 ASAN_BUS_READ_FUNC(, 4, uint32_t)
970 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
971 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
972 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
973 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
974 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
975
976 ASAN_BUS_READ_FUNC(, 8, uint64_t)
977 #if defined(__aarch64__)
978 ASAN_BUS_READ_FUNC(_stream, 8, uint64_t)
979 ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
980 ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
981 ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
982 ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
983 #endif
984
985 #define ASAN_BUS_WRITE_FUNC(func, width, type) \
986 void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
987 bus_space_handle_t hnd, bus_size_t offset, type value) \
988 { \
989 bus_space_write##func##_##width(tag, hnd, offset, value);\
990 } \
991
992 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type) \
993 void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
994 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
995 bus_size_t count) \
996 { \
997 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
998 true, __RET_ADDR); \
999 bus_space_write_##func##_##width(tag, hnd, size, buf, \
1000 count); \
1001 }
1002
1003 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
1004 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1005 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1006 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1007 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1008 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1009
1010 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
1011 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1012 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1013 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1014 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1015 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1016
1017 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
1018 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1019 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1020 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1021 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1022 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1023
1024 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
1025
1026 #define ASAN_BUS_SET_FUNC(func, width, type) \
1027 void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
1028 bus_space_handle_t hnd, bus_size_t offset, type value, \
1029 bus_size_t count) \
1030 { \
1031 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1032 count); \
1033 }
1034
1035 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
1036 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
1037 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1038 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1039
1040 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
1041 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
1042 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1043 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1044
1045 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
1046 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
1047 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1048 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1049
1050 #define ASAN_BUS_PEEK_FUNC(width, type) \
1051 int kasan_bus_space_peek_##width(bus_space_tag_t tag, \
1052 bus_space_handle_t hnd, bus_size_t offset, type *valuep) \
1053 { \
1054 return (bus_space_peek_##width(tag, hnd, offset, \
1055 valuep)); \
1056 }
1057
1058 ASAN_BUS_PEEK_FUNC(1, uint8_t)
1059 ASAN_BUS_PEEK_FUNC(2, uint16_t)
1060 ASAN_BUS_PEEK_FUNC(4, uint32_t)
1061 ASAN_BUS_PEEK_FUNC(8, uint64_t)
1062
1063 #define ASAN_BUS_POKE_FUNC(width, type) \
1064 int kasan_bus_space_poke_##width(bus_space_tag_t tag, \
1065 bus_space_handle_t hnd, bus_size_t offset, type value) \
1066 { \
1067 return (bus_space_poke_##width(tag, hnd, offset, \
1068 value)); \
1069 }
1070
1071 ASAN_BUS_POKE_FUNC(1, uint8_t)
1072 ASAN_BUS_POKE_FUNC(2, uint16_t)
1073 ASAN_BUS_POKE_FUNC(4, uint32_t)
1074 ASAN_BUS_POKE_FUNC(8, uint64_t)
1075
1076 /* -------------------------------------------------------------------------- */
1077
1078 void __asan_register_globals(struct __asan_global *, size_t);
1079 void __asan_unregister_globals(struct __asan_global *, size_t);
1080
1081 void
__asan_register_globals(struct __asan_global * globals,size_t n)1082 __asan_register_globals(struct __asan_global *globals, size_t n)
1083 {
1084 size_t i;
1085
1086 for (i = 0; i < n; i++) {
1087 kasan_mark(globals[i].beg, globals[i].size,
1088 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1089 }
1090 }
1091
1092 void
__asan_unregister_globals(struct __asan_global * globals,size_t n)1093 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1094 {
1095 size_t i;
1096
1097 for (i = 0; i < n; i++) {
1098 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
1099 globals[i].size_with_redzone, 0);
1100 }
1101 }
1102
1103 #define ASAN_LOAD_STORE(size) \
1104 void __asan_load##size(unsigned long); \
1105 void __asan_load##size(unsigned long addr) \
1106 { \
1107 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1108 } \
1109 void __asan_load##size##_noabort(unsigned long); \
1110 void __asan_load##size##_noabort(unsigned long addr) \
1111 { \
1112 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1113 } \
1114 void __asan_store##size(unsigned long); \
1115 void __asan_store##size(unsigned long addr) \
1116 { \
1117 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1118 } \
1119 void __asan_store##size##_noabort(unsigned long); \
1120 void __asan_store##size##_noabort(unsigned long addr) \
1121 { \
1122 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1123 }
1124
1125 ASAN_LOAD_STORE(1);
1126 ASAN_LOAD_STORE(2);
1127 ASAN_LOAD_STORE(4);
1128 ASAN_LOAD_STORE(8);
1129 ASAN_LOAD_STORE(16);
1130
1131 void __asan_loadN(unsigned long, size_t);
1132 void __asan_loadN_noabort(unsigned long, size_t);
1133 void __asan_storeN(unsigned long, size_t);
1134 void __asan_storeN_noabort(unsigned long, size_t);
1135 void __asan_handle_no_return(void);
1136
1137 void
__asan_loadN(unsigned long addr,size_t size)1138 __asan_loadN(unsigned long addr, size_t size)
1139 {
1140 kasan_shadow_check(addr, size, false, __RET_ADDR);
1141 }
1142
1143 void
__asan_loadN_noabort(unsigned long addr,size_t size)1144 __asan_loadN_noabort(unsigned long addr, size_t size)
1145 {
1146 kasan_shadow_check(addr, size, false, __RET_ADDR);
1147 }
1148
1149 void
__asan_storeN(unsigned long addr,size_t size)1150 __asan_storeN(unsigned long addr, size_t size)
1151 {
1152 kasan_shadow_check(addr, size, true, __RET_ADDR);
1153 }
1154
1155 void
__asan_storeN_noabort(unsigned long addr,size_t size)1156 __asan_storeN_noabort(unsigned long addr, size_t size)
1157 {
1158 kasan_shadow_check(addr, size, true, __RET_ADDR);
1159 }
1160
1161 void
__asan_handle_no_return(void)1162 __asan_handle_no_return(void)
1163 {
1164 /* nothing */
1165 }
1166
1167 #define ASAN_SET_SHADOW(byte) \
1168 void __asan_set_shadow_##byte(void *, size_t); \
1169 void __asan_set_shadow_##byte(void *addr, size_t size) \
1170 { \
1171 __builtin_memset((void *)addr, 0x##byte, size); \
1172 }
1173
1174 ASAN_SET_SHADOW(00);
1175 ASAN_SET_SHADOW(f1);
1176 ASAN_SET_SHADOW(f2);
1177 ASAN_SET_SHADOW(f3);
1178 ASAN_SET_SHADOW(f5);
1179 ASAN_SET_SHADOW(f8);
1180
1181 void __asan_poison_stack_memory(const void *, size_t);
1182 void __asan_unpoison_stack_memory(const void *, size_t);
1183
1184 void
__asan_poison_stack_memory(const void * addr,size_t size)1185 __asan_poison_stack_memory(const void *addr, size_t size)
1186 {
1187 size = roundup(size, KASAN_SHADOW_SCALE);
1188 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1189 }
1190
1191 void
__asan_unpoison_stack_memory(const void * addr,size_t size)1192 __asan_unpoison_stack_memory(const void *addr, size_t size)
1193 {
1194 kasan_shadow_Nbyte_markvalid(addr, size);
1195 }
1196
1197 void __asan_alloca_poison(const void *, size_t);
1198 void __asan_allocas_unpoison(const void *, const void *);
1199
1200 void
__asan_alloca_poison(const void * addr,size_t size)1201 __asan_alloca_poison(const void *addr, size_t size)
1202 {
1203 const void *l, *r;
1204
1205 KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1206 ("%s: invalid address %p", __func__, addr));
1207
1208 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1209 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1210
1211 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1212 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1213 KASAN_STACK_MID);
1214 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1215 }
1216
1217 void
__asan_allocas_unpoison(const void * stkbegin,const void * stkend)1218 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1219 {
1220 size_t size;
1221
1222 if (__predict_false(!stkbegin))
1223 return;
1224 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1225 return;
1226 size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1227
1228 kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1229 }
1230
1231 void __asan_poison_memory_region(const void *addr, size_t size);
1232 void __asan_unpoison_memory_region(const void *addr, size_t size);
1233
1234 void
__asan_poison_memory_region(const void * addr,size_t size)1235 __asan_poison_memory_region(const void *addr, size_t size)
1236 {
1237 }
1238
1239 void
__asan_unpoison_memory_region(const void * addr,size_t size)1240 __asan_unpoison_memory_region(const void *addr, size_t size)
1241 {
1242 }
1243