xref: /freebsd/sys/kern/subr_asan.c (revision 7ef62cebc2f965b0f640263e179276928885e33d)
1 /*	$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $	*/
2 
3 /*
4  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
5  * All rights reserved.
6  *
7  * This code is part of the KASAN subsystem of the NetBSD kernel.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #define	SAN_RUNTIME
32 
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35 #if 0
36 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
37 #endif
38 
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/asan.h>
42 #include <sys/kernel.h>
43 #include <sys/stack.h>
44 #include <sys/sysctl.h>
45 
46 #include <machine/asan.h>
47 #include <machine/bus.h>
48 
49 /* ASAN constants. Part of the compiler ABI. */
50 #define KASAN_SHADOW_MASK		(KASAN_SHADOW_SCALE - 1)
51 #define KASAN_ALLOCA_SCALE_SIZE		32
52 
53 /* ASAN ABI version. */
54 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
55 #define ASAN_ABI_VERSION	8
56 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
57 #define ASAN_ABI_VERSION	8
58 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
59 #define ASAN_ABI_VERSION	6
60 #else
61 #error "Unsupported compiler version"
62 #endif
63 
64 #define __RET_ADDR	(unsigned long)__builtin_return_address(0)
65 
66 /* Global variable descriptor. Part of the compiler ABI.  */
67 struct __asan_global_source_location {
68 	const char *filename;
69 	int line_no;
70 	int column_no;
71 };
72 
73 struct __asan_global {
74 	const void *beg;		/* address of the global variable */
75 	size_t size;			/* size of the global variable */
76 	size_t size_with_redzone;	/* size with the redzone */
77 	const void *name;		/* name of the variable */
78 	const void *module_name;	/* name of the module where the var is declared */
79 	unsigned long has_dynamic_init;	/* the var has dyn initializer (c++) */
80 	struct __asan_global_source_location *location;
81 #if ASAN_ABI_VERSION >= 7
82 	uintptr_t odr_indicator;	/* the address of the ODR indicator symbol */
83 #endif
84 };
85 
86 FEATURE(kasan, "Kernel address sanitizer");
87 
88 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
89     "KASAN options");
90 
91 static int panic_on_violation = 1;
92 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
93     &panic_on_violation, 0,
94     "Panic if an invalid access is detected");
95 
96 static bool kasan_enabled __read_mostly = false;
97 
98 /* -------------------------------------------------------------------------- */
99 
100 void
101 kasan_shadow_map(vm_offset_t addr, size_t size)
102 {
103 	size_t sz, npages, i;
104 	vm_offset_t sva, eva;
105 
106 	KASSERT(addr % KASAN_SHADOW_SCALE == 0,
107 	    ("%s: invalid address %#lx", __func__, addr));
108 
109 	sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
110 
111 	sva = kasan_md_addr_to_shad(addr);
112 	eva = kasan_md_addr_to_shad(addr) + sz;
113 
114 	sva = rounddown(sva, PAGE_SIZE);
115 	eva = roundup(eva, PAGE_SIZE);
116 
117 	npages = (eva - sva) / PAGE_SIZE;
118 
119 	KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
120 	    ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
121 
122 	for (i = 0; i < npages; i++)
123 		pmap_san_enter(sva + ptoa(i));
124 }
125 
126 void
127 kasan_init(void)
128 {
129 	int disabled;
130 
131 	disabled = 0;
132 	TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
133 	if (disabled)
134 		return;
135 
136 	/* MD initialization. */
137 	kasan_md_init();
138 
139 	/* Now officially enabled. */
140 	kasan_enabled = true;
141 }
142 
143 void
144 kasan_init_early(vm_offset_t stack, size_t size)
145 {
146 	kasan_md_init_early(stack, size);
147 }
148 
149 static inline const char *
150 kasan_code_name(uint8_t code)
151 {
152 	switch (code) {
153 	case KASAN_GENERIC_REDZONE:
154 		return "GenericRedZone";
155 	case KASAN_MALLOC_REDZONE:
156 		return "MallocRedZone";
157 	case KASAN_KMEM_REDZONE:
158 		return "KmemRedZone";
159 	case KASAN_UMA_FREED:
160 		return "UMAUseAfterFree";
161 	case KASAN_KSTACK_FREED:
162 		return "KernelStack";
163 	case KASAN_EXEC_ARGS_FREED:
164 		return "ExecKVA";
165 	case 1 ... 7:
166 		return "RedZonePartial";
167 	case KASAN_STACK_LEFT:
168 		return "StackLeft";
169 	case KASAN_STACK_MID:
170 		return "StackMiddle";
171 	case KASAN_STACK_RIGHT:
172 		return "StackRight";
173 	case KASAN_USE_AFTER_RET:
174 		return "UseAfterRet";
175 	case KASAN_USE_AFTER_SCOPE:
176 		return "UseAfterScope";
177 	default:
178 		return "Unknown";
179 	}
180 }
181 
182 #define	REPORT(f, ...) do {				\
183 	if (panic_on_violation) {			\
184 		kasan_enabled = false;			\
185 		panic(f, __VA_ARGS__);			\
186 	} else {					\
187 		struct stack st;			\
188 							\
189 		stack_save(&st);			\
190 		printf(f "\n", __VA_ARGS__);		\
191 		stack_print_ddb(&st);			\
192 	}						\
193 } while (0)
194 
195 static void
196 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
197     uint8_t code)
198 {
199 	REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
200 	    size, (write ? "write" : "read"), addr, kasan_code_name(code),
201 	    code);
202 }
203 
204 static __always_inline void
205 kasan_shadow_1byte_markvalid(unsigned long addr)
206 {
207 	int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
208 	int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
209 
210 	*byte = last;
211 }
212 
213 static __always_inline void
214 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
215 {
216 	size_t i;
217 
218 	for (i = 0; i < size; i++) {
219 		kasan_shadow_1byte_markvalid((unsigned long)addr + i);
220 	}
221 }
222 
223 static __always_inline void
224 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
225 {
226 	void *shad;
227 
228 	if (__predict_false(size == 0))
229 		return;
230 	if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
231 		return;
232 
233 	KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
234 	    ("%s: invalid address %p", __func__, addr));
235 	KASSERT(size % KASAN_SHADOW_SCALE == 0,
236 	    ("%s: invalid size %zu", __func__, size));
237 
238 	shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
239 	size = size >> KASAN_SHADOW_SCALE_SHIFT;
240 
241 	__builtin_memset(shad, code, size);
242 }
243 
244 /*
245  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
246  * and the rest as invalid. There are generally two use cases:
247  *
248  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
249  *    the redzone at the end of the buffer as invalid. If the entire is to be
250  *    marked invalid, origsize will be 0.
251  *
252  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
253  */
254 void
255 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
256 {
257 	size_t i, n, redz;
258 	int8_t *shad;
259 
260 	if (__predict_false(!kasan_enabled))
261 		return;
262 
263 	if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
264 	    (vm_offset_t)addr < DMAP_MAX_ADDRESS)
265 		return;
266 
267 	KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
268 	    (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
269 	    ("%s: invalid address %p", __func__, addr));
270 	KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
271 	    ("%s: invalid address %p", __func__, addr));
272 	redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
273 	KASSERT(redz % KASAN_SHADOW_SCALE == 0,
274 	    ("%s: invalid size %zu", __func__, redz));
275 	shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
276 
277 	/* Chunks of 8 bytes, valid. */
278 	n = size / KASAN_SHADOW_SCALE;
279 	for (i = 0; i < n; i++) {
280 		*shad++ = 0;
281 	}
282 
283 	/* Possibly one chunk, mid. */
284 	if ((size & KASAN_SHADOW_MASK) != 0) {
285 		*shad++ = (size & KASAN_SHADOW_MASK);
286 	}
287 
288 	/* Chunks of 8 bytes, invalid. */
289 	n = redz / KASAN_SHADOW_SCALE;
290 	for (i = 0; i < n; i++) {
291 		*shad++ = code;
292 	}
293 }
294 
295 /* -------------------------------------------------------------------------- */
296 
297 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) 		\
298 	(addr >> KASAN_SHADOW_SCALE_SHIFT) !=			\
299 	    ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
300 
301 static __always_inline bool
302 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
303 {
304 	int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
305 	int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
306 
307 	if (__predict_true(*byte == 0 || last <= *byte)) {
308 		return (true);
309 	}
310 	*code = *byte;
311 	return (false);
312 }
313 
314 static __always_inline bool
315 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
316 {
317 	int8_t *byte, last;
318 
319 	if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
320 		return (kasan_shadow_1byte_isvalid(addr, code) &&
321 		    kasan_shadow_1byte_isvalid(addr+1, code));
322 	}
323 
324 	byte = (int8_t *)kasan_md_addr_to_shad(addr);
325 	last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
326 
327 	if (__predict_true(*byte == 0 || last <= *byte)) {
328 		return (true);
329 	}
330 	*code = *byte;
331 	return (false);
332 }
333 
334 static __always_inline bool
335 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
336 {
337 	int8_t *byte, last;
338 
339 	if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
340 		return (kasan_shadow_2byte_isvalid(addr, code) &&
341 		    kasan_shadow_2byte_isvalid(addr+2, code));
342 	}
343 
344 	byte = (int8_t *)kasan_md_addr_to_shad(addr);
345 	last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
346 
347 	if (__predict_true(*byte == 0 || last <= *byte)) {
348 		return (true);
349 	}
350 	*code = *byte;
351 	return (false);
352 }
353 
354 static __always_inline bool
355 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
356 {
357 	int8_t *byte, last;
358 
359 	if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
360 		return (kasan_shadow_4byte_isvalid(addr, code) &&
361 		    kasan_shadow_4byte_isvalid(addr+4, code));
362 	}
363 
364 	byte = (int8_t *)kasan_md_addr_to_shad(addr);
365 	last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
366 
367 	if (__predict_true(*byte == 0 || last <= *byte)) {
368 		return (true);
369 	}
370 	*code = *byte;
371 	return (false);
372 }
373 
374 static __always_inline bool
375 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
376 {
377 	size_t i;
378 
379 	for (i = 0; i < size; i++) {
380 		if (!kasan_shadow_1byte_isvalid(addr+i, code))
381 			return (false);
382 	}
383 
384 	return (true);
385 }
386 
387 static __always_inline void
388 kasan_shadow_check(unsigned long addr, size_t size, bool write,
389     unsigned long retaddr)
390 {
391 	uint8_t code;
392 	bool valid;
393 
394 	if (__predict_false(!kasan_enabled))
395 		return;
396 	if (__predict_false(size == 0))
397 		return;
398 	if (__predict_false(kasan_md_unsupported(addr)))
399 		return;
400 	if (KERNEL_PANICKED())
401 		return;
402 
403 	if (__builtin_constant_p(size)) {
404 		switch (size) {
405 		case 1:
406 			valid = kasan_shadow_1byte_isvalid(addr, &code);
407 			break;
408 		case 2:
409 			valid = kasan_shadow_2byte_isvalid(addr, &code);
410 			break;
411 		case 4:
412 			valid = kasan_shadow_4byte_isvalid(addr, &code);
413 			break;
414 		case 8:
415 			valid = kasan_shadow_8byte_isvalid(addr, &code);
416 			break;
417 		default:
418 			valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
419 			break;
420 		}
421 	} else {
422 		valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
423 	}
424 
425 	if (__predict_false(!valid)) {
426 		kasan_report(addr, size, write, retaddr, code);
427 	}
428 }
429 
430 /* -------------------------------------------------------------------------- */
431 
432 void *
433 kasan_memcpy(void *dst, const void *src, size_t len)
434 {
435 	kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
436 	kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
437 	return (__builtin_memcpy(dst, src, len));
438 }
439 
440 int
441 kasan_memcmp(const void *b1, const void *b2, size_t len)
442 {
443 	kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
444 	kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
445 	return (__builtin_memcmp(b1, b2, len));
446 }
447 
448 void *
449 kasan_memset(void *b, int c, size_t len)
450 {
451 	kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
452 	return (__builtin_memset(b, c, len));
453 }
454 
455 void *
456 kasan_memmove(void *dst, const void *src, size_t len)
457 {
458 	kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
459 	kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
460 	return (__builtin_memmove(dst, src, len));
461 }
462 
463 size_t
464 kasan_strlen(const char *str)
465 {
466 	const char *s;
467 
468 	s = str;
469 	while (1) {
470 		kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
471 		if (*s == '\0')
472 			break;
473 		s++;
474 	}
475 
476 	return (s - str);
477 }
478 
479 char *
480 kasan_strcpy(char *dst, const char *src)
481 {
482 	char *save = dst;
483 
484 	while (1) {
485 		kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
486 		kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
487 		*dst = *src;
488 		if (*src == '\0')
489 			break;
490 		src++, dst++;
491 	}
492 
493 	return save;
494 }
495 
496 int
497 kasan_strcmp(const char *s1, const char *s2)
498 {
499 	while (1) {
500 		kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
501 		kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
502 		if (*s1 != *s2)
503 			break;
504 		if (*s1 == '\0')
505 			return 0;
506 		s1++, s2++;
507 	}
508 
509 	return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
510 }
511 
512 int
513 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
514 {
515 	kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
516 	return (copyin(uaddr, kaddr, len));
517 }
518 
519 int
520 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
521 {
522 	kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
523 	return (copyinstr(uaddr, kaddr, len, done));
524 }
525 
526 int
527 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
528 {
529 	kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
530 	return (copyout(kaddr, uaddr, len));
531 }
532 
533 /* -------------------------------------------------------------------------- */
534 
535 int
536 kasan_fubyte(volatile const void *base)
537 {
538 	return (fubyte(base));
539 }
540 
541 int
542 kasan_fuword16(volatile const void *base)
543 {
544 	return (fuword16(base));
545 }
546 
547 int
548 kasan_fueword(volatile const void *base, long *val)
549 {
550 	kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
551 	return (fueword(base, val));
552 }
553 
554 int
555 kasan_fueword32(volatile const void *base, int32_t *val)
556 {
557 	kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
558 	return (fueword32(base, val));
559 }
560 
561 int
562 kasan_fueword64(volatile const void *base, int64_t *val)
563 {
564 	kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
565 	return (fueword64(base, val));
566 }
567 
568 int
569 kasan_subyte(volatile void *base, int byte)
570 {
571 	return (subyte(base, byte));
572 }
573 
574 int
575 kasan_suword(volatile void *base, long word)
576 {
577 	return (suword(base, word));
578 }
579 
580 int
581 kasan_suword16(volatile void *base, int word)
582 {
583 	return (suword16(base, word));
584 }
585 
586 int
587 kasan_suword32(volatile void *base, int32_t word)
588 {
589 	return (suword32(base, word));
590 }
591 
592 int
593 kasan_suword64(volatile void *base, int64_t word)
594 {
595 	return (suword64(base, word));
596 }
597 
598 int
599 kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
600     uint32_t newval)
601 {
602 	kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
603 	    __RET_ADDR);
604 	return (casueword32(base, oldval, oldvalp, newval));
605 }
606 
607 int
608 kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
609     u_long newval)
610 {
611 	kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
612 	    __RET_ADDR);
613 	return (casueword(base, oldval, oldvalp, newval));
614 }
615 
616 /* -------------------------------------------------------------------------- */
617 
618 #include <machine/atomic.h>
619 #include <sys/atomic_san.h>
620 
621 #define _ASAN_ATOMIC_FUNC_ADD(name, type)				\
622 	void kasan_atomic_add_##name(volatile type *ptr, type val)	\
623 	{								\
624 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
625 		    __RET_ADDR);					\
626 		atomic_add_##name(ptr, val);				\
627 	}
628 
629 #define	ASAN_ATOMIC_FUNC_ADD(name, type)				\
630 	_ASAN_ATOMIC_FUNC_ADD(name, type)				\
631 	_ASAN_ATOMIC_FUNC_ADD(acq_##name, type)				\
632 	_ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
633 
634 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)				\
635 	void kasan_atomic_subtract_##name(volatile type *ptr, type val)	\
636 	{								\
637 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
638 		    __RET_ADDR);					\
639 		atomic_subtract_##name(ptr, val);			\
640 	}
641 
642 #define	ASAN_ATOMIC_FUNC_SUBTRACT(name, type)				\
643 	_ASAN_ATOMIC_FUNC_SUBTRACT(name, type)				\
644 	_ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)			\
645 	_ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
646 
647 #define _ASAN_ATOMIC_FUNC_SET(name, type)				\
648 	void kasan_atomic_set_##name(volatile type *ptr, type val)	\
649 	{								\
650 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
651 		    __RET_ADDR);					\
652 		atomic_set_##name(ptr, val);				\
653 	}
654 
655 #define	ASAN_ATOMIC_FUNC_SET(name, type)				\
656 	_ASAN_ATOMIC_FUNC_SET(name, type)				\
657 	_ASAN_ATOMIC_FUNC_SET(acq_##name, type)				\
658 	_ASAN_ATOMIC_FUNC_SET(rel_##name, type)
659 
660 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type)				\
661 	void kasan_atomic_clear_##name(volatile type *ptr, type val)	\
662 	{								\
663 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
664 		    __RET_ADDR);					\
665 		atomic_clear_##name(ptr, val);				\
666 	}
667 
668 #define	ASAN_ATOMIC_FUNC_CLEAR(name, type)				\
669 	_ASAN_ATOMIC_FUNC_CLEAR(name, type)				\
670 	_ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type)			\
671 	_ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
672 
673 #define	ASAN_ATOMIC_FUNC_FETCHADD(name, type)				\
674 	type kasan_atomic_fetchadd_##name(volatile type *ptr, type val)	\
675 	{								\
676 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
677 		    __RET_ADDR);					\
678 		return (atomic_fetchadd_##name(ptr, val));		\
679 	}
680 
681 #define	ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)			\
682 	type kasan_atomic_readandclear_##name(volatile type *ptr)	\
683 	{								\
684 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
685 		    __RET_ADDR);					\
686 		return (atomic_readandclear_##name(ptr));		\
687 	}
688 
689 #define	ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)			\
690 	int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
691 	{								\
692 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
693 		    __RET_ADDR);					\
694 		return (atomic_testandclear_##name(ptr, v));		\
695 	}
696 
697 #define	ASAN_ATOMIC_FUNC_TESTANDSET(name, type)				\
698 	int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
699 	{								\
700 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
701 		    __RET_ADDR);					\
702 		return (atomic_testandset_##name(ptr, v));		\
703 	}
704 
705 #define	ASAN_ATOMIC_FUNC_SWAP(name, type)				\
706 	type kasan_atomic_swap_##name(volatile type *ptr, type val)	\
707 	{								\
708 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
709 		    __RET_ADDR);					\
710 		return (atomic_swap_##name(ptr, val));			\
711 	}
712 
713 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type)				\
714 	int kasan_atomic_cmpset_##name(volatile type *ptr, type oval,	\
715 	    type nval)							\
716 	{								\
717 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
718 		    __RET_ADDR);					\
719 		return (atomic_cmpset_##name(ptr, oval, nval));		\
720 	}
721 
722 #define	ASAN_ATOMIC_FUNC_CMPSET(name, type)				\
723 	_ASAN_ATOMIC_FUNC_CMPSET(name, type)				\
724 	_ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type)			\
725 	_ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
726 
727 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type)				\
728 	int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval,	\
729 	    type nval)							\
730 	{								\
731 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
732 		    __RET_ADDR);					\
733 		return (atomic_fcmpset_##name(ptr, oval, nval));	\
734 	}
735 
736 #define	ASAN_ATOMIC_FUNC_FCMPSET(name, type)				\
737 	_ASAN_ATOMIC_FUNC_FCMPSET(name, type)				\
738 	_ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)			\
739 	_ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
740 
741 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)				\
742 	void kasan_atomic_thread_fence_##name(void)			\
743 	{								\
744 		atomic_thread_fence_##name();				\
745 	}
746 
747 #define	_ASAN_ATOMIC_FUNC_LOAD(name, type)				\
748 	type kasan_atomic_load_##name(volatile type *ptr)		\
749 	{								\
750 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
751 		    __RET_ADDR);					\
752 		return (atomic_load_##name(ptr));			\
753 	}
754 
755 #define	ASAN_ATOMIC_FUNC_LOAD(name, type)				\
756 	_ASAN_ATOMIC_FUNC_LOAD(name, type)				\
757 	_ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
758 
759 #define	_ASAN_ATOMIC_FUNC_STORE(name, type)				\
760 	void kasan_atomic_store_##name(volatile type *ptr, type val)	\
761 	{								\
762 		kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,	\
763 		    __RET_ADDR);					\
764 		atomic_store_##name(ptr, val);				\
765 	}
766 
767 #define	ASAN_ATOMIC_FUNC_STORE(name, type)				\
768 	_ASAN_ATOMIC_FUNC_STORE(name, type)				\
769 	_ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
770 
771 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
772 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
773 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
774 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
775 ASAN_ATOMIC_FUNC_ADD(int, u_int);
776 ASAN_ATOMIC_FUNC_ADD(long, u_long);
777 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
778 
779 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
780 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
781 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
782 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
783 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
784 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
785 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
786 
787 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
788 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
789 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
790 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
791 ASAN_ATOMIC_FUNC_SET(int, u_int);
792 ASAN_ATOMIC_FUNC_SET(long, u_long);
793 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
794 
795 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
796 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
797 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
798 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
799 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
800 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
801 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
802 
803 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
804 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
805 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
806 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
807 
808 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
809 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
810 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
811 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
812 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
813 
814 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
815 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
816 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
817 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
818 
819 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
820 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
821 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
822 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
823 
824 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
825 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
826 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
827 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
828 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
829 
830 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
831 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
832 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
833 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
834 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
835 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
836 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
837 
838 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
839 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
840 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
841 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
842 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
843 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
844 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
845 
846 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
847 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
848 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
849 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
850 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
851 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
852 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
853 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
854 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
855 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
856 
857 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
858 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
859 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
860 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
861 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
862 ASAN_ATOMIC_FUNC_STORE(char, u_char);
863 ASAN_ATOMIC_FUNC_STORE(short, u_short);
864 ASAN_ATOMIC_FUNC_STORE(int, u_int);
865 ASAN_ATOMIC_FUNC_STORE(long, u_long);
866 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
867 
868 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
869 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
870 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
871 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
872 
873 void
874 kasan_atomic_interrupt_fence(void)
875 {
876 }
877 
878 /* -------------------------------------------------------------------------- */
879 
880 #include <sys/bus.h>
881 #include <machine/bus.h>
882 #include <sys/bus_san.h>
883 
884 int
885 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
886     int flags, bus_space_handle_t *handlep)
887 {
888 	return (bus_space_map(tag, hnd, size, flags, handlep));
889 }
890 
891 void
892 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
893     bus_size_t size)
894 {
895 	bus_space_unmap(tag, hnd, size);
896 }
897 
898 int
899 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
900     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
901 {
902 	return (bus_space_subregion(tag, hnd, offset, size, handlep));
903 }
904 
905 void
906 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
907     bus_size_t size)
908 {
909 	bus_space_free(tag, hnd, size);
910 }
911 
912 void
913 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
914     bus_size_t offset, bus_size_t size, int flags)
915 {
916 	bus_space_barrier(tag, hnd, offset, size, flags);
917 }
918 
919 #define ASAN_BUS_READ_FUNC(func, width, type)				\
920 	type kasan_bus_space_read##func##_##width(bus_space_tag_t tag,	\
921 	    bus_space_handle_t hnd, bus_size_t offset)			\
922 	{								\
923 		return (bus_space_read##func##_##width(tag, hnd,	\
924 		    offset));						\
925 	}								\
926 
927 #define ASAN_BUS_READ_PTR_FUNC(func, width, type)			\
928 	void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag,	\
929 	    bus_space_handle_t hnd, bus_size_t size, type *buf,		\
930 	    bus_size_t count)						\
931 	{								\
932 		kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
933 		    false, __RET_ADDR);					\
934 		bus_space_read_##func##_##width(tag, hnd, size, buf, 	\
935 		    count);						\
936 	}
937 
938 ASAN_BUS_READ_FUNC(, 1, uint8_t)
939 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
940 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
941 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
942 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
943 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
944 
945 ASAN_BUS_READ_FUNC(, 2, uint16_t)
946 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
947 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
948 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
949 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
950 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
951 
952 ASAN_BUS_READ_FUNC(, 4, uint32_t)
953 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
954 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
955 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
956 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
957 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
958 
959 ASAN_BUS_READ_FUNC(, 8, uint64_t)
960 #if defined(__aarch64__)
961 ASAN_BUS_READ_FUNC(_stream, 8, uint64_t)
962 ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
963 ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
964 ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
965 ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
966 #endif
967 
968 #define	ASAN_BUS_WRITE_FUNC(func, width, type)				\
969 	void kasan_bus_space_write##func##_##width(bus_space_tag_t tag,	\
970 	    bus_space_handle_t hnd, bus_size_t offset, type value)	\
971 	{								\
972 		bus_space_write##func##_##width(tag, hnd, offset, value);\
973 	}								\
974 
975 #define	ASAN_BUS_WRITE_PTR_FUNC(func, width, type)			\
976 	void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
977 	    bus_space_handle_t hnd, bus_size_t size, const type *buf,	\
978 	    bus_size_t count)						\
979 	{								\
980 		kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
981 		    true, __RET_ADDR);					\
982 		bus_space_write_##func##_##width(tag, hnd, size, buf, 	\
983 		    count);						\
984 	}
985 
986 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
987 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
988 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
989 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
990 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
991 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
992 
993 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
994 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
995 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
996 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
997 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
998 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
999 
1000 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
1001 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1002 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1003 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1004 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1005 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1006 
1007 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
1008 
1009 #define	ASAN_BUS_SET_FUNC(func, width, type)				\
1010 	void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag,	\
1011 	    bus_space_handle_t hnd, bus_size_t offset, type value,	\
1012 	    bus_size_t count)						\
1013 	{								\
1014 		bus_space_set_##func##_##width(tag, hnd, offset, value,	\
1015 		    count);						\
1016 	}
1017 
1018 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
1019 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
1020 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1021 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1022 
1023 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
1024 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
1025 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1026 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1027 
1028 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
1029 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
1030 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1031 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1032 
1033 #define	ASAN_BUS_PEEK_FUNC(width, type)					\
1034 	int kasan_bus_space_peek_##width(bus_space_tag_t tag,		\
1035 	    bus_space_handle_t hnd, bus_size_t offset, type *valuep)	\
1036 	{								\
1037 		return (bus_space_peek_##width(tag, hnd, offset,	\
1038 		    valuep));						\
1039 	}
1040 
1041 ASAN_BUS_PEEK_FUNC(1, uint8_t)
1042 ASAN_BUS_PEEK_FUNC(2, uint16_t)
1043 ASAN_BUS_PEEK_FUNC(4, uint32_t)
1044 ASAN_BUS_PEEK_FUNC(8, uint64_t)
1045 
1046 #define	ASAN_BUS_POKE_FUNC(width, type)					\
1047 	int kasan_bus_space_poke_##width(bus_space_tag_t tag,		\
1048 	    bus_space_handle_t hnd, bus_size_t offset, type value)	\
1049 	{								\
1050 		return (bus_space_poke_##width(tag, hnd, offset,	\
1051 		    value));						\
1052 	}
1053 
1054 ASAN_BUS_POKE_FUNC(1, uint8_t)
1055 ASAN_BUS_POKE_FUNC(2, uint16_t)
1056 ASAN_BUS_POKE_FUNC(4, uint32_t)
1057 ASAN_BUS_POKE_FUNC(8, uint64_t)
1058 
1059 /* -------------------------------------------------------------------------- */
1060 
1061 void __asan_register_globals(struct __asan_global *, size_t);
1062 void __asan_unregister_globals(struct __asan_global *, size_t);
1063 
1064 void
1065 __asan_register_globals(struct __asan_global *globals, size_t n)
1066 {
1067 	size_t i;
1068 
1069 	for (i = 0; i < n; i++) {
1070 		kasan_mark(globals[i].beg, globals[i].size,
1071 		    globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1072 	}
1073 }
1074 
1075 void
1076 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1077 {
1078 	size_t i;
1079 
1080 	for (i = 0; i < n; i++) {
1081 		kasan_mark(globals[i].beg, globals[i].size_with_redzone,
1082 		    globals[i].size_with_redzone, 0);
1083 	}
1084 }
1085 
1086 #define ASAN_LOAD_STORE(size)					\
1087 	void __asan_load##size(unsigned long);			\
1088 	void __asan_load##size(unsigned long addr)		\
1089 	{							\
1090 		kasan_shadow_check(addr, size, false, __RET_ADDR);\
1091 	} 							\
1092 	void __asan_load##size##_noabort(unsigned long);	\
1093 	void __asan_load##size##_noabort(unsigned long addr)	\
1094 	{							\
1095 		kasan_shadow_check(addr, size, false, __RET_ADDR);\
1096 	}							\
1097 	void __asan_store##size(unsigned long);			\
1098 	void __asan_store##size(unsigned long addr)		\
1099 	{							\
1100 		kasan_shadow_check(addr, size, true, __RET_ADDR);\
1101 	}							\
1102 	void __asan_store##size##_noabort(unsigned long);	\
1103 	void __asan_store##size##_noabort(unsigned long addr)	\
1104 	{							\
1105 		kasan_shadow_check(addr, size, true, __RET_ADDR);\
1106 	}
1107 
1108 ASAN_LOAD_STORE(1);
1109 ASAN_LOAD_STORE(2);
1110 ASAN_LOAD_STORE(4);
1111 ASAN_LOAD_STORE(8);
1112 ASAN_LOAD_STORE(16);
1113 
1114 void __asan_loadN(unsigned long, size_t);
1115 void __asan_loadN_noabort(unsigned long, size_t);
1116 void __asan_storeN(unsigned long, size_t);
1117 void __asan_storeN_noabort(unsigned long, size_t);
1118 void __asan_handle_no_return(void);
1119 
1120 void
1121 __asan_loadN(unsigned long addr, size_t size)
1122 {
1123 	kasan_shadow_check(addr, size, false, __RET_ADDR);
1124 }
1125 
1126 void
1127 __asan_loadN_noabort(unsigned long addr, size_t size)
1128 {
1129 	kasan_shadow_check(addr, size, false, __RET_ADDR);
1130 }
1131 
1132 void
1133 __asan_storeN(unsigned long addr, size_t size)
1134 {
1135 	kasan_shadow_check(addr, size, true, __RET_ADDR);
1136 }
1137 
1138 void
1139 __asan_storeN_noabort(unsigned long addr, size_t size)
1140 {
1141 	kasan_shadow_check(addr, size, true, __RET_ADDR);
1142 }
1143 
1144 void
1145 __asan_handle_no_return(void)
1146 {
1147 	/* nothing */
1148 }
1149 
1150 #define ASAN_SET_SHADOW(byte) \
1151 	void __asan_set_shadow_##byte(void *, size_t);			\
1152 	void __asan_set_shadow_##byte(void *addr, size_t size)		\
1153 	{								\
1154 		__builtin_memset((void *)addr, 0x##byte, size);		\
1155 	}
1156 
1157 ASAN_SET_SHADOW(00);
1158 ASAN_SET_SHADOW(f1);
1159 ASAN_SET_SHADOW(f2);
1160 ASAN_SET_SHADOW(f3);
1161 ASAN_SET_SHADOW(f5);
1162 ASAN_SET_SHADOW(f8);
1163 
1164 void __asan_poison_stack_memory(const void *, size_t);
1165 void __asan_unpoison_stack_memory(const void *, size_t);
1166 
1167 void
1168 __asan_poison_stack_memory(const void *addr, size_t size)
1169 {
1170 	size = roundup(size, KASAN_SHADOW_SCALE);
1171 	kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1172 }
1173 
1174 void
1175 __asan_unpoison_stack_memory(const void *addr, size_t size)
1176 {
1177 	kasan_shadow_Nbyte_markvalid(addr, size);
1178 }
1179 
1180 void __asan_alloca_poison(const void *, size_t);
1181 void __asan_allocas_unpoison(const void *, const void *);
1182 
1183 void
1184 __asan_alloca_poison(const void *addr, size_t size)
1185 {
1186 	const void *l, *r;
1187 
1188 	KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1189 	    ("%s: invalid address %p", __func__, addr));
1190 
1191 	l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1192 	r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1193 
1194 	kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1195 	kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1196 	    KASAN_STACK_MID);
1197 	kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1198 }
1199 
1200 void
1201 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1202 {
1203 	size_t size;
1204 
1205 	if (__predict_false(!stkbegin))
1206 		return;
1207 	if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1208 		return;
1209 	size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1210 
1211 	kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1212 }
1213 
1214 void __asan_poison_memory_region(const void *addr, size_t size);
1215 void __asan_unpoison_memory_region(const void *addr, size_t size);
1216 
1217 void
1218 __asan_poison_memory_region(const void *addr, size_t size)
1219 {
1220 }
1221 
1222 void
1223 __asan_unpoison_memory_region(const void *addr, size_t size)
1224 {
1225 }
1226