1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 /* 4 * This header provides generic wrappers for memory access instrumentation that 5 * the compiler cannot emit for: KASAN, KCSAN, KMSAN. 6 */ 7 #ifndef _LINUX_INSTRUMENTED_H 8 #define _LINUX_INSTRUMENTED_H 9 10 #include <linux/compiler.h> 11 #include <linux/kasan-checks.h> 12 #include <linux/kcsan-checks.h> 13 #include <linux/kmsan-checks.h> 14 #include <linux/types.h> 15 16 /** 17 * instrument_read - instrument regular read access 18 * 19 * Instrument a regular read access. The instrumentation should be inserted 20 * before the actual read happens. 21 * 22 * @ptr address of access 23 * @size size of access 24 */ 25 static __always_inline void instrument_read(const volatile void *v, size_t size) 26 { 27 kasan_check_read(v, size); 28 kcsan_check_read(v, size); 29 } 30 31 /** 32 * instrument_write - instrument regular write access 33 * 34 * Instrument a regular write access. The instrumentation should be inserted 35 * before the actual write happens. 36 * 37 * @ptr address of access 38 * @size size of access 39 */ 40 static __always_inline void instrument_write(const volatile void *v, size_t size) 41 { 42 kasan_check_write(v, size); 43 kcsan_check_write(v, size); 44 } 45 46 /** 47 * instrument_read_write - instrument regular read-write access 48 * 49 * Instrument a regular write access. The instrumentation should be inserted 50 * before the actual write happens. 51 * 52 * @ptr address of access 53 * @size size of access 54 */ 55 static __always_inline void instrument_read_write(const volatile void *v, size_t size) 56 { 57 kasan_check_write(v, size); 58 kcsan_check_read_write(v, size); 59 } 60 61 /** 62 * instrument_atomic_read - instrument atomic read access 63 * 64 * Instrument an atomic read access. The instrumentation should be inserted 65 * before the actual read happens. 66 * 67 * @ptr address of access 68 * @size size of access 69 */ 70 static __always_inline void instrument_atomic_read(const volatile void *v, size_t size) 71 { 72 kasan_check_read(v, size); 73 kcsan_check_atomic_read(v, size); 74 } 75 76 /** 77 * instrument_atomic_write - instrument atomic write access 78 * 79 * Instrument an atomic write access. The instrumentation should be inserted 80 * before the actual write happens. 81 * 82 * @ptr address of access 83 * @size size of access 84 */ 85 static __always_inline void instrument_atomic_write(const volatile void *v, size_t size) 86 { 87 kasan_check_write(v, size); 88 kcsan_check_atomic_write(v, size); 89 } 90 91 /** 92 * instrument_atomic_read_write - instrument atomic read-write access 93 * 94 * Instrument an atomic read-write access. The instrumentation should be 95 * inserted before the actual write happens. 96 * 97 * @ptr address of access 98 * @size size of access 99 */ 100 static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size) 101 { 102 kasan_check_write(v, size); 103 kcsan_check_atomic_read_write(v, size); 104 } 105 106 /** 107 * instrument_copy_to_user - instrument reads of copy_to_user 108 * 109 * Instrument reads from kernel memory, that are due to copy_to_user (and 110 * variants). The instrumentation must be inserted before the accesses. 111 * 112 * @to destination address 113 * @from source address 114 * @n number of bytes to copy 115 */ 116 static __always_inline void 117 instrument_copy_to_user(void __user *to, const void *from, unsigned long n) 118 { 119 kasan_check_read(from, n); 120 kcsan_check_read(from, n); 121 kmsan_copy_to_user(to, from, n, 0); 122 } 123 124 /** 125 * instrument_copy_from_user_before - add instrumentation before copy_from_user 126 * 127 * Instrument writes to kernel memory, that are due to copy_from_user (and 128 * variants). The instrumentation should be inserted before the accesses. 129 * 130 * @to destination address 131 * @from source address 132 * @n number of bytes to copy 133 */ 134 static __always_inline void 135 instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n) 136 { 137 kasan_check_write(to, n); 138 kcsan_check_write(to, n); 139 } 140 141 /** 142 * instrument_copy_from_user_after - add instrumentation after copy_from_user 143 * 144 * Instrument writes to kernel memory, that are due to copy_from_user (and 145 * variants). The instrumentation should be inserted after the accesses. 146 * 147 * @to destination address 148 * @from source address 149 * @n number of bytes to copy 150 * @left number of bytes not copied (as returned by copy_from_user) 151 */ 152 static __always_inline void 153 instrument_copy_from_user_after(const void *to, const void __user *from, 154 unsigned long n, unsigned long left) 155 { 156 kmsan_unpoison_memory(to, n - left); 157 } 158 159 /** 160 * instrument_get_user() - add instrumentation to get_user()-like macros 161 * 162 * get_user() and friends are fragile, so it may depend on the implementation 163 * whether the instrumentation happens before or after the data is copied from 164 * the userspace. 165 * 166 * @to destination variable, may not be address-taken 167 */ 168 #define instrument_get_user(to) \ 169 ({ \ 170 u64 __tmp = (u64)(to); \ 171 kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \ 172 to = __tmp; \ 173 }) 174 175 176 /** 177 * instrument_put_user() - add instrumentation to put_user()-like macros 178 * 179 * put_user() and friends are fragile, so it may depend on the implementation 180 * whether the instrumentation happens before or after the data is copied from 181 * the userspace. 182 * 183 * @from source address 184 * @ptr userspace pointer to copy to 185 * @size number of bytes to copy 186 */ 187 #define instrument_put_user(from, ptr, size) \ 188 ({ \ 189 kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \ 190 }) 191 192 #endif /* _LINUX_INSTRUMENTED_H */ 193