1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 /* 4 * This header provides generic wrappers for memory access instrumentation that 5 * the compiler cannot emit for: KASAN, KCSAN, KMSAN. 6 */ 7 #ifndef _LINUX_INSTRUMENTED_H 8 #define _LINUX_INSTRUMENTED_H 9 10 #include <linux/bug.h> 11 #include <linux/compiler.h> 12 #include <linux/kasan-checks.h> 13 #include <linux/kcsan-checks.h> 14 #include <linux/kmsan-checks.h> 15 #include <linux/types.h> 16 17 /** 18 * instrument_read - instrument regular read access 19 * @v: address of access 20 * @size: size of access 21 * 22 * Instrument a regular read access. The instrumentation should be inserted 23 * before the actual read happens. 24 */ 25 static __always_inline void instrument_read(const volatile void *v, size_t size) 26 { 27 kasan_check_read(v, size); 28 kcsan_check_read(v, size); 29 } 30 31 /** 32 * instrument_write - instrument regular write access 33 * @v: address of access 34 * @size: size of access 35 * 36 * Instrument a regular write access. The instrumentation should be inserted 37 * before the actual write happens. 38 */ 39 static __always_inline void instrument_write(const volatile void *v, size_t size) 40 { 41 kasan_check_write(v, size); 42 kcsan_check_write(v, size); 43 } 44 45 /** 46 * instrument_read_write - instrument regular read-write access 47 * @v: address of access 48 * @size: size of access 49 * 50 * Instrument a regular write access. The instrumentation should be inserted 51 * before the actual write happens. 52 */ 53 static __always_inline void instrument_read_write(const volatile void *v, size_t size) 54 { 55 kasan_check_write(v, size); 56 kcsan_check_read_write(v, size); 57 } 58 59 static __always_inline void instrument_atomic_check_alignment(const volatile void *v, size_t size) 60 { 61 #ifndef __DISABLE_EXPORTS 62 if (IS_ENABLED(CONFIG_DEBUG_ATOMIC)) { 63 unsigned int mask = size - 1; 64 65 if (IS_ENABLED(CONFIG_DEBUG_ATOMIC_LARGEST_ALIGN)) 66 mask &= sizeof(struct { long x; } __aligned_largest) - 1; 67 WARN_ON_ONCE((unsigned long)v & mask); 68 } 69 #endif 70 } 71 72 /** 73 * instrument_atomic_read - instrument atomic read access 74 * @v: address of access 75 * @size: size of access 76 * 77 * Instrument an atomic read access. The instrumentation should be inserted 78 * before the actual read happens. 79 */ 80 static __always_inline void instrument_atomic_read(const volatile void *v, size_t size) 81 { 82 kasan_check_read(v, size); 83 kcsan_check_atomic_read(v, size); 84 instrument_atomic_check_alignment(v, size); 85 } 86 87 /** 88 * instrument_atomic_write - instrument atomic write access 89 * @v: address of access 90 * @size: size of access 91 * 92 * Instrument an atomic write access. The instrumentation should be inserted 93 * before the actual write happens. 94 */ 95 static __always_inline void instrument_atomic_write(const volatile void *v, size_t size) 96 { 97 kasan_check_write(v, size); 98 kcsan_check_atomic_write(v, size); 99 instrument_atomic_check_alignment(v, size); 100 } 101 102 /** 103 * instrument_atomic_read_write - instrument atomic read-write access 104 * @v: address of access 105 * @size: size of access 106 * 107 * Instrument an atomic read-write access. The instrumentation should be 108 * inserted before the actual write happens. 109 */ 110 static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size) 111 { 112 kasan_check_write(v, size); 113 kcsan_check_atomic_read_write(v, size); 114 instrument_atomic_check_alignment(v, size); 115 } 116 117 /** 118 * instrument_copy_to_user - instrument reads of copy_to_user 119 * @to: destination address 120 * @from: source address 121 * @n: number of bytes to copy 122 * 123 * Instrument reads from kernel memory, that are due to copy_to_user (and 124 * variants). The instrumentation must be inserted before the accesses. 125 */ 126 static __always_inline void 127 instrument_copy_to_user(void __user *to, const void *from, unsigned long n) 128 { 129 kasan_check_read(from, n); 130 kcsan_check_read(from, n); 131 kmsan_copy_to_user(to, from, n, 0); 132 } 133 134 /** 135 * instrument_copy_from_user_before - add instrumentation before copy_from_user 136 * @to: destination address 137 * @from: source address 138 * @n: number of bytes to copy 139 * 140 * Instrument writes to kernel memory, that are due to copy_from_user (and 141 * variants). The instrumentation should be inserted before the accesses. 142 */ 143 static __always_inline void 144 instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n) 145 { 146 kasan_check_write(to, n); 147 kcsan_check_write(to, n); 148 } 149 150 /** 151 * instrument_copy_from_user_after - add instrumentation after copy_from_user 152 * @to: destination address 153 * @from: source address 154 * @n: number of bytes to copy 155 * @left: number of bytes not copied (as returned by copy_from_user) 156 * 157 * Instrument writes to kernel memory, that are due to copy_from_user (and 158 * variants). The instrumentation should be inserted after the accesses. 159 */ 160 static __always_inline void 161 instrument_copy_from_user_after(const void *to, const void __user *from, 162 unsigned long n, unsigned long left) 163 { 164 kmsan_unpoison_memory(to, n - left); 165 } 166 167 /** 168 * instrument_memcpy_before - add instrumentation before non-instrumented memcpy 169 * @to: destination address 170 * @from: source address 171 * @n: number of bytes to copy 172 * 173 * Instrument memory accesses that happen in custom memcpy implementations. The 174 * instrumentation should be inserted before the memcpy call. 175 */ 176 static __always_inline void instrument_memcpy_before(void *to, const void *from, 177 unsigned long n) 178 { 179 kasan_check_write(to, n); 180 kasan_check_read(from, n); 181 kcsan_check_write(to, n); 182 kcsan_check_read(from, n); 183 } 184 185 /** 186 * instrument_memcpy_after - add instrumentation after non-instrumented memcpy 187 * @to: destination address 188 * @from: source address 189 * @n: number of bytes to copy 190 * @left: number of bytes not copied (if known) 191 * 192 * Instrument memory accesses that happen in custom memcpy implementations. The 193 * instrumentation should be inserted after the memcpy call. 194 */ 195 static __always_inline void instrument_memcpy_after(void *to, const void *from, 196 unsigned long n, 197 unsigned long left) 198 { 199 kmsan_memmove(to, from, n - left); 200 } 201 202 /** 203 * instrument_get_user() - add instrumentation to get_user()-like macros 204 * @to: destination variable, may not be address-taken 205 * 206 * get_user() and friends are fragile, so it may depend on the implementation 207 * whether the instrumentation happens before or after the data is copied from 208 * the userspace. 209 */ 210 #define instrument_get_user(to) \ 211 ({ \ 212 u64 __tmp = (u64)(to); \ 213 kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \ 214 to = __tmp; \ 215 }) 216 217 218 /** 219 * instrument_put_user() - add instrumentation to put_user()-like macros 220 * @from: source address 221 * @ptr: userspace pointer to copy to 222 * @size: number of bytes to copy 223 * 224 * put_user() and friends are fragile, so it may depend on the implementation 225 * whether the instrumentation happens before or after the data is copied from 226 * the userspace. 227 */ 228 #define instrument_put_user(from, ptr, size) \ 229 ({ \ 230 kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \ 231 }) 232 233 #endif /* _LINUX_INSTRUMENTED_H */ 234