1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __MM_KASAN_KASAN_H 3 #define __MM_KASAN_KASAN_H 4 5 #include <linux/kasan.h> 6 #include <linux/stackdepot.h> 7 8 #ifdef CONFIG_KASAN_HW_TAGS 9 #include <linux/static_key.h> 10 DECLARE_STATIC_KEY_FALSE(kasan_flag_stacktrace); 11 static inline bool kasan_stack_collection_enabled(void) 12 { 13 return static_branch_unlikely(&kasan_flag_stacktrace); 14 } 15 #else 16 static inline bool kasan_stack_collection_enabled(void) 17 { 18 return true; 19 } 20 #endif 21 22 extern bool kasan_flag_panic __ro_after_init; 23 24 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 25 #define KASAN_GRANULE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT) 26 #else 27 #include <asm/mte-kasan.h> 28 #define KASAN_GRANULE_SIZE MTE_GRANULE_SIZE 29 #endif 30 31 #define KASAN_GRANULE_MASK (KASAN_GRANULE_SIZE - 1) 32 33 #define KASAN_MEMORY_PER_SHADOW_PAGE (KASAN_GRANULE_SIZE << PAGE_SHIFT) 34 35 #define KASAN_TAG_KERNEL 0xFF /* native kernel pointers tag */ 36 #define KASAN_TAG_INVALID 0xFE /* inaccessible memory tag */ 37 #define KASAN_TAG_MAX 0xFD /* maximum value for random tags */ 38 39 #ifdef CONFIG_KASAN_GENERIC 40 #define KASAN_FREE_PAGE 0xFF /* page was freed */ 41 #define KASAN_PAGE_REDZONE 0xFE /* redzone for kmalloc_large allocations */ 42 #define KASAN_KMALLOC_REDZONE 0xFC /* redzone inside slub object */ 43 #define KASAN_KMALLOC_FREE 0xFB /* object was freed (kmem_cache_free/kfree) */ 44 #define KASAN_KMALLOC_FREETRACK 0xFA /* object was freed and has free track set */ 45 #else 46 #define KASAN_FREE_PAGE KASAN_TAG_INVALID 47 #define KASAN_PAGE_REDZONE KASAN_TAG_INVALID 48 #define KASAN_KMALLOC_REDZONE KASAN_TAG_INVALID 49 #define KASAN_KMALLOC_FREE KASAN_TAG_INVALID 50 #define KASAN_KMALLOC_FREETRACK KASAN_TAG_INVALID 51 #endif 52 53 #define KASAN_GLOBAL_REDZONE 0xF9 /* redzone for global variable */ 54 #define KASAN_VMALLOC_INVALID 0xF8 /* unallocated space in vmapped page */ 55 56 /* 57 * Stack redzone shadow values 58 * (Those are compiler's ABI, don't change them) 59 */ 60 #define KASAN_STACK_LEFT 0xF1 61 #define KASAN_STACK_MID 0xF2 62 #define KASAN_STACK_RIGHT 0xF3 63 #define KASAN_STACK_PARTIAL 0xF4 64 65 /* 66 * alloca redzone shadow values 67 */ 68 #define KASAN_ALLOCA_LEFT 0xCA 69 #define KASAN_ALLOCA_RIGHT 0xCB 70 71 #define KASAN_ALLOCA_REDZONE_SIZE 32 72 73 /* 74 * Stack frame marker (compiler ABI). 75 */ 76 #define KASAN_CURRENT_STACK_FRAME_MAGIC 0x41B58AB3 77 78 /* Don't break randconfig/all*config builds */ 79 #ifndef KASAN_ABI_VERSION 80 #define KASAN_ABI_VERSION 1 81 #endif 82 83 /* Metadata layout customization. */ 84 #define META_BYTES_PER_BLOCK 1 85 #define META_BLOCKS_PER_ROW 16 86 #define META_BYTES_PER_ROW (META_BLOCKS_PER_ROW * META_BYTES_PER_BLOCK) 87 #define META_MEM_BYTES_PER_ROW (META_BYTES_PER_ROW * KASAN_GRANULE_SIZE) 88 #define META_ROWS_AROUND_ADDR 2 89 90 struct kasan_access_info { 91 const void *access_addr; 92 const void *first_bad_addr; 93 size_t access_size; 94 bool is_write; 95 unsigned long ip; 96 }; 97 98 /* The layout of struct dictated by compiler */ 99 struct kasan_source_location { 100 const char *filename; 101 int line_no; 102 int column_no; 103 }; 104 105 /* The layout of struct dictated by compiler */ 106 struct kasan_global { 107 const void *beg; /* Address of the beginning of the global variable. */ 108 size_t size; /* Size of the global variable. */ 109 size_t size_with_redzone; /* Size of the variable + size of the red zone. 32 bytes aligned */ 110 const void *name; 111 const void *module_name; /* Name of the module where the global variable is declared. */ 112 unsigned long has_dynamic_init; /* This needed for C++ */ 113 #if KASAN_ABI_VERSION >= 4 114 struct kasan_source_location *location; 115 #endif 116 #if KASAN_ABI_VERSION >= 5 117 char *odr_indicator; 118 #endif 119 }; 120 121 /** 122 * Structures to keep alloc and free tracks * 123 */ 124 125 #define KASAN_STACK_DEPTH 64 126 127 struct kasan_track { 128 u32 pid; 129 depot_stack_handle_t stack; 130 }; 131 132 #ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY 133 #define KASAN_NR_FREE_STACKS 5 134 #else 135 #define KASAN_NR_FREE_STACKS 1 136 #endif 137 138 struct kasan_alloc_meta { 139 struct kasan_track alloc_track; 140 #ifdef CONFIG_KASAN_GENERIC 141 /* 142 * call_rcu() call stack is stored into struct kasan_alloc_meta. 143 * The free stack is stored into struct kasan_free_meta. 144 */ 145 depot_stack_handle_t aux_stack[2]; 146 #else 147 struct kasan_track free_track[KASAN_NR_FREE_STACKS]; 148 #endif 149 #ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY 150 u8 free_pointer_tag[KASAN_NR_FREE_STACKS]; 151 u8 free_track_idx; 152 #endif 153 }; 154 155 struct qlist_node { 156 struct qlist_node *next; 157 }; 158 159 /* 160 * Generic mode either stores free meta in the object itself or in the redzone 161 * after the object. In the former case free meta offset is 0, in the latter 162 * case it has some sane value smaller than INT_MAX. Use INT_MAX as free meta 163 * offset when free meta isn't present. 164 */ 165 #define KASAN_NO_FREE_META INT_MAX 166 167 struct kasan_free_meta { 168 #ifdef CONFIG_KASAN_GENERIC 169 /* This field is used while the object is in the quarantine. 170 * Otherwise it might be used for the allocator freelist. 171 */ 172 struct qlist_node quarantine_link; 173 struct kasan_track free_track; 174 #endif 175 }; 176 177 struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, 178 const void *object); 179 #ifdef CONFIG_KASAN_GENERIC 180 struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, 181 const void *object); 182 #endif 183 184 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 185 186 static inline const void *kasan_shadow_to_mem(const void *shadow_addr) 187 { 188 return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET) 189 << KASAN_SHADOW_SCALE_SHIFT); 190 } 191 192 static inline bool addr_has_metadata(const void *addr) 193 { 194 return (addr >= kasan_shadow_to_mem((void *)KASAN_SHADOW_START)); 195 } 196 197 /** 198 * check_memory_region - Check memory region, and report if invalid access. 199 * @addr: the accessed address 200 * @size: the accessed size 201 * @write: true if access is a write access 202 * @ret_ip: return address 203 * @return: true if access was valid, false if invalid 204 */ 205 bool check_memory_region(unsigned long addr, size_t size, bool write, 206 unsigned long ret_ip); 207 208 #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ 209 210 static inline bool addr_has_metadata(const void *addr) 211 { 212 return true; 213 } 214 215 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ 216 217 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) 218 void print_tags(u8 addr_tag, const void *addr); 219 #else 220 static inline void print_tags(u8 addr_tag, const void *addr) { } 221 #endif 222 223 void *find_first_bad_addr(void *addr, size_t size); 224 const char *get_bug_type(struct kasan_access_info *info); 225 void metadata_fetch_row(char *buffer, void *row); 226 227 #if defined(CONFIG_KASAN_GENERIC) && CONFIG_KASAN_STACK 228 void print_address_stack_frame(const void *addr); 229 #else 230 static inline void print_address_stack_frame(const void *addr) { } 231 #endif 232 233 bool kasan_report(unsigned long addr, size_t size, 234 bool is_write, unsigned long ip); 235 void kasan_report_invalid_free(void *object, unsigned long ip); 236 237 struct page *kasan_addr_to_page(const void *addr); 238 239 depot_stack_handle_t kasan_save_stack(gfp_t flags); 240 void kasan_set_track(struct kasan_track *track, gfp_t flags); 241 void kasan_set_free_info(struct kmem_cache *cache, void *object, u8 tag); 242 struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, 243 void *object, u8 tag); 244 245 #if defined(CONFIG_KASAN_GENERIC) && \ 246 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB)) 247 bool quarantine_put(struct kmem_cache *cache, void *object); 248 void quarantine_reduce(void); 249 void quarantine_remove_cache(struct kmem_cache *cache); 250 #else 251 static inline bool quarantine_put(struct kmem_cache *cache, void *object) { return false; } 252 static inline void quarantine_reduce(void) { } 253 static inline void quarantine_remove_cache(struct kmem_cache *cache) { } 254 #endif 255 256 #ifndef arch_kasan_set_tag 257 static inline const void *arch_kasan_set_tag(const void *addr, u8 tag) 258 { 259 return addr; 260 } 261 #endif 262 #ifndef arch_kasan_get_tag 263 #define arch_kasan_get_tag(addr) 0 264 #endif 265 266 #define set_tag(addr, tag) ((void *)arch_kasan_set_tag((addr), (tag))) 267 #define get_tag(addr) arch_kasan_get_tag(addr) 268 269 #ifdef CONFIG_KASAN_HW_TAGS 270 271 #ifndef arch_enable_tagging 272 #define arch_enable_tagging() 273 #endif 274 #ifndef arch_init_tags 275 #define arch_init_tags(max_tag) 276 #endif 277 #ifndef arch_get_random_tag 278 #define arch_get_random_tag() (0xFF) 279 #endif 280 #ifndef arch_get_mem_tag 281 #define arch_get_mem_tag(addr) (0xFF) 282 #endif 283 #ifndef arch_set_mem_tag_range 284 #define arch_set_mem_tag_range(addr, size, tag) ((void *)(addr)) 285 #endif 286 287 #define hw_enable_tagging() arch_enable_tagging() 288 #define hw_init_tags(max_tag) arch_init_tags(max_tag) 289 #define hw_get_random_tag() arch_get_random_tag() 290 #define hw_get_mem_tag(addr) arch_get_mem_tag(addr) 291 #define hw_set_mem_tag_range(addr, size, tag) arch_set_mem_tag_range((addr), (size), (tag)) 292 293 #endif /* CONFIG_KASAN_HW_TAGS */ 294 295 #ifdef CONFIG_KASAN_SW_TAGS 296 u8 random_tag(void); 297 #elif defined(CONFIG_KASAN_HW_TAGS) 298 static inline u8 random_tag(void) { return hw_get_random_tag(); } 299 #else 300 static inline u8 random_tag(void) { return 0; } 301 #endif 302 303 #ifdef CONFIG_KASAN_HW_TAGS 304 305 static inline void poison_range(const void *address, size_t size, u8 value) 306 { 307 hw_set_mem_tag_range(kasan_reset_tag(address), 308 round_up(size, KASAN_GRANULE_SIZE), value); 309 } 310 311 static inline void unpoison_range(const void *address, size_t size) 312 { 313 hw_set_mem_tag_range(kasan_reset_tag(address), 314 round_up(size, KASAN_GRANULE_SIZE), get_tag(address)); 315 } 316 317 static inline bool check_invalid_free(void *addr) 318 { 319 u8 ptr_tag = get_tag(addr); 320 u8 mem_tag = hw_get_mem_tag(addr); 321 322 return (mem_tag == KASAN_TAG_INVALID) || 323 (ptr_tag != KASAN_TAG_KERNEL && ptr_tag != mem_tag); 324 } 325 326 #else /* CONFIG_KASAN_HW_TAGS */ 327 328 void poison_range(const void *address, size_t size, u8 value); 329 void unpoison_range(const void *address, size_t size); 330 bool check_invalid_free(void *addr); 331 332 #endif /* CONFIG_KASAN_HW_TAGS */ 333 334 /* 335 * Exported functions for interfaces called from assembly or from generated 336 * code. Declarations here to avoid warning about missing declarations. 337 */ 338 asmlinkage void kasan_unpoison_task_stack_below(const void *watermark); 339 void __asan_register_globals(struct kasan_global *globals, size_t size); 340 void __asan_unregister_globals(struct kasan_global *globals, size_t size); 341 void __asan_handle_no_return(void); 342 void __asan_alloca_poison(unsigned long addr, size_t size); 343 void __asan_allocas_unpoison(const void *stack_top, const void *stack_bottom); 344 345 void __asan_load1(unsigned long addr); 346 void __asan_store1(unsigned long addr); 347 void __asan_load2(unsigned long addr); 348 void __asan_store2(unsigned long addr); 349 void __asan_load4(unsigned long addr); 350 void __asan_store4(unsigned long addr); 351 void __asan_load8(unsigned long addr); 352 void __asan_store8(unsigned long addr); 353 void __asan_load16(unsigned long addr); 354 void __asan_store16(unsigned long addr); 355 void __asan_loadN(unsigned long addr, size_t size); 356 void __asan_storeN(unsigned long addr, size_t size); 357 358 void __asan_load1_noabort(unsigned long addr); 359 void __asan_store1_noabort(unsigned long addr); 360 void __asan_load2_noabort(unsigned long addr); 361 void __asan_store2_noabort(unsigned long addr); 362 void __asan_load4_noabort(unsigned long addr); 363 void __asan_store4_noabort(unsigned long addr); 364 void __asan_load8_noabort(unsigned long addr); 365 void __asan_store8_noabort(unsigned long addr); 366 void __asan_load16_noabort(unsigned long addr); 367 void __asan_store16_noabort(unsigned long addr); 368 void __asan_loadN_noabort(unsigned long addr, size_t size); 369 void __asan_storeN_noabort(unsigned long addr, size_t size); 370 371 void __asan_report_load1_noabort(unsigned long addr); 372 void __asan_report_store1_noabort(unsigned long addr); 373 void __asan_report_load2_noabort(unsigned long addr); 374 void __asan_report_store2_noabort(unsigned long addr); 375 void __asan_report_load4_noabort(unsigned long addr); 376 void __asan_report_store4_noabort(unsigned long addr); 377 void __asan_report_load8_noabort(unsigned long addr); 378 void __asan_report_store8_noabort(unsigned long addr); 379 void __asan_report_load16_noabort(unsigned long addr); 380 void __asan_report_store16_noabort(unsigned long addr); 381 void __asan_report_load_n_noabort(unsigned long addr, size_t size); 382 void __asan_report_store_n_noabort(unsigned long addr, size_t size); 383 384 void __asan_set_shadow_00(const void *addr, size_t size); 385 void __asan_set_shadow_f1(const void *addr, size_t size); 386 void __asan_set_shadow_f2(const void *addr, size_t size); 387 void __asan_set_shadow_f3(const void *addr, size_t size); 388 void __asan_set_shadow_f5(const void *addr, size_t size); 389 void __asan_set_shadow_f8(const void *addr, size_t size); 390 391 void __hwasan_load1_noabort(unsigned long addr); 392 void __hwasan_store1_noabort(unsigned long addr); 393 void __hwasan_load2_noabort(unsigned long addr); 394 void __hwasan_store2_noabort(unsigned long addr); 395 void __hwasan_load4_noabort(unsigned long addr); 396 void __hwasan_store4_noabort(unsigned long addr); 397 void __hwasan_load8_noabort(unsigned long addr); 398 void __hwasan_store8_noabort(unsigned long addr); 399 void __hwasan_load16_noabort(unsigned long addr); 400 void __hwasan_store16_noabort(unsigned long addr); 401 void __hwasan_loadN_noabort(unsigned long addr, size_t size); 402 void __hwasan_storeN_noabort(unsigned long addr, size_t size); 403 404 void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size); 405 406 #endif 407