1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Kernel Electric-Fence (KFENCE). For more info please see 4 * Documentation/dev-tools/kfence.rst. 5 * 6 * Copyright (C) 2020, Google LLC. 7 */ 8 9 #ifndef MM_KFENCE_KFENCE_H 10 #define MM_KFENCE_KFENCE_H 11 12 #include <linux/mm.h> 13 #include <linux/slab.h> 14 #include <linux/spinlock.h> 15 #include <linux/types.h> 16 17 #include "../slab.h" /* for struct kmem_cache */ 18 19 extern bool kfence_enabled; 20 21 /* 22 * Get the canary byte pattern for @addr. Use a pattern that varies based on the 23 * lower 3 bits of the address, to detect memory corruptions with higher 24 * probability, where similar constants are used. 25 */ 26 #define KFENCE_CANARY_PATTERN_U8(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7)) 27 28 /* 29 * Define a continuous 8-byte canary starting from a multiple of 8. The canary 30 * of each byte is only related to the lowest three bits of its address, so the 31 * canary of every 8 bytes is the same. 64-bit memory can be filled and checked 32 * at a time instead of byte by byte to improve performance. 33 */ 34 #define KFENCE_CANARY_PATTERN_U64 ((u64)0xaaaaaaaaaaaaaaaa ^ (u64)(le64_to_cpu(0x0706050403020100))) 35 36 /* Maximum stack depth for reports. */ 37 #define KFENCE_STACK_DEPTH 64 38 39 extern raw_spinlock_t kfence_freelist_lock; 40 41 /* KFENCE object states. */ 42 enum kfence_object_state { 43 KFENCE_OBJECT_UNUSED, /* Object is unused. */ 44 KFENCE_OBJECT_ALLOCATED, /* Object is currently allocated. */ 45 KFENCE_OBJECT_RCU_FREEING, /* Object was allocated, and then being freed by rcu. */ 46 KFENCE_OBJECT_FREED, /* Object was allocated, and then freed. */ 47 }; 48 49 /* Alloc/free tracking information. */ 50 struct kfence_track { 51 pid_t pid; 52 int cpu; 53 u64 ts_nsec; 54 int num_stack_entries; 55 unsigned long stack_entries[KFENCE_STACK_DEPTH]; 56 }; 57 58 /* KFENCE metadata per guarded allocation. */ 59 struct kfence_metadata { 60 struct list_head list __guarded_by(&kfence_freelist_lock); /* Freelist node. */ 61 struct rcu_head rcu_head; /* For delayed freeing. */ 62 63 /* 64 * Lock protecting below data; to ensure consistency of the below data, 65 * since the following may execute concurrently: __kfence_alloc(), 66 * __kfence_free(), kfence_handle_page_fault(). However, note that we 67 * cannot grab the same metadata off the freelist twice, and multiple 68 * __kfence_alloc() cannot run concurrently on the same metadata. 69 */ 70 raw_spinlock_t lock; 71 72 /* The current state of the object; see above. */ 73 enum kfence_object_state state; 74 75 /* 76 * Allocated object address; cannot be calculated from size, because of 77 * alignment requirements. 78 * 79 * Invariant: ALIGN_DOWN(addr, PAGE_SIZE) is constant. 80 */ 81 unsigned long addr; 82 83 /* 84 * The size of the original allocation. 85 */ 86 size_t size; 87 88 /* 89 * The kmem_cache cache of the last allocation; NULL if never allocated 90 * or the cache has already been destroyed. 91 */ 92 struct kmem_cache *cache; 93 94 /* 95 * In case of an invalid access, the page that was unprotected; we 96 * optimistically only store one address. 97 */ 98 unsigned long unprotected_page __guarded_by(&lock); 99 100 /* Allocation and free stack information. */ 101 struct kfence_track alloc_track __guarded_by(&lock); 102 struct kfence_track free_track __guarded_by(&lock); 103 /* For updating alloc_covered on frees. */ 104 u32 alloc_stack_hash __guarded_by(&lock); 105 #ifdef CONFIG_MEMCG 106 struct slabobj_ext obj_exts; 107 #endif 108 }; 109 110 #define KFENCE_METADATA_SIZE PAGE_ALIGN(sizeof(struct kfence_metadata) * \ 111 CONFIG_KFENCE_NUM_OBJECTS) 112 113 extern struct kfence_metadata *kfence_metadata; 114 115 static inline struct kfence_metadata *addr_to_metadata(unsigned long addr) 116 { 117 long index; 118 119 /* The checks do not affect performance; only called from slow-paths. */ 120 121 if (!is_kfence_address((void *)addr)) 122 return NULL; 123 124 /* 125 * May be an invalid index if called with an address at the edge of 126 * __kfence_pool, in which case we would report an "invalid access" 127 * error. 128 */ 129 index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1; 130 if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS) 131 return NULL; 132 133 return &kfence_metadata[index]; 134 } 135 136 /* KFENCE error types for report generation. */ 137 enum kfence_error_type { 138 KFENCE_ERROR_OOB, /* Detected a out-of-bounds access. */ 139 KFENCE_ERROR_UAF, /* Detected a use-after-free access. */ 140 KFENCE_ERROR_CORRUPTION, /* Detected a memory corruption on free. */ 141 KFENCE_ERROR_INVALID, /* Invalid access of unknown type. */ 142 KFENCE_ERROR_INVALID_FREE, /* Invalid free. */ 143 }; 144 145 enum kfence_fault { 146 KFENCE_FAULT_NONE, 147 KFENCE_FAULT_REPORT, 148 KFENCE_FAULT_OOPS, 149 KFENCE_FAULT_PANIC, 150 }; 151 152 enum kfence_fault 153 kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs, 154 const struct kfence_metadata *meta, enum kfence_error_type type); 155 156 void kfence_handle_fault(enum kfence_fault fault); 157 158 void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) __must_hold(&meta->lock); 159 160 #endif /* MM_KFENCE_KFENCE_H */ 161