1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Kernel Electric-Fence (KFENCE). For more info please see
4 * Documentation/dev-tools/kfence.rst.
5 *
6 * Copyright (C) 2020, Google LLC.
7 */
8
9 #ifndef MM_KFENCE_KFENCE_H
10 #define MM_KFENCE_KFENCE_H
11
12 #include <linux/mm.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <linux/types.h>
16
17 #include "../slab.h" /* for struct kmem_cache */
18
19 extern bool kfence_enabled;
20
21 /*
22 * Get the canary byte pattern for @addr. Use a pattern that varies based on the
23 * lower 3 bits of the address, to detect memory corruptions with higher
24 * probability, where similar constants are used.
25 */
26 #define KFENCE_CANARY_PATTERN_U8(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7))
27
28 /*
29 * Define a continuous 8-byte canary starting from a multiple of 8. The canary
30 * of each byte is only related to the lowest three bits of its address, so the
31 * canary of every 8 bytes is the same. 64-bit memory can be filled and checked
32 * at a time instead of byte by byte to improve performance.
33 */
34 #define KFENCE_CANARY_PATTERN_U64 ((u64)0xaaaaaaaaaaaaaaaa ^ (u64)(le64_to_cpu(0x0706050403020100)))
35
36 /* Maximum stack depth for reports. */
37 #define KFENCE_STACK_DEPTH 64
38
39 extern raw_spinlock_t kfence_freelist_lock;
40
41 /* KFENCE object states. */
42 enum kfence_object_state {
43 KFENCE_OBJECT_UNUSED, /* Object is unused. */
44 KFENCE_OBJECT_ALLOCATED, /* Object is currently allocated. */
45 KFENCE_OBJECT_RCU_FREEING, /* Object was allocated, and then being freed by rcu. */
46 KFENCE_OBJECT_FREED, /* Object was allocated, and then freed. */
47 };
48
49 /* Alloc/free tracking information. */
50 struct kfence_track {
51 pid_t pid;
52 int cpu;
53 u64 ts_nsec;
54 int num_stack_entries;
55 unsigned long stack_entries[KFENCE_STACK_DEPTH];
56 };
57
58 /* KFENCE metadata per guarded allocation. */
59 struct kfence_metadata {
60 struct list_head list __guarded_by(&kfence_freelist_lock); /* Freelist node. */
61 struct rcu_head rcu_head; /* For delayed freeing. */
62
63 /*
64 * Lock protecting below data; to ensure consistency of the below data,
65 * since the following may execute concurrently: __kfence_alloc(),
66 * __kfence_free(), kfence_handle_page_fault(). However, note that we
67 * cannot grab the same metadata off the freelist twice, and multiple
68 * __kfence_alloc() cannot run concurrently on the same metadata.
69 */
70 raw_spinlock_t lock;
71
72 /* The current state of the object; see above. */
73 enum kfence_object_state state;
74
75 /*
76 * Allocated object address; cannot be calculated from size, because of
77 * alignment requirements.
78 *
79 * Invariant: ALIGN_DOWN(addr, PAGE_SIZE) is constant.
80 */
81 unsigned long addr;
82
83 /*
84 * The size of the original allocation.
85 */
86 size_t size;
87
88 /*
89 * The kmem_cache cache of the last allocation; NULL if never allocated
90 * or the cache has already been destroyed.
91 */
92 struct kmem_cache *cache;
93
94 /*
95 * In case of an invalid access, the page that was unprotected; we
96 * optimistically only store one address.
97 */
98 unsigned long unprotected_page __guarded_by(&lock);
99
100 /* Allocation and free stack information. */
101 struct kfence_track alloc_track __guarded_by(&lock);
102 struct kfence_track free_track __guarded_by(&lock);
103 /* For updating alloc_covered on frees. */
104 u32 alloc_stack_hash __guarded_by(&lock);
105 };
106
107 #define KFENCE_METADATA_SIZE PAGE_ALIGN(sizeof(struct kfence_metadata) * \
108 CONFIG_KFENCE_NUM_OBJECTS)
109
110 extern struct kfence_metadata *kfence_metadata;
111
addr_to_metadata(unsigned long addr)112 static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
113 {
114 long index;
115
116 /* The checks do not affect performance; only called from slow-paths. */
117
118 if (!is_kfence_address((void *)addr))
119 return NULL;
120
121 /*
122 * May be an invalid index if called with an address at the edge of
123 * __kfence_pool, in which case we would report an "invalid access"
124 * error.
125 */
126 index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
127 if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS)
128 return NULL;
129
130 return &kfence_metadata[index];
131 }
132
133 /* KFENCE error types for report generation. */
134 enum kfence_error_type {
135 KFENCE_ERROR_OOB, /* Detected a out-of-bounds access. */
136 KFENCE_ERROR_UAF, /* Detected a use-after-free access. */
137 KFENCE_ERROR_CORRUPTION, /* Detected a memory corruption on free. */
138 KFENCE_ERROR_INVALID, /* Invalid access of unknown type. */
139 KFENCE_ERROR_INVALID_FREE, /* Invalid free. */
140 };
141
142 enum kfence_fault {
143 KFENCE_FAULT_NONE,
144 KFENCE_FAULT_REPORT,
145 KFENCE_FAULT_OOPS,
146 KFENCE_FAULT_PANIC,
147 };
148
149 enum kfence_fault
150 kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
151 const struct kfence_metadata *meta, enum kfence_error_type type);
152
153 void kfence_handle_fault(enum kfence_fault fault);
154
155 void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) __must_hold(&meta->lock);
156
157 #endif /* MM_KFENCE_KFENCE_H */
158