1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KFENCE reporting. 4 * 5 * Copyright (C) 2020, Google LLC. 6 */ 7 8 #include <linux/stdarg.h> 9 10 #include <linux/kernel.h> 11 #include <linux/lockdep.h> 12 #include <linux/math.h> 13 #include <linux/printk.h> 14 #include <linux/sched/debug.h> 15 #include <linux/seq_file.h> 16 #include <linux/sprintf.h> 17 #include <linux/stacktrace.h> 18 #include <linux/string.h> 19 #include <linux/string_choices.h> 20 #include <linux/sched/clock.h> 21 #include <trace/events/error_report.h> 22 23 #include <asm/kfence.h> 24 25 #include "kfence.h" 26 27 /* May be overridden by <asm/kfence.h>. */ 28 #ifndef ARCH_FUNC_PREFIX 29 #define ARCH_FUNC_PREFIX "" 30 #endif 31 32 /* Helper function to either print to a seq_file or to console. */ 33 __printf(2, 3) 34 static void seq_con_printf(struct seq_file *seq, const char *fmt, ...) 35 { 36 va_list args; 37 38 va_start(args, fmt); 39 if (seq) 40 seq_vprintf(seq, fmt, args); 41 else 42 vprintk(fmt, args); 43 va_end(args); 44 } 45 46 /* 47 * Get the number of stack entries to skip to get out of MM internals. @type is 48 * optional, and if set to NULL, assumes an allocation or free stack. 49 */ 50 static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries, 51 const enum kfence_error_type *type) 52 { 53 char buf[64]; 54 int skipnr, fallback = 0; 55 56 if (type) { 57 /* Depending on error type, find different stack entries. */ 58 switch (*type) { 59 case KFENCE_ERROR_UAF: 60 case KFENCE_ERROR_OOB: 61 case KFENCE_ERROR_INVALID: 62 /* 63 * kfence_handle_page_fault() may be called with pt_regs 64 * set to NULL; in that case we'll simply show the full 65 * stack trace. 66 */ 67 return 0; 68 case KFENCE_ERROR_CORRUPTION: 69 case KFENCE_ERROR_INVALID_FREE: 70 break; 71 } 72 } 73 74 for (skipnr = 0; skipnr < num_entries; skipnr++) { 75 int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]); 76 77 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") || 78 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") || 79 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") || 80 !strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) { 81 /* 82 * In case of tail calls from any of the below to any of 83 * the above, optimized by the compiler such that the 84 * stack trace would omit the initial entry point below. 85 */ 86 fallback = skipnr + 1; 87 } 88 89 /* 90 * The below list should only include the initial entry points 91 * into the slab allocators. Includes the *_bulk() variants by 92 * checking prefixes. 93 */ 94 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") || 95 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") || 96 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") || 97 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc")) 98 goto found; 99 } 100 if (fallback < num_entries) 101 return fallback; 102 found: 103 skipnr++; 104 return skipnr < num_entries ? skipnr : 0; 105 } 106 107 static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta, 108 bool show_alloc) 109 __must_hold(&meta->lock) 110 { 111 const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track; 112 u64 ts_sec = track->ts_nsec; 113 unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC); 114 u64 interval_nsec = local_clock() - track->ts_nsec; 115 unsigned long rem_interval_nsec = do_div(interval_nsec, NSEC_PER_SEC); 116 117 /* Timestamp matches printk timestamp format. */ 118 seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus (%lu.%06lus ago):\n", 119 show_alloc ? "allocated" : meta->state == KFENCE_OBJECT_RCU_FREEING ? 120 "rcu freeing" : "freed", track->pid, 121 track->cpu, (unsigned long)ts_sec, rem_nsec / 1000, 122 (unsigned long)interval_nsec, rem_interval_nsec / 1000); 123 124 if (track->num_stack_entries) { 125 /* Skip allocation/free internals stack. */ 126 int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL); 127 128 /* stack_trace_seq_print() does not exist; open code our own. */ 129 for (; i < track->num_stack_entries; i++) 130 seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]); 131 } else { 132 seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation"); 133 } 134 } 135 136 void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) 137 { 138 const int size = abs(meta->size); 139 const unsigned long start = meta->addr; 140 const struct kmem_cache *const cache = meta->cache; 141 142 lockdep_assert_held(&meta->lock); 143 144 if (meta->state == KFENCE_OBJECT_UNUSED) { 145 seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata); 146 return; 147 } 148 149 seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n", 150 meta - kfence_metadata, (void *)start, (void *)(start + size - 1), 151 size, (cache && cache->name) ? cache->name : "<destroyed>"); 152 153 kfence_print_stack(seq, meta, true); 154 155 if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) { 156 seq_con_printf(seq, "\n"); 157 kfence_print_stack(seq, meta, false); 158 } 159 } 160 161 /* 162 * Show bytes at @addr that are different from the expected canary values, up to 163 * @max_bytes. 164 */ 165 static void print_diff_canary(unsigned long address, size_t bytes_to_show, 166 const struct kfence_metadata *meta) 167 { 168 const unsigned long show_until_addr = address + bytes_to_show; 169 const u8 *cur, *end; 170 171 /* Do not show contents of object nor read into following guard page. */ 172 end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr) 173 : min(show_until_addr, PAGE_ALIGN(address))); 174 175 pr_cont("["); 176 for (cur = (const u8 *)address; cur < end; cur++) { 177 if (*cur == KFENCE_CANARY_PATTERN_U8(cur)) 178 pr_cont(" ."); 179 else if (no_hash_pointers) 180 pr_cont(" 0x%02x", *cur); 181 else /* Do not leak kernel memory in non-debug builds. */ 182 pr_cont(" !"); 183 } 184 pr_cont(" ]"); 185 } 186 187 static const char *get_access_type(bool is_write) 188 { 189 return str_write_read(is_write); 190 } 191 192 void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs, 193 const struct kfence_metadata *meta, enum kfence_error_type type) 194 { 195 unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 }; 196 const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1; 197 int num_stack_entries; 198 int skipnr = 0; 199 200 if (regs) { 201 num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0); 202 } else { 203 num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1); 204 skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type); 205 } 206 207 /* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */ 208 if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta)) 209 return; 210 211 /* 212 * Because we may generate reports in printk-unfriendly parts of the 213 * kernel, such as scheduler code, the use of printk() could deadlock. 214 * Until such time that all printing code here is safe in all parts of 215 * the kernel, accept the risk, and just get our message out (given the 216 * system might already behave unpredictably due to the memory error). 217 * As such, also disable lockdep to hide warnings, and avoid disabling 218 * lockdep for the rest of the kernel. 219 */ 220 lockdep_off(); 221 222 pr_err("==================================================================\n"); 223 /* Print report header. */ 224 switch (type) { 225 case KFENCE_ERROR_OOB: { 226 const bool left_of_object = address < meta->addr; 227 228 pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write), 229 (void *)stack_entries[skipnr]); 230 pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n", 231 get_access_type(is_write), (void *)address, 232 left_of_object ? meta->addr - address : address - meta->addr, 233 left_of_object ? "left" : "right", object_index); 234 break; 235 } 236 case KFENCE_ERROR_UAF: 237 pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write), 238 (void *)stack_entries[skipnr]); 239 pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n", 240 get_access_type(is_write), (void *)address, object_index); 241 break; 242 case KFENCE_ERROR_CORRUPTION: 243 pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]); 244 pr_err("Corrupted memory at 0x%p ", (void *)address); 245 print_diff_canary(address, 16, meta); 246 pr_cont(" (in kfence-#%td):\n", object_index); 247 break; 248 case KFENCE_ERROR_INVALID: 249 pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write), 250 (void *)stack_entries[skipnr]); 251 pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write), 252 (void *)address); 253 break; 254 case KFENCE_ERROR_INVALID_FREE: 255 pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]); 256 pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address, 257 object_index); 258 break; 259 } 260 261 /* Print stack trace and object info. */ 262 stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0); 263 264 if (meta) { 265 lockdep_assert_held(&meta->lock); 266 pr_err("\n"); 267 kfence_print_object(NULL, meta); 268 } 269 270 /* Print report footer. */ 271 pr_err("\n"); 272 if (no_hash_pointers && regs) 273 show_regs(regs); 274 else 275 dump_stack_print_info(KERN_ERR); 276 trace_error_report_end(ERROR_DETECTOR_KFENCE, address); 277 pr_err("==================================================================\n"); 278 279 lockdep_on(); 280 281 check_panic_on_warn("KFENCE"); 282 283 /* We encountered a memory safety error, taint the kernel! */ 284 add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK); 285 } 286 287 #ifdef CONFIG_PRINTK 288 static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack) 289 { 290 int i, j; 291 292 i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL); 293 for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j) 294 kp_stack[j] = (void *)track->stack_entries[i]; 295 if (j < KS_ADDRS_COUNT) 296 kp_stack[j] = NULL; 297 } 298 299 bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) 300 { 301 struct kfence_metadata *meta = addr_to_metadata((unsigned long)object); 302 unsigned long flags; 303 304 if (!meta) 305 return false; 306 307 /* 308 * If state is UNUSED at least show the pointer requested; the rest 309 * would be garbage data. 310 */ 311 kpp->kp_ptr = object; 312 313 /* Requesting info an a never-used object is almost certainly a bug. */ 314 if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED)) 315 return true; 316 317 raw_spin_lock_irqsave(&meta->lock, flags); 318 319 kpp->kp_slab = slab; 320 kpp->kp_slab_cache = meta->cache; 321 kpp->kp_objp = (void *)meta->addr; 322 kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack); 323 if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) 324 kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack); 325 /* get_stack_skipnr() ensures the first entry is outside allocator. */ 326 kpp->kp_ret = kpp->kp_stack[0]; 327 328 raw_spin_unlock_irqrestore(&meta->lock, flags); 329 330 return true; 331 } 332 #endif 333