1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KFENCE reporting. 4 * 5 * Copyright (C) 2020, Google LLC. 6 */ 7 8 #include <linux/stdarg.h> 9 10 #include <linux/bug.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/lockdep.h> 14 #include <linux/math.h> 15 #include <linux/panic.h> 16 #include <linux/printk.h> 17 #include <linux/sched/debug.h> 18 #include <linux/seq_file.h> 19 #include <linux/sprintf.h> 20 #include <linux/stacktrace.h> 21 #include <linux/string.h> 22 #include <linux/string_choices.h> 23 #include <linux/sched/clock.h> 24 #include <trace/events/error_report.h> 25 26 #include <asm/kfence.h> 27 28 #include "kfence.h" 29 30 /* May be overridden by <asm/kfence.h>. */ 31 #ifndef ARCH_FUNC_PREFIX 32 #define ARCH_FUNC_PREFIX "" 33 #endif 34 35 static enum kfence_fault kfence_fault __ro_after_init = KFENCE_FAULT_REPORT; 36 37 static int __init early_kfence_fault(char *arg) 38 { 39 if (!arg) 40 return -EINVAL; 41 42 if (!strcmp(arg, "report")) 43 kfence_fault = KFENCE_FAULT_REPORT; 44 else if (!strcmp(arg, "oops")) 45 kfence_fault = KFENCE_FAULT_OOPS; 46 else if (!strcmp(arg, "panic")) 47 kfence_fault = KFENCE_FAULT_PANIC; 48 else 49 return -EINVAL; 50 51 return 0; 52 } 53 early_param("kfence.fault", early_kfence_fault); 54 55 /* Helper function to either print to a seq_file or to console. */ 56 __printf(2, 3) 57 static void seq_con_printf(struct seq_file *seq, const char *fmt, ...) 58 { 59 va_list args; 60 61 va_start(args, fmt); 62 if (seq) 63 seq_vprintf(seq, fmt, args); 64 else 65 vprintk(fmt, args); 66 va_end(args); 67 } 68 69 /* 70 * Get the number of stack entries to skip to get out of MM internals. @type is 71 * optional, and if set to NULL, assumes an allocation or free stack. 72 */ 73 static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries, 74 const enum kfence_error_type *type) 75 { 76 char buf[64]; 77 int skipnr, fallback = 0; 78 79 if (type) { 80 /* Depending on error type, find different stack entries. */ 81 switch (*type) { 82 case KFENCE_ERROR_UAF: 83 case KFENCE_ERROR_OOB: 84 case KFENCE_ERROR_INVALID: 85 /* 86 * kfence_handle_page_fault() may be called with pt_regs 87 * set to NULL; in that case we'll simply show the full 88 * stack trace. 89 */ 90 return 0; 91 case KFENCE_ERROR_CORRUPTION: 92 case KFENCE_ERROR_INVALID_FREE: 93 break; 94 } 95 } 96 97 for (skipnr = 0; skipnr < num_entries; skipnr++) { 98 int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]); 99 100 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") || 101 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") || 102 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") || 103 !strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) { 104 /* 105 * In case of tail calls from any of the below to any of 106 * the above, optimized by the compiler such that the 107 * stack trace would omit the initial entry point below. 108 */ 109 fallback = skipnr + 1; 110 } 111 112 /* 113 * The below list should only include the initial entry points 114 * into the slab allocators. Includes the *_bulk() variants by 115 * checking prefixes. 116 */ 117 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") || 118 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") || 119 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") || 120 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc")) 121 goto found; 122 } 123 if (fallback < num_entries) 124 return fallback; 125 found: 126 skipnr++; 127 return skipnr < num_entries ? skipnr : 0; 128 } 129 130 static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta, 131 bool show_alloc) 132 __must_hold(&meta->lock) 133 { 134 const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track; 135 u64 ts_sec = track->ts_nsec; 136 unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC); 137 u64 interval_nsec = local_clock() - track->ts_nsec; 138 unsigned long rem_interval_nsec = do_div(interval_nsec, NSEC_PER_SEC); 139 140 /* Timestamp matches printk timestamp format. */ 141 seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus (%lu.%06lus ago):\n", 142 show_alloc ? "allocated" : meta->state == KFENCE_OBJECT_RCU_FREEING ? 143 "rcu freeing" : "freed", track->pid, 144 track->cpu, (unsigned long)ts_sec, rem_nsec / 1000, 145 (unsigned long)interval_nsec, rem_interval_nsec / 1000); 146 147 if (track->num_stack_entries) { 148 /* Skip allocation/free internals stack. */ 149 int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL); 150 151 /* stack_trace_seq_print() does not exist; open code our own. */ 152 for (; i < track->num_stack_entries; i++) 153 seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]); 154 } else { 155 seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation"); 156 } 157 } 158 159 void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) 160 { 161 const int size = abs(meta->size); 162 const unsigned long start = meta->addr; 163 const struct kmem_cache *const cache = meta->cache; 164 165 lockdep_assert_held(&meta->lock); 166 167 if (meta->state == KFENCE_OBJECT_UNUSED) { 168 seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata); 169 return; 170 } 171 172 seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n", 173 meta - kfence_metadata, (void *)start, (void *)(start + size - 1), 174 size, (cache && cache->name) ? cache->name : "<destroyed>"); 175 176 kfence_print_stack(seq, meta, true); 177 178 if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) { 179 seq_con_printf(seq, "\n"); 180 kfence_print_stack(seq, meta, false); 181 } 182 } 183 184 /* 185 * Show bytes at @addr that are different from the expected canary values, up to 186 * @max_bytes. 187 */ 188 static void print_diff_canary(unsigned long address, size_t bytes_to_show, 189 const struct kfence_metadata *meta) 190 { 191 const unsigned long show_until_addr = address + bytes_to_show; 192 const u8 *cur, *end; 193 194 /* Do not show contents of object nor read into following guard page. */ 195 end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr) 196 : min(show_until_addr, PAGE_ALIGN(address))); 197 198 pr_cont("["); 199 for (cur = (const u8 *)address; cur < end; cur++) { 200 if (*cur == KFENCE_CANARY_PATTERN_U8(cur)) 201 pr_cont(" ."); 202 else if (no_hash_pointers) 203 pr_cont(" 0x%02x", *cur); 204 else /* Do not leak kernel memory in non-debug builds. */ 205 pr_cont(" !"); 206 } 207 pr_cont(" ]"); 208 } 209 210 static const char *get_access_type(bool is_write) 211 { 212 return str_write_read(is_write); 213 } 214 215 enum kfence_fault 216 kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs, 217 const struct kfence_metadata *meta, enum kfence_error_type type) 218 { 219 unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 }; 220 const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1; 221 int num_stack_entries; 222 int skipnr = 0; 223 224 if (regs) { 225 num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0); 226 } else { 227 num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1); 228 skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type); 229 } 230 231 /* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */ 232 if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta)) 233 return KFENCE_FAULT_NONE; 234 235 /* 236 * Because we may generate reports in printk-unfriendly parts of the 237 * kernel, such as scheduler code, the use of printk() could deadlock. 238 * Until such time that all printing code here is safe in all parts of 239 * the kernel, accept the risk, and just get our message out (given the 240 * system might already behave unpredictably due to the memory error). 241 * As such, also disable lockdep to hide warnings, and avoid disabling 242 * lockdep for the rest of the kernel. 243 */ 244 lockdep_off(); 245 246 pr_err("==================================================================\n"); 247 /* Print report header. */ 248 switch (type) { 249 case KFENCE_ERROR_OOB: { 250 const bool left_of_object = address < meta->addr; 251 252 pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write), 253 (void *)stack_entries[skipnr]); 254 pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n", 255 get_access_type(is_write), (void *)address, 256 left_of_object ? meta->addr - address : address - meta->addr, 257 left_of_object ? "left" : "right", object_index); 258 break; 259 } 260 case KFENCE_ERROR_UAF: 261 pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write), 262 (void *)stack_entries[skipnr]); 263 pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n", 264 get_access_type(is_write), (void *)address, object_index); 265 break; 266 case KFENCE_ERROR_CORRUPTION: 267 pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]); 268 pr_err("Corrupted memory at 0x%p ", (void *)address); 269 print_diff_canary(address, 16, meta); 270 pr_cont(" (in kfence-#%td):\n", object_index); 271 break; 272 case KFENCE_ERROR_INVALID: 273 pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write), 274 (void *)stack_entries[skipnr]); 275 pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write), 276 (void *)address); 277 break; 278 case KFENCE_ERROR_INVALID_FREE: 279 pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]); 280 pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address, 281 object_index); 282 break; 283 } 284 285 /* Print stack trace and object info. */ 286 stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0); 287 288 if (meta) { 289 lockdep_assert_held(&meta->lock); 290 pr_err("\n"); 291 kfence_print_object(NULL, meta); 292 } 293 294 /* Print report footer. */ 295 pr_err("\n"); 296 if (no_hash_pointers && regs) 297 show_regs(regs); 298 else 299 dump_stack_print_info(KERN_ERR); 300 trace_error_report_end(ERROR_DETECTOR_KFENCE, address); 301 pr_err("==================================================================\n"); 302 303 lockdep_on(); 304 305 check_panic_on_warn("KFENCE"); 306 307 /* We encountered a memory safety error, taint the kernel! */ 308 add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK); 309 310 return kfence_fault; 311 } 312 313 void kfence_handle_fault(enum kfence_fault fault) 314 { 315 switch (fault) { 316 case KFENCE_FAULT_NONE: 317 case KFENCE_FAULT_REPORT: 318 break; 319 case KFENCE_FAULT_OOPS: 320 BUG(); 321 break; 322 case KFENCE_FAULT_PANIC: 323 /* Disable KFENCE to avoid recursion if check_on_panic is set. */ 324 WRITE_ONCE(kfence_enabled, false); 325 panic("kfence.fault=panic set ...\n"); 326 break; 327 } 328 } 329 330 #ifdef CONFIG_PRINTK 331 static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack) 332 { 333 int i, j; 334 335 i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL); 336 for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j) 337 kp_stack[j] = (void *)track->stack_entries[i]; 338 if (j < KS_ADDRS_COUNT) 339 kp_stack[j] = NULL; 340 } 341 342 bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) 343 { 344 struct kfence_metadata *meta = addr_to_metadata((unsigned long)object); 345 unsigned long flags; 346 347 if (!meta) 348 return false; 349 350 /* 351 * If state is UNUSED at least show the pointer requested; the rest 352 * would be garbage data. 353 */ 354 kpp->kp_ptr = object; 355 356 /* Requesting info an a never-used object is almost certainly a bug. */ 357 if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED)) 358 return true; 359 360 raw_spin_lock_irqsave(&meta->lock, flags); 361 362 kpp->kp_slab = slab; 363 kpp->kp_slab_cache = meta->cache; 364 kpp->kp_objp = (void *)meta->addr; 365 kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack); 366 if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) 367 kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack); 368 /* get_stack_skipnr() ensures the first entry is outside allocator. */ 369 kpp->kp_ret = kpp->kp_stack[0]; 370 371 raw_spin_unlock_irqrestore(&meta->lock, flags); 372 373 return true; 374 } 375 #endif 376