xref: /linux/arch/riscv/kernel/traps.c (revision 5ba15d419fab848a3813eb56bbcad00e291fbc49) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Regents of the University of California
4  */
5 
6 #include <linux/cpu.h>
7 #include <linux/kernel.h>
8 #include <linux/init.h>
9 #include <linux/irqflags.h>
10 #include <linux/randomize_kstack.h>
11 #include <linux/sched.h>
12 #include <linux/sched/debug.h>
13 #include <linux/sched/signal.h>
14 #include <linux/signal.h>
15 #include <linux/kdebug.h>
16 #include <linux/uaccess.h>
17 #include <linux/kprobes.h>
18 #include <linux/uprobes.h>
19 #include <asm/uprobes.h>
20 #include <linux/mm.h>
21 #include <linux/module.h>
22 #include <linux/irq.h>
23 #include <linux/kexec.h>
24 #include <linux/entry-common.h>
25 
26 #include <asm/asm-prototypes.h>
27 #include <asm/bug.h>
28 #include <asm/cfi.h>
29 #include <asm/csr.h>
30 #include <asm/processor.h>
31 #include <asm/ptrace.h>
32 #include <asm/syscall.h>
33 #include <asm/thread_info.h>
34 #include <asm/vector.h>
35 #include <asm/irq_stack.h>
36 
37 int show_unhandled_signals = 1;
38 
39 static DEFINE_RAW_SPINLOCK(die_lock);
40 
41 static int copy_code(struct pt_regs *regs, u16 *val, const u16 *insns)
42 {
43 	const void __user *uaddr = (__force const void __user *)insns;
44 
45 	if (!user_mode(regs))
46 		return get_kernel_nofault(*val, insns);
47 
48 	/* The user space code from other tasks cannot be accessed. */
49 	if (regs != task_pt_regs(current))
50 		return -EPERM;
51 
52 	return copy_from_user_nofault(val, uaddr, sizeof(*val));
53 }
54 
55 static void dump_instr(const char *loglvl, struct pt_regs *regs)
56 {
57 	char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
58 	const u16 *insns = (u16 *)instruction_pointer(regs);
59 	long bad;
60 	u16 val;
61 	int i;
62 
63 	for (i = -10; i < 2; i++) {
64 		bad = copy_code(regs, &val, &insns[i]);
65 		if (!bad) {
66 			p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
67 		} else {
68 			printk("%sCode: Unable to access instruction at 0x%px.\n",
69 			       loglvl, &insns[i]);
70 			return;
71 		}
72 	}
73 	printk("%sCode: %s\n", loglvl, str);
74 }
75 
76 void die(struct pt_regs *regs, const char *str)
77 {
78 	static int die_counter;
79 	int ret;
80 	long cause;
81 	unsigned long flags;
82 
83 	oops_enter();
84 
85 	raw_spin_lock_irqsave(&die_lock, flags);
86 	console_verbose();
87 	bust_spinlocks(1);
88 
89 	pr_emerg("%s [#%d]\n", str, ++die_counter);
90 	print_modules();
91 	if (regs) {
92 		show_regs(regs);
93 		dump_instr(KERN_EMERG, regs);
94 	}
95 
96 	cause = regs ? regs->cause : -1;
97 	ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
98 
99 	if (kexec_should_crash(current))
100 		crash_kexec(regs);
101 
102 	bust_spinlocks(0);
103 	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
104 	raw_spin_unlock_irqrestore(&die_lock, flags);
105 	oops_exit();
106 
107 	if (in_interrupt())
108 		panic("Fatal exception in interrupt");
109 	if (panic_on_oops)
110 		panic("Fatal exception");
111 	if (ret != NOTIFY_STOP)
112 		make_task_dead(SIGSEGV);
113 }
114 
115 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
116 {
117 	struct task_struct *tsk = current;
118 
119 	if (show_unhandled_signals && unhandled_signal(tsk, signo)
120 	    && printk_ratelimit()) {
121 		pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
122 			tsk->comm, task_pid_nr(tsk), signo, code, addr);
123 		print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
124 		pr_cont("\n");
125 		__show_regs(regs);
126 		dump_instr(KERN_INFO, regs);
127 	}
128 
129 	force_sig_fault(signo, code, (void __user *)addr);
130 }
131 
132 static void do_trap_error(struct pt_regs *regs, int signo, int code,
133 	unsigned long addr, const char *str)
134 {
135 	current->thread.bad_cause = regs->cause;
136 
137 	if (user_mode(regs)) {
138 		do_trap(regs, signo, code, addr);
139 	} else {
140 		if (!fixup_exception(regs))
141 			die(regs, str);
142 	}
143 }
144 
145 #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
146 #define __trap_section __noinstr_section(".xip.traps")
147 #else
148 #define __trap_section noinstr
149 #endif
150 #define DO_ERROR_INFO(name, signo, code, str)					\
151 asmlinkage __visible __trap_section void name(struct pt_regs *regs)		\
152 {										\
153 	if (user_mode(regs)) {							\
154 		irqentry_enter_from_user_mode(regs);				\
155 		local_irq_enable();						\
156 		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
157 		local_irq_disable();						\
158 		irqentry_exit_to_user_mode(regs);				\
159 	} else {								\
160 		irqentry_state_t state = irqentry_nmi_enter(regs);		\
161 		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
162 		irqentry_nmi_exit(regs, state);					\
163 	}									\
164 }
165 
166 DO_ERROR_INFO(do_trap_unknown,
167 	SIGILL, ILL_ILLTRP, "unknown exception");
168 DO_ERROR_INFO(do_trap_hardware_error,
169 	SIGBUS, BUS_MCEERR_AR, "hardware error");
170 DO_ERROR_INFO(do_trap_insn_misaligned,
171 	SIGBUS, BUS_ADRALN, "instruction address misaligned");
172 DO_ERROR_INFO(do_trap_insn_fault,
173 	SIGSEGV, SEGV_ACCERR, "instruction access fault");
174 
175 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
176 {
177 	bool handled;
178 
179 	if (user_mode(regs)) {
180 		irqentry_enter_from_user_mode(regs);
181 		local_irq_enable();
182 
183 		handled = riscv_v_first_use_handler(regs);
184 		if (!handled)
185 			do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
186 				      "Oops - illegal instruction");
187 
188 		local_irq_disable();
189 		irqentry_exit_to_user_mode(regs);
190 	} else {
191 		irqentry_state_t state = irqentry_nmi_enter(regs);
192 
193 		do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
194 			      "Oops - illegal instruction");
195 
196 		irqentry_nmi_exit(regs, state);
197 	}
198 }
199 
200 DO_ERROR_INFO(do_trap_load_fault,
201 	SIGSEGV, SEGV_ACCERR, "load access fault");
202 
203 enum misaligned_access_type {
204 	MISALIGNED_STORE,
205 	MISALIGNED_LOAD,
206 };
207 static const struct {
208 	const char *type_str;
209 	int (*handler)(struct pt_regs *regs);
210 } misaligned_handler[] = {
211 	[MISALIGNED_STORE] = {
212 		.type_str = "Oops - store (or AMO) address misaligned",
213 		.handler = handle_misaligned_store,
214 	},
215 	[MISALIGNED_LOAD] = {
216 		.type_str = "Oops - load address misaligned",
217 		.handler = handle_misaligned_load,
218 	},
219 };
220 
221 static void do_trap_misaligned(struct pt_regs *regs, enum misaligned_access_type type)
222 {
223 	irqentry_state_t state;
224 
225 	if (user_mode(regs)) {
226 		irqentry_enter_from_user_mode(regs);
227 		local_irq_enable();
228 	} else {
229 		state = irqentry_nmi_enter(regs);
230 	}
231 
232 	if (misaligned_handler[type].handler(regs))
233 		do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
234 			      misaligned_handler[type].type_str);
235 
236 	if (user_mode(regs)) {
237 		local_irq_disable();
238 		irqentry_exit_to_user_mode(regs);
239 	} else {
240 		irqentry_nmi_exit(regs, state);
241 	}
242 }
243 
244 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
245 {
246 	do_trap_misaligned(regs, MISALIGNED_LOAD);
247 }
248 
249 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
250 {
251 	do_trap_misaligned(regs, MISALIGNED_STORE);
252 }
253 
254 DO_ERROR_INFO(do_trap_store_fault,
255 	SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
256 DO_ERROR_INFO(do_trap_ecall_s,
257 	SIGILL, ILL_ILLTRP, "environment call from S-mode");
258 DO_ERROR_INFO(do_trap_ecall_m,
259 	SIGILL, ILL_ILLTRP, "environment call from M-mode");
260 
261 static inline unsigned long get_break_insn_length(unsigned long pc)
262 {
263 	bug_insn_t insn;
264 
265 	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
266 		return 0;
267 
268 	return GET_INSN_LENGTH(insn);
269 }
270 
271 static bool probe_single_step_handler(struct pt_regs *regs)
272 {
273 	bool user = user_mode(regs);
274 
275 	return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs);
276 }
277 
278 static bool probe_breakpoint_handler(struct pt_regs *regs)
279 {
280 	bool user = user_mode(regs);
281 
282 	return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs);
283 }
284 
285 void handle_break(struct pt_regs *regs)
286 {
287 	if (probe_single_step_handler(regs))
288 		return;
289 
290 	if (probe_breakpoint_handler(regs))
291 		return;
292 
293 	current->thread.bad_cause = regs->cause;
294 
295 	if (user_mode(regs))
296 		force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
297 #ifdef CONFIG_KGDB
298 	else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
299 								== NOTIFY_STOP)
300 		return;
301 #endif
302 	else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN ||
303 		 handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
304 		regs->epc += get_break_insn_length(regs->epc);
305 	else
306 		die(regs, "Kernel BUG");
307 }
308 
309 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
310 {
311 	if (user_mode(regs)) {
312 		irqentry_enter_from_user_mode(regs);
313 		local_irq_enable();
314 
315 		handle_break(regs);
316 
317 		local_irq_disable();
318 		irqentry_exit_to_user_mode(regs);
319 	} else {
320 		irqentry_state_t state = irqentry_nmi_enter(regs);
321 
322 		handle_break(regs);
323 
324 		irqentry_nmi_exit(regs, state);
325 	}
326 }
327 
328 asmlinkage __visible __trap_section  __no_stack_protector
329 void do_trap_ecall_u(struct pt_regs *regs)
330 {
331 	if (user_mode(regs)) {
332 		long syscall = regs->a7;
333 
334 		regs->epc += 4;
335 		regs->orig_a0 = regs->a0;
336 		regs->a0 = -ENOSYS;
337 
338 		riscv_v_vstate_discard(regs);
339 
340 		syscall = syscall_enter_from_user_mode(regs, syscall);
341 
342 		add_random_kstack_offset();
343 
344 		if (syscall >= 0 && syscall < NR_syscalls) {
345 			syscall = array_index_nospec(syscall, NR_syscalls);
346 			syscall_handler(regs, syscall);
347 		}
348 
349 		/*
350 		 * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(),
351 		 * so the maximum stack offset is 1k bytes (10 bits).
352 		 *
353 		 * The actual entropy will be further reduced by the compiler when
354 		 * applying stack alignment constraints: 16-byte (i.e. 4-bit) aligned
355 		 * for RV32I or RV64I.
356 		 *
357 		 * The resulting 6 bits of entropy is seen in SP[9:4].
358 		 */
359 		choose_random_kstack_offset(get_random_u16());
360 
361 		syscall_exit_to_user_mode(regs);
362 	} else {
363 		irqentry_state_t state = irqentry_nmi_enter(regs);
364 
365 		do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
366 			"Oops - environment call from U-mode");
367 
368 		irqentry_nmi_exit(regs, state);
369 	}
370 
371 }
372 
373 #define CFI_TVAL_FCFI_CODE	2
374 #define CFI_TVAL_BCFI_CODE	3
375 /* handle cfi violations */
376 bool handle_user_cfi_violation(struct pt_regs *regs)
377 {
378 	unsigned long tval = csr_read(CSR_TVAL);
379 	bool is_fcfi = (tval == CFI_TVAL_FCFI_CODE && cpu_supports_indirect_br_lp_instr());
380 	bool is_bcfi = (tval == CFI_TVAL_BCFI_CODE && cpu_supports_shadow_stack());
381 
382 	/*
383 	 * Handle uprobe event first. The probe point can be a valid target
384 	 * of indirect jumps or calls, in this case, forward cfi violation
385 	 * will be triggered instead of breakpoint exception. Clear ELP flag
386 	 * on sstatus image as well to avoid recurring fault.
387 	 */
388 	if (is_fcfi && probe_breakpoint_handler(regs)) {
389 		regs->status &= ~SR_ELP;
390 		return true;
391 	}
392 
393 	if (is_fcfi || is_bcfi) {
394 		do_trap_error(regs, SIGSEGV, SEGV_CPERR, regs->epc,
395 			      "Oops - control flow violation");
396 		return true;
397 	}
398 
399 	return false;
400 }
401 
402 /*
403  * software check exception is defined with risc-v cfi spec. Software check
404  * exception is raised when:
405  * a) An indirect branch doesn't land on 4 byte aligned PC or `lpad`
406  *    instruction or `label` value programmed in `lpad` instr doesn't
407  *    match with value setup in `x7`. reported code in `xtval` is 2.
408  * b) `sspopchk` instruction finds a mismatch between top of shadow stack (ssp)
409  *    and x1/x5. reported code in `xtval` is 3.
410  */
411 asmlinkage __visible __trap_section void do_trap_software_check(struct pt_regs *regs)
412 {
413 	if (user_mode(regs)) {
414 		irqentry_enter_from_user_mode(regs);
415 
416 		/* not a cfi violation, then merge into flow of unknown trap handler */
417 		if (!handle_user_cfi_violation(regs))
418 			do_trap_unknown(regs);
419 
420 		irqentry_exit_to_user_mode(regs);
421 	} else {
422 		/* sw check exception coming from kernel is a bug in kernel */
423 		die(regs, "Kernel BUG");
424 	}
425 }
426 
427 #ifdef CONFIG_MMU
428 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
429 {
430 	irqentry_state_t state = irqentry_enter(regs);
431 
432 	handle_page_fault(regs);
433 
434 	local_irq_disable();
435 
436 	irqentry_exit(regs, state);
437 }
438 #endif
439 
440 static void noinstr handle_riscv_irq(struct pt_regs *regs)
441 {
442 	struct pt_regs *old_regs;
443 
444 	irq_enter_rcu();
445 	old_regs = set_irq_regs(regs);
446 	handle_arch_irq(regs);
447 	set_irq_regs(old_regs);
448 	irq_exit_rcu();
449 }
450 
451 asmlinkage void noinstr do_irq(struct pt_regs *regs)
452 {
453 	irqentry_state_t state = irqentry_enter(regs);
454 
455 	if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack())
456 		call_on_irq_stack(regs, handle_riscv_irq);
457 	else
458 		handle_riscv_irq(regs);
459 
460 	irqentry_exit(regs, state);
461 }
462 
463 #ifdef CONFIG_GENERIC_BUG
464 int is_valid_bugaddr(unsigned long pc)
465 {
466 	bug_insn_t insn;
467 
468 	if (pc < VMALLOC_START)
469 		return 0;
470 	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
471 		return 0;
472 	if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
473 		return (insn == __BUG_INSN_32);
474 	else
475 		return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
476 }
477 #endif /* CONFIG_GENERIC_BUG */
478 
479 #ifdef CONFIG_VMAP_STACK
480 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
481 		overflow_stack)__aligned(16);
482 
483 asmlinkage void handle_bad_stack(struct pt_regs *regs)
484 {
485 	unsigned long tsk_stk = (unsigned long)current->stack;
486 	unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
487 
488 	console_verbose();
489 
490 	pr_emerg("Insufficient stack space to handle exception!\n");
491 	pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
492 			tsk_stk, tsk_stk + THREAD_SIZE);
493 	pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
494 			ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
495 
496 	__show_regs(regs);
497 	panic("Kernel stack overflow");
498 
499 	for (;;)
500 		wait_for_interrupt();
501 }
502 #endif
503