xref: /linux/arch/riscv/kernel/traps.c (revision 59e6295fac26b8e85c1ea859cdd89fa1e47519d7) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Regents of the University of California
4  */
5 
6 #include <linux/cpu.h>
7 #include <linux/kernel.h>
8 #include <linux/init.h>
9 #include <linux/irqflags.h>
10 #include <linux/sched.h>
11 #include <linux/sched/debug.h>
12 #include <linux/sched/signal.h>
13 #include <linux/signal.h>
14 #include <linux/kdebug.h>
15 #include <linux/uaccess.h>
16 #include <linux/kprobes.h>
17 #include <linux/uprobes.h>
18 #include <asm/uprobes.h>
19 #include <linux/mm.h>
20 #include <linux/module.h>
21 #include <linux/irq.h>
22 #include <linux/kexec.h>
23 #include <linux/entry-common.h>
24 
25 #include <asm/asm-prototypes.h>
26 #include <asm/bug.h>
27 #include <asm/cfi.h>
28 #include <asm/csr.h>
29 #include <asm/processor.h>
30 #include <asm/ptrace.h>
31 #include <asm/syscall.h>
32 #include <asm/thread_info.h>
33 #include <asm/vector.h>
34 #include <asm/irq_stack.h>
35 
36 int show_unhandled_signals = 1;
37 
38 static DEFINE_RAW_SPINLOCK(die_lock);
39 
40 static int copy_code(struct pt_regs *regs, u16 *val, const u16 *insns)
41 {
42 	const void __user *uaddr = (__force const void __user *)insns;
43 
44 	if (!user_mode(regs))
45 		return get_kernel_nofault(*val, insns);
46 
47 	/* The user space code from other tasks cannot be accessed. */
48 	if (regs != task_pt_regs(current))
49 		return -EPERM;
50 
51 	return copy_from_user_nofault(val, uaddr, sizeof(*val));
52 }
53 
54 static void dump_instr(const char *loglvl, struct pt_regs *regs)
55 {
56 	char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
57 	const u16 *insns = (u16 *)instruction_pointer(regs);
58 	long bad;
59 	u16 val;
60 	int i;
61 
62 	for (i = -10; i < 2; i++) {
63 		bad = copy_code(regs, &val, &insns[i]);
64 		if (!bad) {
65 			p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
66 		} else {
67 			printk("%sCode: Unable to access instruction at 0x%px.\n",
68 			       loglvl, &insns[i]);
69 			return;
70 		}
71 	}
72 	printk("%sCode: %s\n", loglvl, str);
73 }
74 
75 void die(struct pt_regs *regs, const char *str)
76 {
77 	static int die_counter;
78 	int ret;
79 	long cause;
80 	unsigned long flags;
81 
82 	oops_enter();
83 
84 	raw_spin_lock_irqsave(&die_lock, flags);
85 	console_verbose();
86 	bust_spinlocks(1);
87 
88 	pr_emerg("%s [#%d]\n", str, ++die_counter);
89 	print_modules();
90 	if (regs) {
91 		show_regs(regs);
92 		dump_instr(KERN_EMERG, regs);
93 	}
94 
95 	cause = regs ? regs->cause : -1;
96 	ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
97 
98 	if (kexec_should_crash(current))
99 		crash_kexec(regs);
100 
101 	bust_spinlocks(0);
102 	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
103 	raw_spin_unlock_irqrestore(&die_lock, flags);
104 	oops_exit();
105 
106 	if (in_interrupt())
107 		panic("Fatal exception in interrupt");
108 	if (panic_on_oops)
109 		panic("Fatal exception");
110 	if (ret != NOTIFY_STOP)
111 		make_task_dead(SIGSEGV);
112 }
113 
114 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
115 {
116 	struct task_struct *tsk = current;
117 
118 	if (show_unhandled_signals && unhandled_signal(tsk, signo)
119 	    && printk_ratelimit()) {
120 		pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
121 			tsk->comm, task_pid_nr(tsk), signo, code, addr);
122 		print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
123 		pr_cont("\n");
124 		__show_regs(regs);
125 		dump_instr(KERN_INFO, regs);
126 	}
127 
128 	force_sig_fault(signo, code, (void __user *)addr);
129 }
130 
131 static void do_trap_error(struct pt_regs *regs, int signo, int code,
132 	unsigned long addr, const char *str)
133 {
134 	current->thread.bad_cause = regs->cause;
135 
136 	if (user_mode(regs)) {
137 		do_trap(regs, signo, code, addr);
138 	} else {
139 		if (!fixup_exception(regs))
140 			die(regs, str);
141 	}
142 }
143 
144 #define __trap_section noinstr
145 #define DO_ERROR_INFO(name, signo, code, str)					\
146 asmlinkage __visible __trap_section void name(struct pt_regs *regs)		\
147 {										\
148 	if (user_mode(regs)) {							\
149 		irqentry_enter_from_user_mode(regs);				\
150 		local_irq_enable();						\
151 		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
152 		local_irq_disable();						\
153 		irqentry_exit_to_user_mode(regs);				\
154 	} else {								\
155 		irqentry_state_t state = irqentry_nmi_enter(regs);		\
156 		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
157 		irqentry_nmi_exit(regs, state);					\
158 	}									\
159 }
160 
161 DO_ERROR_INFO(do_trap_unknown,
162 	SIGILL, ILL_ILLTRP, "unknown exception");
163 DO_ERROR_INFO(do_trap_hardware_error,
164 	SIGBUS, BUS_MCEERR_AR, "hardware error");
165 DO_ERROR_INFO(do_trap_insn_misaligned,
166 	SIGBUS, BUS_ADRALN, "instruction address misaligned");
167 DO_ERROR_INFO(do_trap_insn_fault,
168 	SIGSEGV, SEGV_ACCERR, "instruction access fault");
169 
170 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
171 {
172 	bool handled;
173 
174 	if (user_mode(regs)) {
175 		irqentry_enter_from_user_mode(regs);
176 		local_irq_enable();
177 
178 		handled = riscv_v_first_use_handler(regs);
179 		if (!handled)
180 			do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
181 				      "Oops - illegal instruction");
182 
183 		local_irq_disable();
184 		irqentry_exit_to_user_mode(regs);
185 	} else {
186 		irqentry_state_t state = irqentry_nmi_enter(regs);
187 
188 		do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
189 			      "Oops - illegal instruction");
190 
191 		irqentry_nmi_exit(regs, state);
192 	}
193 }
194 
195 DO_ERROR_INFO(do_trap_load_fault,
196 	SIGSEGV, SEGV_ACCERR, "load access fault");
197 
198 enum misaligned_access_type {
199 	MISALIGNED_STORE,
200 	MISALIGNED_LOAD,
201 };
202 static const struct {
203 	const char *type_str;
204 	int (*handler)(struct pt_regs *regs);
205 } misaligned_handler[] = {
206 	[MISALIGNED_STORE] = {
207 		.type_str = "Oops - store (or AMO) address misaligned",
208 		.handler = handle_misaligned_store,
209 	},
210 	[MISALIGNED_LOAD] = {
211 		.type_str = "Oops - load address misaligned",
212 		.handler = handle_misaligned_load,
213 	},
214 };
215 
216 static void do_trap_misaligned(struct pt_regs *regs, enum misaligned_access_type type)
217 {
218 	irqentry_state_t state;
219 
220 	if (user_mode(regs)) {
221 		irqentry_enter_from_user_mode(regs);
222 		local_irq_enable();
223 	} else {
224 		state = irqentry_nmi_enter(regs);
225 	}
226 
227 	if (misaligned_handler[type].handler(regs))
228 		do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
229 			      misaligned_handler[type].type_str);
230 
231 	if (user_mode(regs)) {
232 		local_irq_disable();
233 		irqentry_exit_to_user_mode(regs);
234 	} else {
235 		irqentry_nmi_exit(regs, state);
236 	}
237 }
238 
239 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
240 {
241 	do_trap_misaligned(regs, MISALIGNED_LOAD);
242 }
243 
244 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
245 {
246 	do_trap_misaligned(regs, MISALIGNED_STORE);
247 }
248 
249 DO_ERROR_INFO(do_trap_store_fault,
250 	SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
251 DO_ERROR_INFO(do_trap_ecall_s,
252 	SIGILL, ILL_ILLTRP, "environment call from S-mode");
253 DO_ERROR_INFO(do_trap_ecall_m,
254 	SIGILL, ILL_ILLTRP, "environment call from M-mode");
255 
256 static inline unsigned long get_break_insn_length(unsigned long pc)
257 {
258 	bug_insn_t insn;
259 
260 	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
261 		return 0;
262 
263 	return GET_INSN_LENGTH(insn);
264 }
265 
266 static bool probe_single_step_handler(struct pt_regs *regs)
267 {
268 	bool user = user_mode(regs);
269 
270 	return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs);
271 }
272 
273 static bool probe_breakpoint_handler(struct pt_regs *regs)
274 {
275 	bool user = user_mode(regs);
276 
277 	return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs);
278 }
279 
280 void handle_break(struct pt_regs *regs)
281 {
282 	if (probe_single_step_handler(regs))
283 		return;
284 
285 	if (probe_breakpoint_handler(regs))
286 		return;
287 
288 	current->thread.bad_cause = regs->cause;
289 
290 	if (user_mode(regs))
291 		force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
292 #ifdef CONFIG_KGDB
293 	else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
294 								== NOTIFY_STOP)
295 		return;
296 #endif
297 	else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN ||
298 		 handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
299 		regs->epc += get_break_insn_length(regs->epc);
300 	else
301 		die(regs, "Kernel BUG");
302 }
303 
304 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
305 {
306 	if (user_mode(regs)) {
307 		irqentry_enter_from_user_mode(regs);
308 		local_irq_enable();
309 
310 		handle_break(regs);
311 
312 		local_irq_disable();
313 		irqentry_exit_to_user_mode(regs);
314 	} else {
315 		irqentry_state_t state = irqentry_nmi_enter(regs);
316 
317 		handle_break(regs);
318 
319 		irqentry_nmi_exit(regs, state);
320 	}
321 }
322 
323 asmlinkage __visible __trap_section  __no_stack_protector
324 void do_trap_ecall_u(struct pt_regs *regs)
325 {
326 	if (user_mode(regs)) {
327 		long syscall = regs->a7;
328 
329 		regs->epc += 4;
330 		regs->orig_a0 = regs->a0;
331 		regs->a0 = -ENOSYS;
332 
333 		riscv_v_vstate_discard(regs);
334 
335 		if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &syscall))) {
336 			if (syscall >= 0 && syscall < NR_syscalls) {
337 				syscall = array_index_nospec(syscall, NR_syscalls);
338 				syscall_handler(regs, syscall);
339 			}
340 		}
341 		syscall_exit_to_user_mode(regs);
342 	} else {
343 		irqentry_state_t state = irqentry_nmi_enter(regs);
344 
345 		do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
346 			"Oops - environment call from U-mode");
347 
348 		irqentry_nmi_exit(regs, state);
349 	}
350 
351 }
352 
353 #define CFI_TVAL_FCFI_CODE	2
354 #define CFI_TVAL_BCFI_CODE	3
355 /* handle cfi violations */
356 bool handle_user_cfi_violation(struct pt_regs *regs)
357 {
358 	unsigned long tval = csr_read(CSR_TVAL);
359 	bool is_fcfi = (tval == CFI_TVAL_FCFI_CODE && cpu_supports_indirect_br_lp_instr());
360 	bool is_bcfi = (tval == CFI_TVAL_BCFI_CODE && cpu_supports_shadow_stack());
361 
362 	/*
363 	 * Handle uprobe event first. The probe point can be a valid target
364 	 * of indirect jumps or calls, in this case, forward cfi violation
365 	 * will be triggered instead of breakpoint exception. Clear ELP flag
366 	 * on sstatus image as well to avoid recurring fault.
367 	 */
368 	if (is_fcfi && probe_breakpoint_handler(regs)) {
369 		regs->status &= ~SR_ELP;
370 		return true;
371 	}
372 
373 	if (is_fcfi || is_bcfi) {
374 		do_trap_error(regs, SIGSEGV, SEGV_CPERR, regs->epc,
375 			      "Oops - control flow violation");
376 		return true;
377 	}
378 
379 	return false;
380 }
381 
382 /*
383  * software check exception is defined with risc-v cfi spec. Software check
384  * exception is raised when:
385  * a) An indirect branch doesn't land on 4 byte aligned PC or `lpad`
386  *    instruction or `label` value programmed in `lpad` instr doesn't
387  *    match with value setup in `x7`. reported code in `xtval` is 2.
388  * b) `sspopchk` instruction finds a mismatch between top of shadow stack (ssp)
389  *    and x1/x5. reported code in `xtval` is 3.
390  */
391 asmlinkage __visible __trap_section void do_trap_software_check(struct pt_regs *regs)
392 {
393 	if (user_mode(regs)) {
394 		irqentry_enter_from_user_mode(regs);
395 
396 		/* not a cfi violation, then merge into flow of unknown trap handler */
397 		if (!handle_user_cfi_violation(regs))
398 			do_trap_unknown(regs);
399 
400 		irqentry_exit_to_user_mode(regs);
401 	} else {
402 		/* sw check exception coming from kernel is a bug in kernel */
403 		die(regs, "Kernel BUG");
404 	}
405 }
406 
407 #ifdef CONFIG_MMU
408 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
409 {
410 	irqentry_state_t state = irqentry_enter(regs);
411 
412 	handle_page_fault(regs);
413 
414 	local_irq_disable();
415 
416 	irqentry_exit(regs, state);
417 }
418 #endif
419 
420 static void noinstr handle_riscv_irq(struct pt_regs *regs)
421 {
422 	struct pt_regs *old_regs;
423 
424 	irq_enter_rcu();
425 	old_regs = set_irq_regs(regs);
426 	handle_arch_irq(regs);
427 	set_irq_regs(old_regs);
428 	irq_exit_rcu();
429 }
430 
431 asmlinkage void noinstr do_irq(struct pt_regs *regs)
432 {
433 	irqentry_state_t state = irqentry_enter(regs);
434 
435 	if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack())
436 		call_on_irq_stack(regs, handle_riscv_irq);
437 	else
438 		handle_riscv_irq(regs);
439 
440 	irqentry_exit(regs, state);
441 }
442 
443 #ifdef CONFIG_GENERIC_BUG
444 int is_valid_bugaddr(unsigned long pc)
445 {
446 	bug_insn_t insn;
447 
448 	if (pc < VMALLOC_START)
449 		return 0;
450 	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
451 		return 0;
452 	if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
453 		return (insn == __BUG_INSN_32);
454 	else
455 		return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
456 }
457 #endif /* CONFIG_GENERIC_BUG */
458 
459 #ifdef CONFIG_VMAP_STACK
460 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
461 		overflow_stack)__aligned(16);
462 
463 asmlinkage void handle_bad_stack(struct pt_regs *regs)
464 {
465 	unsigned long tsk_stk = (unsigned long)current->stack;
466 	unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
467 
468 	console_verbose();
469 
470 	pr_emerg("Insufficient stack space to handle exception!\n");
471 	pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
472 			tsk_stk, tsk_stk + THREAD_SIZE);
473 	pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
474 			ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
475 
476 	__show_regs(regs);
477 	panic("Kernel stack overflow");
478 
479 	for (;;)
480 		wait_for_interrupt();
481 }
482 #endif
483