1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012 Regents of the University of California
4 */
5
6 #include <linux/cpu.h>
7 #include <linux/kernel.h>
8 #include <linux/init.h>
9 #include <linux/irqflags.h>
10 #include <linux/sched.h>
11 #include <linux/sched/debug.h>
12 #include <linux/sched/signal.h>
13 #include <linux/signal.h>
14 #include <linux/kdebug.h>
15 #include <linux/uaccess.h>
16 #include <linux/kprobes.h>
17 #include <linux/uprobes.h>
18 #include <asm/uprobes.h>
19 #include <linux/mm.h>
20 #include <linux/module.h>
21 #include <linux/irq.h>
22 #include <linux/kexec.h>
23 #include <linux/entry-common.h>
24
25 #include <asm/asm-prototypes.h>
26 #include <asm/bug.h>
27 #include <asm/cfi.h>
28 #include <asm/csr.h>
29 #include <asm/processor.h>
30 #include <asm/ptrace.h>
31 #include <asm/syscall.h>
32 #include <asm/thread_info.h>
33 #include <asm/vector.h>
34 #include <asm/irq_stack.h>
35
36 int show_unhandled_signals = 1;
37
38 static DEFINE_RAW_SPINLOCK(die_lock);
39
copy_code(struct pt_regs * regs,u16 * val,const u16 * insns)40 static int copy_code(struct pt_regs *regs, u16 *val, const u16 *insns)
41 {
42 const void __user *uaddr = (__force const void __user *)insns;
43
44 if (!user_mode(regs))
45 return get_kernel_nofault(*val, insns);
46
47 /* The user space code from other tasks cannot be accessed. */
48 if (regs != task_pt_regs(current))
49 return -EPERM;
50
51 return copy_from_user_nofault(val, uaddr, sizeof(*val));
52 }
53
dump_instr(const char * loglvl,struct pt_regs * regs)54 static void dump_instr(const char *loglvl, struct pt_regs *regs)
55 {
56 char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
57 const u16 *insns = (u16 *)instruction_pointer(regs);
58 long bad;
59 u16 val;
60 int i;
61
62 for (i = -10; i < 2; i++) {
63 bad = copy_code(regs, &val, &insns[i]);
64 if (!bad) {
65 p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
66 } else {
67 printk("%sCode: Unable to access instruction at 0x%px.\n",
68 loglvl, &insns[i]);
69 return;
70 }
71 }
72 printk("%sCode: %s\n", loglvl, str);
73 }
74
die(struct pt_regs * regs,const char * str)75 void die(struct pt_regs *regs, const char *str)
76 {
77 static int die_counter;
78 int ret;
79 long cause;
80 unsigned long flags;
81
82 oops_enter();
83
84 raw_spin_lock_irqsave(&die_lock, flags);
85 console_verbose();
86 bust_spinlocks(1);
87
88 pr_emerg("%s [#%d]\n", str, ++die_counter);
89 print_modules();
90 if (regs) {
91 show_regs(regs);
92 dump_instr(KERN_EMERG, regs);
93 }
94
95 cause = regs ? regs->cause : -1;
96 ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
97
98 if (kexec_should_crash(current))
99 crash_kexec(regs);
100
101 bust_spinlocks(0);
102 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
103 raw_spin_unlock_irqrestore(&die_lock, flags);
104 oops_exit();
105
106 if (in_interrupt())
107 panic("Fatal exception in interrupt");
108 if (panic_on_oops)
109 panic("Fatal exception");
110 if (ret != NOTIFY_STOP)
111 make_task_dead(SIGSEGV);
112 }
113
do_trap(struct pt_regs * regs,int signo,int code,unsigned long addr)114 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
115 {
116 struct task_struct *tsk = current;
117
118 if (show_unhandled_signals && unhandled_signal(tsk, signo)
119 && printk_ratelimit()) {
120 pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
121 tsk->comm, task_pid_nr(tsk), signo, code, addr);
122 print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
123 pr_cont("\n");
124 __show_regs(regs);
125 dump_instr(KERN_INFO, regs);
126 }
127
128 force_sig_fault(signo, code, (void __user *)addr);
129 }
130
do_trap_error(struct pt_regs * regs,int signo,int code,unsigned long addr,const char * str)131 static void do_trap_error(struct pt_regs *regs, int signo, int code,
132 unsigned long addr, const char *str)
133 {
134 current->thread.bad_cause = regs->cause;
135
136 if (user_mode(regs)) {
137 do_trap(regs, signo, code, addr);
138 } else {
139 if (!fixup_exception(regs))
140 die(regs, str);
141 }
142 }
143
144 #define __trap_section noinstr
145 #define DO_ERROR_INFO(name, signo, code, str) \
146 asmlinkage __visible __trap_section void name(struct pt_regs *regs) \
147 { \
148 if (user_mode(regs)) { \
149 irqentry_enter_from_user_mode(regs); \
150 local_irq_enable(); \
151 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
152 local_irq_disable(); \
153 irqentry_exit_to_user_mode(regs); \
154 } else { \
155 irqentry_state_t state = irqentry_nmi_enter(regs); \
156 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
157 irqentry_nmi_exit(regs, state); \
158 } \
159 }
160
161 DO_ERROR_INFO(do_trap_unknown,
162 SIGILL, ILL_ILLTRP, "unknown exception");
163 DO_ERROR_INFO(do_trap_hardware_error,
164 SIGBUS, BUS_MCEERR_AR, "hardware error");
165 DO_ERROR_INFO(do_trap_insn_misaligned,
166 SIGBUS, BUS_ADRALN, "instruction address misaligned");
167 DO_ERROR_INFO(do_trap_insn_fault,
168 SIGSEGV, SEGV_ACCERR, "instruction access fault");
169
do_trap_insn_illegal(struct pt_regs * regs)170 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
171 {
172 bool handled;
173
174 if (user_mode(regs)) {
175 irqentry_enter_from_user_mode(regs);
176 local_irq_enable();
177
178 handled = riscv_v_first_use_handler(regs);
179 if (!handled)
180 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
181 "Oops - illegal instruction");
182
183 local_irq_disable();
184 irqentry_exit_to_user_mode(regs);
185 } else {
186 irqentry_state_t state = irqentry_nmi_enter(regs);
187
188 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
189 "Oops - illegal instruction");
190
191 irqentry_nmi_exit(regs, state);
192 }
193 }
194
195 DO_ERROR_INFO(do_trap_load_fault,
196 SIGSEGV, SEGV_ACCERR, "load access fault");
197
198 enum misaligned_access_type {
199 MISALIGNED_STORE,
200 MISALIGNED_LOAD,
201 };
202 static const struct {
203 const char *type_str;
204 int (*handler)(struct pt_regs *regs);
205 } misaligned_handler[] = {
206 [MISALIGNED_STORE] = {
207 .type_str = "Oops - store (or AMO) address misaligned",
208 .handler = handle_misaligned_store,
209 },
210 [MISALIGNED_LOAD] = {
211 .type_str = "Oops - load address misaligned",
212 .handler = handle_misaligned_load,
213 },
214 };
215
do_trap_misaligned(struct pt_regs * regs,enum misaligned_access_type type)216 static void do_trap_misaligned(struct pt_regs *regs, enum misaligned_access_type type)
217 {
218 irqentry_state_t state;
219
220 if (user_mode(regs)) {
221 irqentry_enter_from_user_mode(regs);
222 local_irq_enable();
223 } else {
224 state = irqentry_nmi_enter(regs);
225 }
226
227 if (misaligned_handler[type].handler(regs))
228 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
229 misaligned_handler[type].type_str);
230
231 if (user_mode(regs)) {
232 local_irq_disable();
233 irqentry_exit_to_user_mode(regs);
234 } else {
235 irqentry_nmi_exit(regs, state);
236 }
237 }
238
do_trap_load_misaligned(struct pt_regs * regs)239 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
240 {
241 do_trap_misaligned(regs, MISALIGNED_LOAD);
242 }
243
do_trap_store_misaligned(struct pt_regs * regs)244 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
245 {
246 do_trap_misaligned(regs, MISALIGNED_STORE);
247 }
248
249 DO_ERROR_INFO(do_trap_store_fault,
250 SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
251 DO_ERROR_INFO(do_trap_ecall_s,
252 SIGILL, ILL_ILLTRP, "environment call from S-mode");
253 DO_ERROR_INFO(do_trap_ecall_m,
254 SIGILL, ILL_ILLTRP, "environment call from M-mode");
255
get_break_insn_length(unsigned long pc)256 static inline unsigned long get_break_insn_length(unsigned long pc)
257 {
258 bug_insn_t insn;
259
260 if (get_kernel_nofault(insn, (bug_insn_t *)pc))
261 return 0;
262
263 return GET_INSN_LENGTH(insn);
264 }
265
probe_single_step_handler(struct pt_regs * regs)266 static bool probe_single_step_handler(struct pt_regs *regs)
267 {
268 bool user = user_mode(regs);
269
270 return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs);
271 }
272 NOKPROBE_SYMBOL(probe_single_step_handler);
273
probe_breakpoint_handler(struct pt_regs * regs)274 static bool probe_breakpoint_handler(struct pt_regs *regs)
275 {
276 bool user = user_mode(regs);
277
278 return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs);
279 }
280 NOKPROBE_SYMBOL(probe_breakpoint_handler);
281
handle_break(struct pt_regs * regs)282 void handle_break(struct pt_regs *regs)
283 {
284 if (probe_single_step_handler(regs))
285 return;
286
287 if (probe_breakpoint_handler(regs))
288 return;
289
290 current->thread.bad_cause = regs->cause;
291
292 if (user_mode(regs))
293 force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
294 #ifdef CONFIG_KGDB
295 else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
296 == NOTIFY_STOP)
297 return;
298 #endif
299 else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN ||
300 handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
301 regs->epc += get_break_insn_length(regs->epc);
302 else
303 die(regs, "Kernel BUG");
304 }
305 NOKPROBE_SYMBOL(handle_break);
306
do_trap_break(struct pt_regs * regs)307 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
308 {
309 if (user_mode(regs)) {
310 irqentry_enter_from_user_mode(regs);
311 local_irq_enable();
312
313 handle_break(regs);
314
315 local_irq_disable();
316 irqentry_exit_to_user_mode(regs);
317 } else {
318 irqentry_state_t state = irqentry_nmi_enter(regs);
319
320 handle_break(regs);
321
322 irqentry_nmi_exit(regs, state);
323 }
324 }
325
326 asmlinkage __visible __trap_section __no_stack_protector
do_trap_ecall_u(struct pt_regs * regs)327 void do_trap_ecall_u(struct pt_regs *regs)
328 {
329 if (user_mode(regs)) {
330 long syscall = regs->a7;
331
332 regs->epc += 4;
333 regs->orig_a0 = regs->a0;
334 regs->a0 = -ENOSYS;
335
336 riscv_v_vstate_discard(regs);
337
338 if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &syscall))) {
339 if (syscall >= 0 && syscall < NR_syscalls) {
340 syscall = array_index_nospec(syscall, NR_syscalls);
341 syscall_handler(regs, syscall);
342 }
343 }
344 syscall_exit_to_user_mode(regs);
345 } else {
346 irqentry_state_t state = irqentry_nmi_enter(regs);
347
348 do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
349 "Oops - environment call from U-mode");
350
351 irqentry_nmi_exit(regs, state);
352 }
353
354 }
355
356 #define CFI_TVAL_FCFI_CODE 2
357 #define CFI_TVAL_BCFI_CODE 3
358 /* handle cfi violations */
handle_user_cfi_violation(struct pt_regs * regs)359 bool handle_user_cfi_violation(struct pt_regs *regs)
360 {
361 unsigned long tval = csr_read(CSR_TVAL);
362 bool is_fcfi = (tval == CFI_TVAL_FCFI_CODE && cpu_supports_indirect_br_lp_instr());
363 bool is_bcfi = (tval == CFI_TVAL_BCFI_CODE && cpu_supports_shadow_stack());
364
365 /*
366 * Handle uprobe event first. The probe point can be a valid target
367 * of indirect jumps or calls, in this case, forward cfi violation
368 * will be triggered instead of breakpoint exception. Clear ELP flag
369 * on sstatus image as well to avoid recurring fault.
370 */
371 if (is_fcfi && probe_breakpoint_handler(regs)) {
372 regs->status &= ~SR_ELP;
373 return true;
374 }
375
376 if (is_fcfi || is_bcfi) {
377 do_trap_error(regs, SIGSEGV, SEGV_CPERR, regs->epc,
378 "Oops - control flow violation");
379 return true;
380 }
381
382 return false;
383 }
384
385 /*
386 * software check exception is defined with risc-v cfi spec. Software check
387 * exception is raised when:
388 * a) An indirect branch doesn't land on 4 byte aligned PC or `lpad`
389 * instruction or `label` value programmed in `lpad` instr doesn't
390 * match with value setup in `x7`. reported code in `xtval` is 2.
391 * b) `sspopchk` instruction finds a mismatch between top of shadow stack (ssp)
392 * and x1/x5. reported code in `xtval` is 3.
393 */
do_trap_software_check(struct pt_regs * regs)394 asmlinkage __visible __trap_section void do_trap_software_check(struct pt_regs *regs)
395 {
396 if (user_mode(regs)) {
397 irqentry_enter_from_user_mode(regs);
398
399 /* not a cfi violation, then merge into flow of unknown trap handler */
400 if (!handle_user_cfi_violation(regs))
401 do_trap_unknown(regs);
402
403 irqentry_exit_to_user_mode(regs);
404 } else {
405 /* sw check exception coming from kernel is a bug in kernel */
406 die(regs, "Kernel BUG");
407 }
408 }
409
410 #ifdef CONFIG_MMU
do_page_fault(struct pt_regs * regs)411 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
412 {
413 irqentry_state_t state = irqentry_enter(regs);
414
415 handle_page_fault(regs);
416
417 local_irq_disable();
418
419 irqentry_exit(regs, state);
420 }
421 #endif
422
handle_riscv_irq(struct pt_regs * regs)423 static void noinstr handle_riscv_irq(struct pt_regs *regs)
424 {
425 struct pt_regs *old_regs;
426
427 irq_enter_rcu();
428 old_regs = set_irq_regs(regs);
429 handle_arch_irq(regs);
430 set_irq_regs(old_regs);
431 irq_exit_rcu();
432 }
433
do_irq(struct pt_regs * regs)434 asmlinkage void noinstr do_irq(struct pt_regs *regs)
435 {
436 irqentry_state_t state = irqentry_enter(regs);
437
438 if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack())
439 call_on_irq_stack(regs, handle_riscv_irq);
440 else
441 handle_riscv_irq(regs);
442
443 irqentry_exit(regs, state);
444 }
445
446 #ifdef CONFIG_GENERIC_BUG
is_valid_bugaddr(unsigned long pc)447 int is_valid_bugaddr(unsigned long pc)
448 {
449 bug_insn_t insn;
450
451 if (pc < VMALLOC_START)
452 return 0;
453 if (get_kernel_nofault(insn, (bug_insn_t *)pc))
454 return 0;
455 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
456 return (insn == __BUG_INSN_32);
457 else
458 return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
459 }
460 #endif /* CONFIG_GENERIC_BUG */
461
462 #ifdef CONFIG_VMAP_STACK
463 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
464 overflow_stack)__aligned(16);
465
handle_bad_stack(struct pt_regs * regs)466 asmlinkage void handle_bad_stack(struct pt_regs *regs)
467 {
468 unsigned long tsk_stk = (unsigned long)current->stack;
469 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
470
471 console_verbose();
472
473 pr_emerg("Insufficient stack space to handle exception!\n");
474 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
475 tsk_stk, tsk_stk + THREAD_SIZE);
476 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
477 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
478
479 __show_regs(regs);
480 panic("Kernel stack overflow");
481
482 for (;;)
483 wait_for_interrupt();
484 }
485 #endif
486