1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28 #include "opt_ddb.h"
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/asan.h>
33 #include <sys/kernel.h>
34 #include <sys/ktr.h>
35 #include <sys/lock.h>
36 #include <sys/msan.h>
37 #include <sys/mutex.h>
38 #include <sys/proc.h>
39 #include <sys/ptrace.h>
40 #include <sys/syscall.h>
41 #include <sys/sysent.h>
42 #ifdef KDB
43 #include <sys/kdb.h>
44 #endif
45
46 #include <vm/vm.h>
47 #include <vm/pmap.h>
48 #include <vm/vm_kern.h>
49 #include <vm/vm_map.h>
50 #include <vm/vm_param.h>
51 #include <vm/vm_extern.h>
52
53 #include <machine/frame.h>
54 #include <machine/md_var.h>
55 #include <machine/pcb.h>
56 #include <machine/pcpu.h>
57 #include <machine/undefined.h>
58
59 #ifdef KDTRACE_HOOKS
60 #include <sys/dtrace_bsd.h>
61 #endif
62
63 #ifdef VFP
64 #include <machine/vfp.h>
65 #endif
66
67 #ifdef KDB
68 #include <machine/db_machdep.h>
69 #endif
70
71 #ifdef DDB
72 #include <ddb/ddb.h>
73 #include <ddb/db_sym.h>
74 #endif
75
76 /* Called from exception.S */
77 void do_el1h_sync(struct thread *, struct trapframe *);
78 void do_el0_sync(struct thread *, struct trapframe *);
79 void do_el0_error(struct trapframe *);
80 void do_serror(struct trapframe *);
81 void unhandled_exception(struct trapframe *);
82
83 static void print_gp_register(const char *name, uint64_t value);
84 static void print_registers(struct trapframe *frame);
85
86 int (*dtrace_invop_jump_addr)(struct trapframe *);
87
88 u_long cnt_efirt_faults;
89 int print_efirt_faults;
90
91 typedef void (abort_handler)(struct thread *, struct trapframe *, uint64_t,
92 uint64_t, int);
93
94 static abort_handler align_abort;
95 static abort_handler data_abort;
96 static abort_handler external_abort;
97 static abort_handler tag_check_abort;
98
99 static abort_handler *abort_handlers[] = {
100 [ISS_DATA_DFSC_TF_L0] = data_abort,
101 [ISS_DATA_DFSC_TF_L1] = data_abort,
102 [ISS_DATA_DFSC_TF_L2] = data_abort,
103 [ISS_DATA_DFSC_TF_L3] = data_abort,
104 [ISS_DATA_DFSC_AFF_L1] = data_abort,
105 [ISS_DATA_DFSC_AFF_L2] = data_abort,
106 [ISS_DATA_DFSC_AFF_L3] = data_abort,
107 [ISS_DATA_DFSC_PF_L1] = data_abort,
108 [ISS_DATA_DFSC_PF_L2] = data_abort,
109 [ISS_DATA_DFSC_PF_L3] = data_abort,
110 [ISS_DATA_DFSC_TAG] = tag_check_abort,
111 [ISS_DATA_DFSC_ALIGN] = align_abort,
112 [ISS_DATA_DFSC_EXT] = external_abort,
113 [ISS_DATA_DFSC_EXT_L0] = external_abort,
114 [ISS_DATA_DFSC_EXT_L1] = external_abort,
115 [ISS_DATA_DFSC_EXT_L2] = external_abort,
116 [ISS_DATA_DFSC_EXT_L3] = external_abort,
117 [ISS_DATA_DFSC_ECC] = external_abort,
118 [ISS_DATA_DFSC_ECC_L0] = external_abort,
119 [ISS_DATA_DFSC_ECC_L1] = external_abort,
120 [ISS_DATA_DFSC_ECC_L2] = external_abort,
121 [ISS_DATA_DFSC_ECC_L3] = external_abort,
122 };
123
124 static __inline void
call_trapsignal(struct thread * td,int sig,int code,void * addr,int trapno)125 call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno)
126 {
127 ksiginfo_t ksi;
128
129 ksiginfo_init_trap(&ksi);
130 ksi.ksi_signo = sig;
131 ksi.ksi_code = code;
132 ksi.ksi_addr = addr;
133 ksi.ksi_trapno = trapno;
134 ksi.ksi_flags |= KSI_EXCEPT;
135 trapsignal(td, &ksi);
136 }
137
138 int
cpu_fetch_syscall_args(struct thread * td)139 cpu_fetch_syscall_args(struct thread *td)
140 {
141 struct proc *p;
142 syscallarg_t *ap, *dst_ap;
143 struct syscall_args *sa;
144
145 p = td->td_proc;
146 sa = &td->td_sa;
147 ap = td->td_frame->tf_x;
148 dst_ap = &sa->args[0];
149
150 sa->code = td->td_frame->tf_x[8];
151 sa->original_code = sa->code;
152
153 if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) {
154 sa->code = *ap++;
155 } else {
156 *dst_ap++ = *ap++;
157 }
158
159 if (__predict_false(sa->code >= p->p_sysent->sv_size))
160 sa->callp = &nosys_sysent;
161 else
162 sa->callp = &p->p_sysent->sv_table[sa->code];
163
164 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
165 ("Syscall %d takes too many arguments", sa->code));
166
167 memcpy(dst_ap, ap, (nitems(sa->args) - 1) * sizeof(*dst_ap));
168
169 td->td_retval[0] = 0;
170 td->td_retval[1] = 0;
171
172 return (0);
173 }
174
175 #include "../../kern/subr_syscall.c"
176
177 /*
178 * Test for fault generated by given access instruction in
179 * bus_peek_<foo> or bus_poke_<foo> bus function.
180 */
181 extern uint32_t generic_bs_peek_1f, generic_bs_peek_2f;
182 extern uint32_t generic_bs_peek_4f, generic_bs_peek_8f;
183 extern uint32_t generic_bs_poke_1f, generic_bs_poke_2f;
184 extern uint32_t generic_bs_poke_4f, generic_bs_poke_8f;
185
186 static bool
test_bs_fault(void * addr)187 test_bs_fault(void *addr)
188 {
189 return (addr == &generic_bs_peek_1f ||
190 addr == &generic_bs_peek_2f ||
191 addr == &generic_bs_peek_4f ||
192 addr == &generic_bs_peek_8f ||
193 addr == &generic_bs_poke_1f ||
194 addr == &generic_bs_poke_2f ||
195 addr == &generic_bs_poke_4f ||
196 addr == &generic_bs_poke_8f);
197 }
198
199 static bool
svc_handler(struct thread * td,struct trapframe * frame)200 svc_handler(struct thread *td, struct trapframe *frame)
201 {
202
203 if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) {
204 syscallenter(td);
205 syscallret(td);
206 /* Skip userret as syscallret already called it */
207 return (true);
208 } else {
209 call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr,
210 ESR_ELx_EXCEPTION(frame->tf_esr));
211 return (false);
212 }
213 }
214
215 static void
align_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)216 align_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
217 uint64_t far, int lower)
218 {
219 if (!lower) {
220 print_registers(frame);
221 print_gp_register("far", far);
222 printf(" esr: 0x%.16lx\n", esr);
223 panic("Misaligned access from kernel space!");
224 }
225
226 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
227 ESR_ELx_EXCEPTION(frame->tf_esr));
228 }
229
230
231 static void
external_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)232 external_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
233 uint64_t far, int lower)
234 {
235 if (lower) {
236 call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)far,
237 ESR_ELx_EXCEPTION(frame->tf_esr));
238 return;
239 }
240
241 /*
242 * Try to handle synchronous external aborts caused by
243 * bus_space_peek() and/or bus_space_poke() functions.
244 */
245 if (test_bs_fault((void *)frame->tf_elr)) {
246 frame->tf_elr = (uint64_t)generic_bs_fault;
247 return;
248 }
249
250 print_registers(frame);
251 print_gp_register("far", far);
252 printf(" esr: 0x%.16lx\n", esr);
253 panic("Unhandled external data abort");
254 }
255
256 static void
tag_check_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)257 tag_check_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
258 uint64_t far, int lower)
259 {
260 /*
261 * A Tag Check Fault should be handled as a SIGSEGV if it occurs
262 * at EL0 and a kernel panic if at EL1.
263 */
264 if (!lower) {
265 print_registers(frame);
266 print_gp_register("far", far);
267 printf(" esr: 0x%.16lx\n", esr);
268 panic("Tag Check Fault");
269 }
270
271 call_trapsignal(td, SIGSEGV, SEGV_MTESERR, (void *)far,
272 ESR_ELx_EXCEPTION(frame->tf_esr));
273 userret(td, frame);
274 }
275
276 /*
277 * It is unsafe to access the stack canary value stored in "td" until
278 * kernel map translation faults are handled, see the pmap_klookup() call below.
279 * Thus, stack-smashing detection with per-thread canaries must be disabled in
280 * this function.
281 */
282 static void NO_PERTHREAD_SSP
data_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)283 data_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
284 uint64_t far, int lower)
285 {
286 struct vm_map *map;
287 struct pcb *pcb;
288 vm_offset_t fault_va;
289 vm_prot_t ftype;
290 int error, sig, ucode;
291 #ifdef KDB
292 bool handled;
293 #endif
294
295 /*
296 * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive
297 * and Store-Exclusive instruction usage restrictions", state
298 * of the exclusive monitors after data abort exception is unknown.
299 */
300 clrex();
301
302 #ifdef KDB
303 if (kdb_active) {
304 kdb_reenter();
305 return;
306 }
307 #endif
308
309 fault_va = far;
310 if (lower) {
311 map = &td->td_proc->p_vmspace->vm_map;
312 if ((td->td_proc->p_md.md_tcr & TCR_TBI0) != 0)
313 fault_va = ADDR_MAKE_CANONICAL(far);
314 } else if (!ADDR_IS_CANONICAL(far)) {
315 /* We received a TBI/PAC/etc. fault from the kernel */
316 error = KERN_INVALID_ADDRESS;
317 pcb = td->td_pcb;
318 goto bad_far;
319 } else if (ADDR_IS_KERNEL(far)) {
320 /*
321 * Handle a special case: the data abort was caused by accessing
322 * a thread structure while its mapping was being promoted or
323 * demoted, as a consequence of the break-before-make rule. It
324 * is not safe to enable interrupts or dereference "td" before
325 * this case is handled.
326 *
327 * In principle, if pmap_klookup() fails, there is no need to
328 * call pmap_fault() below, but avoiding that call is not worth
329 * the effort.
330 */
331 if (ESR_ELx_EXCEPTION(esr) == EXCP_DATA_ABORT) {
332 switch (esr & ISS_DATA_DFSC_MASK) {
333 case ISS_DATA_DFSC_TF_L0:
334 case ISS_DATA_DFSC_TF_L1:
335 case ISS_DATA_DFSC_TF_L2:
336 case ISS_DATA_DFSC_TF_L3:
337 if (pmap_klookup(far, NULL))
338 return;
339 break;
340 }
341 }
342 if (td->td_md.md_spinlock_count == 0 &&
343 (frame->tf_spsr & PSR_DAIF_INTR) != PSR_DAIF_INTR) {
344 MPASS((frame->tf_spsr & PSR_DAIF_INTR) == 0);
345 intr_enable();
346 }
347 map = kernel_map;
348 } else {
349 if (td->td_md.md_spinlock_count == 0 &&
350 (frame->tf_spsr & PSR_DAIF_INTR) != PSR_DAIF_INTR) {
351 MPASS((frame->tf_spsr & PSR_DAIF_INTR) == 0);
352 intr_enable();
353 }
354 map = &td->td_proc->p_vmspace->vm_map;
355 if (map == NULL)
356 map = kernel_map;
357 }
358 pcb = td->td_pcb;
359
360 /*
361 * Try to handle translation, access flag, and permission faults.
362 * Translation faults may occur as a result of the required
363 * break-before-make sequence used when promoting or demoting
364 * superpages. Such faults must not occur while holding the pmap lock,
365 * or pmap_fault() will recurse on that lock.
366 */
367 if ((lower || map == kernel_map || pcb->pcb_onfault != 0) &&
368 pmap_fault(map->pmap, esr, fault_va) == KERN_SUCCESS)
369 return;
370
371 #ifdef INVARIANTS
372 if (td->td_md.md_spinlock_count != 0) {
373 print_registers(frame);
374 print_gp_register("far", far);
375 printf(" esr: 0x%.16lx\n", esr);
376 panic("data abort with spinlock held (spinlock count %d != 0)",
377 td->td_md.md_spinlock_count);
378 }
379 #endif
380 if ((td->td_pflags & TDP_NOFAULTING) == 0 &&
381 (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
382 WARN_GIANTOK, NULL, "Kernel page fault") != 0)) {
383 print_registers(frame);
384 print_gp_register("far", far);
385 printf(" esr: 0x%.16lx\n", esr);
386 panic("data abort in critical section or under mutex");
387 }
388
389 switch (ESR_ELx_EXCEPTION(esr)) {
390 case EXCP_INSN_ABORT:
391 case EXCP_INSN_ABORT_L:
392 ftype = VM_PROT_EXECUTE;
393 break;
394 default:
395 /*
396 * If the exception was because of a read or cache operation
397 * pass a read fault type into the vm code. Cache operations
398 * need read permission but will set the WnR flag when the
399 * memory is unmapped.
400 */
401 if ((esr & ISS_DATA_WnR) == 0 || (esr & ISS_DATA_CM) != 0)
402 ftype = VM_PROT_READ;
403 else
404 ftype = VM_PROT_WRITE;
405 break;
406 }
407
408 /* Fault in the page. */
409 error = vm_fault_trap(map, fault_va, ftype, VM_FAULT_NORMAL, &sig,
410 &ucode);
411 if (error != KERN_SUCCESS) {
412 if (lower) {
413 call_trapsignal(td, sig, ucode, (void *)far,
414 ESR_ELx_EXCEPTION(esr));
415 } else {
416 bad_far:
417 if (td->td_intr_nesting_level == 0 &&
418 pcb->pcb_onfault != 0) {
419 frame->tf_elr = pcb->pcb_onfault;
420 return;
421 }
422
423 printf("Fatal data abort:\n");
424 print_registers(frame);
425 print_gp_register("far", far);
426 printf(" esr: 0x%.16lx\n", esr);
427
428 #ifdef KDB
429 if (debugger_on_trap) {
430 kdb_why = KDB_WHY_TRAP;
431 handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0,
432 frame);
433 kdb_why = KDB_WHY_UNSET;
434 if (handled)
435 return;
436 }
437 #endif
438 panic("vm_fault failed: 0x%lx error %d",
439 frame->tf_elr, error);
440 }
441 }
442 }
443
444 static void
print_gp_register(const char * name,uint64_t value)445 print_gp_register(const char *name, uint64_t value)
446 {
447 #if defined(DDB)
448 c_db_sym_t sym;
449 const char *sym_name;
450 db_expr_t sym_value;
451 db_expr_t offset;
452 #endif
453
454 printf(" %s: 0x%.16lx", name, value);
455 #if defined(DDB)
456 /* If this looks like a kernel address try to find the symbol */
457 if (value >= VM_MIN_KERNEL_ADDRESS) {
458 sym = db_search_symbol(value, DB_STGY_ANY, &offset);
459 if (sym != C_DB_SYM_NULL) {
460 db_symbol_values(sym, &sym_name, &sym_value);
461 printf(" (%s + 0x%lx)", sym_name, offset);
462 }
463 }
464 #endif
465 printf("\n");
466 }
467
468 static void
print_registers(struct trapframe * frame)469 print_registers(struct trapframe *frame)
470 {
471 char name[4];
472 u_int reg;
473
474 for (reg = 0; reg < nitems(frame->tf_x); reg++) {
475 snprintf(name, sizeof(name), "%sx%d", (reg < 10) ? " " : "",
476 reg);
477 print_gp_register(name, frame->tf_x[reg]);
478 }
479 printf(" sp: 0x%.16lx\n", frame->tf_sp);
480 print_gp_register(" lr", frame->tf_lr);
481 print_gp_register("elr", frame->tf_elr);
482 printf("spsr: 0x%.16lx\n", frame->tf_spsr);
483 }
484
485 #ifdef VFP
486 static void
fpe_trap(struct thread * td,void * addr,uint32_t exception)487 fpe_trap(struct thread *td, void *addr, uint32_t exception)
488 {
489 int code;
490
491 code = FPE_FLTIDO;
492 if ((exception & ISS_FP_TFV) != 0) {
493 if ((exception & ISS_FP_IOF) != 0)
494 code = FPE_FLTINV;
495 else if ((exception & ISS_FP_DZF) != 0)
496 code = FPE_FLTDIV;
497 else if ((exception & ISS_FP_OFF) != 0)
498 code = FPE_FLTOVF;
499 else if ((exception & ISS_FP_UFF) != 0)
500 code = FPE_FLTUND;
501 else if ((exception & ISS_FP_IXF) != 0)
502 code = FPE_FLTRES;
503 }
504 call_trapsignal(td, SIGFPE, code, addr, exception);
505 }
506 #endif
507
508 static void
handle_moe(struct thread * td,struct trapframe * frame,uint64_t esr)509 handle_moe(struct thread *td, struct trapframe *frame, uint64_t esr)
510 {
511 uint64_t src;
512 uint64_t dest;
513 uint64_t size;
514 int src_reg;
515 int dest_reg;
516 int size_reg;
517 int format_option;
518
519 format_option = esr & ISS_MOE_FORMAT_OPTION_MASK;
520 dest_reg = (esr & ISS_MOE_DESTREG_MASK) >> ISS_MOE_DESTREG_SHIFT;
521 size_reg = (esr & ISS_MOE_SIZEREG_MASK) >> ISS_MOE_SIZEREG_SHIFT;
522 dest = frame->tf_x[dest_reg];
523 size = frame->tf_x[size_reg];
524
525 /*
526 * Put the registers back in the original format suitable for a
527 * prologue instruction, using the generic return routine from the
528 * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH.
529 */
530 if (esr & ISS_MOE_MEMINST) {
531 /* SET* instruction */
532 if (format_option == ISS_MOE_FORMAT_OPTION_A ||
533 format_option == ISS_MOE_FORMAT_OPTION_A2) {
534 /* Format is from Option A; forward set */
535 frame->tf_x[dest_reg] = dest + size;
536 frame->tf_x[size_reg] = -size;
537 }
538 } else {
539 /* CPY* instruction */
540 src_reg = (esr & ISS_MOE_SRCREG_MASK) >> ISS_MOE_SRCREG_SHIFT;
541 src = frame->tf_x[src_reg];
542
543 if (format_option == ISS_MOE_FORMAT_OPTION_B ||
544 format_option == ISS_MOE_FORMAT_OPTION_B2) {
545 /* Format is from Option B */
546 if (frame->tf_spsr & PSR_N) {
547 /* Backward copy */
548 frame->tf_x[dest_reg] = dest - size;
549 frame->tf_x[src_reg] = src + size;
550 }
551 } else {
552 /* Format is from Option A */
553 if (frame->tf_x[size_reg] & (1UL << 63)) {
554 /* Forward copy */
555 frame->tf_x[dest_reg] = dest + size;
556 frame->tf_x[src_reg] = src + size;
557 frame->tf_x[size_reg] = -size;
558 }
559 }
560 }
561
562 if (esr & ISS_MOE_FROM_EPILOGUE)
563 frame->tf_elr -= 8;
564 else
565 frame->tf_elr -= 4;
566 }
567
568 /*
569 * See the comment above data_abort().
570 */
571 void NO_PERTHREAD_SSP
do_el1h_sync(struct thread * td,struct trapframe * frame)572 do_el1h_sync(struct thread *td, struct trapframe *frame)
573 {
574 uint32_t exception;
575 uint64_t esr, far;
576 int dfsc;
577
578 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
579 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
580
581 far = frame->tf_far;
582 /* Read the esr register to get the exception details */
583 esr = frame->tf_esr;
584 exception = ESR_ELx_EXCEPTION(esr);
585
586 #ifdef KDTRACE_HOOKS
587 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
588 return;
589 #endif
590
591 CTR4(KTR_TRAP, "%s: exception=%lu, elr=0x%lx, esr=0x%lx",
592 __func__, exception, frame->tf_elr, esr);
593
594 /*
595 * Enable debug exceptions if we aren't already handling one. They will
596 * be masked again in the exception handler's epilogue.
597 */
598 switch (exception) {
599 case EXCP_BRK:
600 case EXCP_BRKPT_EL1:
601 case EXCP_WATCHPT_EL1:
602 case EXCP_SOFTSTP_EL1:
603 break;
604 default:
605 dbg_enable();
606 break;
607 }
608
609 switch (exception) {
610 case EXCP_FP_SIMD:
611 case EXCP_TRAP_FP:
612 #ifdef VFP
613 if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) {
614 vfp_restore_state();
615 } else
616 #endif
617 {
618 print_registers(frame);
619 printf(" esr: 0x%.16lx\n", esr);
620 panic("VFP exception in the kernel");
621 }
622 break;
623 case EXCP_INSN_ABORT:
624 case EXCP_DATA_ABORT:
625 dfsc = esr & ISS_DATA_DFSC_MASK;
626 if (dfsc < nitems(abort_handlers) &&
627 abort_handlers[dfsc] != NULL) {
628 abort_handlers[dfsc](td, frame, esr, far, 0);
629 } else {
630 print_registers(frame);
631 print_gp_register("far", far);
632 printf(" esr: 0x%.16lx\n", esr);
633 panic("Unhandled EL1 %s abort: 0x%x",
634 exception == EXCP_INSN_ABORT ? "instruction" :
635 "data", dfsc);
636 }
637 break;
638 case EXCP_BRK:
639 #ifdef KDTRACE_HOOKS
640 if ((esr & ESR_ELx_ISS_MASK) == 0x40d /* BRK_IMM16_VAL */ &&
641 dtrace_invop_jump_addr != NULL &&
642 dtrace_invop_jump_addr(frame) == 0)
643 break;
644 #endif
645 #ifdef KDB
646 kdb_trap(exception, 0, frame);
647 #else
648 panic("No debugger in kernel.");
649 #endif
650 break;
651 case EXCP_BRKPT_EL1:
652 case EXCP_WATCHPT_EL1:
653 case EXCP_SOFTSTP_EL1:
654 #ifdef KDB
655 kdb_trap(exception, 0, frame);
656 #else
657 panic("No debugger in kernel.");
658 #endif
659 break;
660 case EXCP_FPAC:
661 /* We can see this if the authentication on PAC fails */
662 print_registers(frame);
663 print_gp_register("far", far);
664 panic("FPAC kernel exception");
665 break;
666 case EXCP_UNKNOWN:
667 print_registers(frame);
668 print_gp_register("far", far);
669 panic("Undefined instruction: %08x",
670 *(uint32_t *)frame->tf_elr);
671 break;
672 case EXCP_BTI:
673 print_registers(frame);
674 print_gp_register("far", far);
675 panic("Branch Target exception");
676 break;
677 case EXCP_MOE:
678 handle_moe(td, frame, esr);
679 break;
680 default:
681 print_registers(frame);
682 print_gp_register("far", far);
683 panic("Unknown kernel exception 0x%x esr_el1 0x%lx", exception,
684 esr);
685 }
686 }
687
688 void
do_el0_sync(struct thread * td,struct trapframe * frame)689 do_el0_sync(struct thread *td, struct trapframe *frame)
690 {
691 pcpu_bp_harden bp_harden;
692 uint32_t exception;
693 uint64_t esr, far;
694 int dfsc;
695 bool skip_userret;
696
697 /* Check we have a sane environment when entering from userland */
698 KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
699 ("Invalid pcpu address from userland: %p (tpidr 0x%lx)",
700 get_pcpu(), READ_SPECIALREG(tpidr_el1)));
701
702 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
703 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
704
705 far = frame->tf_far;
706 esr = frame->tf_esr;
707 exception = ESR_ELx_EXCEPTION(esr);
708 if (exception == EXCP_INSN_ABORT_L && far > VM_MAXUSER_ADDRESS) {
709 /*
710 * Userspace may be trying to train the branch predictor to
711 * attack the kernel. If we are on a CPU affected by this
712 * call the handler to clear the branch predictor state.
713 */
714 bp_harden = PCPU_GET(bp_harden);
715 if (bp_harden != NULL)
716 bp_harden();
717 }
718 intr_enable();
719
720 CTR4(KTR_TRAP, "%s: exception=%lu, elr=0x%lx, esr=0x%lx",
721 __func__, exception, frame->tf_elr, esr);
722
723 skip_userret = false;
724 switch (exception) {
725 case EXCP_FP_SIMD:
726 #ifdef VFP
727 vfp_restore_state();
728 #else
729 panic("VFP exception in userland");
730 #endif
731 break;
732 case EXCP_TRAP_FP:
733 #ifdef VFP
734 fpe_trap(td, (void *)frame->tf_elr, esr);
735 #else
736 panic("VFP exception in userland");
737 #endif
738 break;
739 case EXCP_SVE:
740 /* Returns true if this thread can use SVE */
741 if (!sve_restore_state(td))
742 call_trapsignal(td, SIGILL, ILL_ILLTRP,
743 (void *)frame->tf_elr, exception);
744 break;
745 case EXCP_SVC32:
746 case EXCP_SVC64:
747 skip_userret = svc_handler(td, frame);
748 break;
749 case EXCP_INSN_ABORT_L:
750 case EXCP_DATA_ABORT_L:
751 case EXCP_DATA_ABORT:
752 dfsc = esr & ISS_DATA_DFSC_MASK;
753 if (dfsc < nitems(abort_handlers) &&
754 abort_handlers[dfsc] != NULL)
755 abort_handlers[dfsc](td, frame, esr, far, 1);
756 else {
757 print_registers(frame);
758 print_gp_register("far", far);
759 printf(" esr: 0x%.16lx\n", esr);
760 panic("Unhandled EL0 %s abort: 0x%x",
761 exception == EXCP_INSN_ABORT_L ? "instruction" :
762 "data", dfsc);
763 }
764 break;
765 case EXCP_UNKNOWN:
766 if (!undef_insn(frame))
767 call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far,
768 exception);
769 break;
770 case EXCP_FPAC:
771 call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr,
772 exception);
773 break;
774 case EXCP_SP_ALIGN:
775 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp,
776 exception);
777 break;
778 case EXCP_PC_ALIGN:
779 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
780 exception);
781 break;
782 case EXCP_BRKPT_EL0:
783 case EXCP_BRK:
784 #ifdef COMPAT_FREEBSD32
785 case EXCP_BRKPT_32:
786 #endif /* COMPAT_FREEBSD32 */
787 call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr,
788 exception);
789 break;
790 case EXCP_WATCHPT_EL0:
791 call_trapsignal(td, SIGTRAP, TRAP_TRACE, (void *)far,
792 exception);
793 break;
794 case EXCP_MSR:
795 /*
796 * The CPU can raise EXCP_MSR when userspace executes an mrs
797 * instruction to access a special register userspace doesn't
798 * have access to.
799 */
800 if (!undef_insn(frame))
801 call_trapsignal(td, SIGILL, ILL_PRVOPC,
802 (void *)frame->tf_elr, exception);
803 break;
804 case EXCP_SOFTSTP_EL0:
805 PROC_LOCK(td->td_proc);
806 if ((td->td_dbgflags & TDB_STEP) != 0) {
807 td->td_frame->tf_spsr &= ~PSR_SS;
808 td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
809 WRITE_SPECIALREG(mdscr_el1,
810 READ_SPECIALREG(mdscr_el1) & ~MDSCR_SS);
811 }
812 PROC_UNLOCK(td->td_proc);
813 call_trapsignal(td, SIGTRAP, TRAP_TRACE,
814 (void *)frame->tf_elr, exception);
815 break;
816 case EXCP_BTI:
817 call_trapsignal(td, SIGILL, ILL_ILLOPC, (void *)frame->tf_elr,
818 exception);
819 break;
820 case EXCP_MOE:
821 handle_moe(td, frame, esr);
822 break;
823 default:
824 call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr,
825 exception);
826 break;
827 }
828
829 if (!skip_userret)
830 userret(td, frame);
831 KASSERT(
832 (td->td_pcb->pcb_fpflags & ~(PCB_FP_USERMASK|PCB_FP_SVEVALID)) == 0,
833 ("Kernel VFP flags set while entering userspace"));
834 KASSERT(
835 td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate,
836 ("Kernel VFP state in use when entering userspace"));
837 }
838
839 /*
840 * TODO: We will need to handle these later when we support ARMv8.2 RAS.
841 */
842 void
do_serror(struct trapframe * frame)843 do_serror(struct trapframe *frame)
844 {
845 uint64_t esr, far;
846
847 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
848 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
849
850 far = frame->tf_far;
851 esr = frame->tf_esr;
852
853 print_registers(frame);
854 print_gp_register("far", far);
855 printf(" esr: 0x%.16lx\n", esr);
856 panic("Unhandled System Error");
857 }
858
859 void
unhandled_exception(struct trapframe * frame)860 unhandled_exception(struct trapframe *frame)
861 {
862 uint64_t esr, far;
863
864 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
865 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
866
867 far = frame->tf_far;
868 esr = frame->tf_esr;
869
870 print_registers(frame);
871 print_gp_register("far", far);
872 printf(" esr: 0x%.16lx\n", esr);
873 panic("Unhandled exception");
874 }
875