1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28 #include "opt_ddb.h"
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/asan.h>
33 #include <sys/kernel.h>
34 #include <sys/ktr.h>
35 #include <sys/lock.h>
36 #include <sys/msan.h>
37 #include <sys/mutex.h>
38 #include <sys/proc.h>
39 #include <sys/ptrace.h>
40 #include <sys/syscall.h>
41 #include <sys/sysent.h>
42 #ifdef KDB
43 #include <sys/kdb.h>
44 #endif
45
46 #include <vm/vm.h>
47 #include <vm/pmap.h>
48 #include <vm/vm_kern.h>
49 #include <vm/vm_map.h>
50 #include <vm/vm_param.h>
51 #include <vm/vm_extern.h>
52
53 #include <machine/frame.h>
54 #include <machine/md_var.h>
55 #include <machine/pcb.h>
56 #include <machine/pcpu.h>
57 #include <machine/undefined.h>
58
59 #ifdef KDTRACE_HOOKS
60 #include <sys/dtrace_bsd.h>
61 #endif
62
63 #ifdef VFP
64 #include <machine/vfp.h>
65 #endif
66
67 #ifdef KDB
68 #include <machine/db_machdep.h>
69 #endif
70
71 #ifdef DDB
72 #include <ddb/ddb.h>
73 #include <ddb/db_sym.h>
74 #endif
75
76 /* Called from exception.S */
77 void do_el1h_sync(struct thread *, struct trapframe *);
78 void do_el0_sync(struct thread *, struct trapframe *);
79 void do_el0_error(struct trapframe *);
80 void do_serror(struct trapframe *);
81 void unhandled_exception(struct trapframe *);
82
83 static void print_gp_register(const char *name, uint64_t value);
84 static void print_registers(struct trapframe *frame);
85
86 int (*dtrace_invop_jump_addr)(struct trapframe *);
87
88 u_long cnt_efirt_faults;
89 int print_efirt_faults;
90
91 typedef void (abort_handler)(struct thread *, struct trapframe *, uint64_t,
92 uint64_t, int);
93
94 static abort_handler align_abort;
95 static abort_handler data_abort;
96 static abort_handler external_abort;
97
98 static abort_handler *abort_handlers[] = {
99 [ISS_DATA_DFSC_TF_L0] = data_abort,
100 [ISS_DATA_DFSC_TF_L1] = data_abort,
101 [ISS_DATA_DFSC_TF_L2] = data_abort,
102 [ISS_DATA_DFSC_TF_L3] = data_abort,
103 [ISS_DATA_DFSC_AFF_L1] = data_abort,
104 [ISS_DATA_DFSC_AFF_L2] = data_abort,
105 [ISS_DATA_DFSC_AFF_L3] = data_abort,
106 [ISS_DATA_DFSC_PF_L1] = data_abort,
107 [ISS_DATA_DFSC_PF_L2] = data_abort,
108 [ISS_DATA_DFSC_PF_L3] = data_abort,
109 [ISS_DATA_DFSC_ALIGN] = align_abort,
110 [ISS_DATA_DFSC_EXT] = external_abort,
111 [ISS_DATA_DFSC_EXT_L0] = external_abort,
112 [ISS_DATA_DFSC_EXT_L1] = external_abort,
113 [ISS_DATA_DFSC_EXT_L2] = external_abort,
114 [ISS_DATA_DFSC_EXT_L3] = external_abort,
115 [ISS_DATA_DFSC_ECC] = external_abort,
116 [ISS_DATA_DFSC_ECC_L0] = external_abort,
117 [ISS_DATA_DFSC_ECC_L1] = external_abort,
118 [ISS_DATA_DFSC_ECC_L2] = external_abort,
119 [ISS_DATA_DFSC_ECC_L3] = external_abort,
120 };
121
122 static __inline void
call_trapsignal(struct thread * td,int sig,int code,void * addr,int trapno)123 call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno)
124 {
125 ksiginfo_t ksi;
126
127 ksiginfo_init_trap(&ksi);
128 ksi.ksi_signo = sig;
129 ksi.ksi_code = code;
130 ksi.ksi_addr = addr;
131 ksi.ksi_trapno = trapno;
132 trapsignal(td, &ksi);
133 }
134
135 int
cpu_fetch_syscall_args(struct thread * td)136 cpu_fetch_syscall_args(struct thread *td)
137 {
138 struct proc *p;
139 syscallarg_t *ap, *dst_ap;
140 struct syscall_args *sa;
141
142 p = td->td_proc;
143 sa = &td->td_sa;
144 ap = td->td_frame->tf_x;
145 dst_ap = &sa->args[0];
146
147 sa->code = td->td_frame->tf_x[8];
148 sa->original_code = sa->code;
149
150 if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) {
151 sa->code = *ap++;
152 } else {
153 *dst_ap++ = *ap++;
154 }
155
156 if (__predict_false(sa->code >= p->p_sysent->sv_size))
157 sa->callp = &nosys_sysent;
158 else
159 sa->callp = &p->p_sysent->sv_table[sa->code];
160
161 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
162 ("Syscall %d takes too many arguments", sa->code));
163
164 memcpy(dst_ap, ap, (nitems(sa->args) - 1) * sizeof(*dst_ap));
165
166 td->td_retval[0] = 0;
167 td->td_retval[1] = 0;
168
169 return (0);
170 }
171
172 #include "../../kern/subr_syscall.c"
173
174 /*
175 * Test for fault generated by given access instruction in
176 * bus_peek_<foo> or bus_poke_<foo> bus function.
177 */
178 extern uint32_t generic_bs_peek_1f, generic_bs_peek_2f;
179 extern uint32_t generic_bs_peek_4f, generic_bs_peek_8f;
180 extern uint32_t generic_bs_poke_1f, generic_bs_poke_2f;
181 extern uint32_t generic_bs_poke_4f, generic_bs_poke_8f;
182
183 static bool
test_bs_fault(void * addr)184 test_bs_fault(void *addr)
185 {
186 return (addr == &generic_bs_peek_1f ||
187 addr == &generic_bs_peek_2f ||
188 addr == &generic_bs_peek_4f ||
189 addr == &generic_bs_peek_8f ||
190 addr == &generic_bs_poke_1f ||
191 addr == &generic_bs_poke_2f ||
192 addr == &generic_bs_poke_4f ||
193 addr == &generic_bs_poke_8f);
194 }
195
196 static void
svc_handler(struct thread * td,struct trapframe * frame)197 svc_handler(struct thread *td, struct trapframe *frame)
198 {
199
200 if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) {
201 syscallenter(td);
202 syscallret(td);
203 } else {
204 call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr,
205 ESR_ELx_EXCEPTION(frame->tf_esr));
206 userret(td, frame);
207 }
208 }
209
210 static void
align_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)211 align_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
212 uint64_t far, int lower)
213 {
214 if (!lower) {
215 print_registers(frame);
216 print_gp_register("far", far);
217 printf(" esr: 0x%.16lx\n", esr);
218 panic("Misaligned access from kernel space!");
219 }
220
221 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
222 ESR_ELx_EXCEPTION(frame->tf_esr));
223 userret(td, frame);
224 }
225
226
227 static void
external_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)228 external_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
229 uint64_t far, int lower)
230 {
231 if (lower) {
232 call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)far,
233 ESR_ELx_EXCEPTION(frame->tf_esr));
234 userret(td, frame);
235 return;
236 }
237
238 /*
239 * Try to handle synchronous external aborts caused by
240 * bus_space_peek() and/or bus_space_poke() functions.
241 */
242 if (test_bs_fault((void *)frame->tf_elr)) {
243 frame->tf_elr = (uint64_t)generic_bs_fault;
244 return;
245 }
246
247 print_registers(frame);
248 print_gp_register("far", far);
249 printf(" esr: 0x%.16lx\n", esr);
250 panic("Unhandled external data abort");
251 }
252
253 /*
254 * It is unsafe to access the stack canary value stored in "td" until
255 * kernel map translation faults are handled, see the pmap_klookup() call below.
256 * Thus, stack-smashing detection with per-thread canaries must be disabled in
257 * this function.
258 */
259 static void NO_PERTHREAD_SSP
data_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)260 data_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
261 uint64_t far, int lower)
262 {
263 struct vm_map *map;
264 struct pcb *pcb;
265 vm_prot_t ftype;
266 int error, sig, ucode;
267 #ifdef KDB
268 bool handled;
269 #endif
270
271 /*
272 * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive
273 * and Store-Exclusive instruction usage restrictions", state
274 * of the exclusive monitors after data abort exception is unknown.
275 */
276 clrex();
277
278 #ifdef KDB
279 if (kdb_active) {
280 kdb_reenter();
281 return;
282 }
283 #endif
284
285 if (lower) {
286 map = &td->td_proc->p_vmspace->vm_map;
287 } else if (!ADDR_IS_CANONICAL(far)) {
288 /* We received a TBI/PAC/etc. fault from the kernel */
289 error = KERN_INVALID_ADDRESS;
290 pcb = td->td_pcb;
291 goto bad_far;
292 } else if (ADDR_IS_KERNEL(far)) {
293 /*
294 * Handle a special case: the data abort was caused by accessing
295 * a thread structure while its mapping was being promoted or
296 * demoted, as a consequence of the break-before-make rule. It
297 * is not safe to enable interrupts or dereference "td" before
298 * this case is handled.
299 *
300 * In principle, if pmap_klookup() fails, there is no need to
301 * call pmap_fault() below, but avoiding that call is not worth
302 * the effort.
303 */
304 if (ESR_ELx_EXCEPTION(esr) == EXCP_DATA_ABORT) {
305 switch (esr & ISS_DATA_DFSC_MASK) {
306 case ISS_DATA_DFSC_TF_L0:
307 case ISS_DATA_DFSC_TF_L1:
308 case ISS_DATA_DFSC_TF_L2:
309 case ISS_DATA_DFSC_TF_L3:
310 if (pmap_klookup(far, NULL))
311 return;
312 break;
313 }
314 }
315 if (td->td_md.md_spinlock_count == 0 &&
316 (frame->tf_spsr & PSR_DAIF_INTR) != PSR_DAIF_INTR) {
317 MPASS((frame->tf_spsr & PSR_DAIF_INTR) == 0);
318 intr_enable();
319 }
320 map = kernel_map;
321 } else {
322 if (td->td_md.md_spinlock_count == 0 &&
323 (frame->tf_spsr & PSR_DAIF_INTR) != PSR_DAIF_INTR) {
324 MPASS((frame->tf_spsr & PSR_DAIF_INTR) == 0);
325 intr_enable();
326 }
327 map = &td->td_proc->p_vmspace->vm_map;
328 if (map == NULL)
329 map = kernel_map;
330 }
331 pcb = td->td_pcb;
332
333 /*
334 * Try to handle translation, access flag, and permission faults.
335 * Translation faults may occur as a result of the required
336 * break-before-make sequence used when promoting or demoting
337 * superpages. Such faults must not occur while holding the pmap lock,
338 * or pmap_fault() will recurse on that lock.
339 */
340 if ((lower || map == kernel_map || pcb->pcb_onfault != 0) &&
341 pmap_fault(map->pmap, esr, far) == KERN_SUCCESS)
342 return;
343
344 #ifdef INVARIANTS
345 if (td->td_md.md_spinlock_count != 0) {
346 print_registers(frame);
347 print_gp_register("far", far);
348 printf(" esr: 0x%.16lx\n", esr);
349 panic("data abort with spinlock held (spinlock count %d != 0)",
350 td->td_md.md_spinlock_count);
351 }
352 #endif
353 if ((td->td_pflags & TDP_NOFAULTING) == 0 &&
354 (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
355 WARN_GIANTOK, NULL, "Kernel page fault") != 0)) {
356 print_registers(frame);
357 print_gp_register("far", far);
358 printf(" esr: 0x%.16lx\n", esr);
359 panic("data abort in critical section or under mutex");
360 }
361
362 switch (ESR_ELx_EXCEPTION(esr)) {
363 case EXCP_INSN_ABORT:
364 case EXCP_INSN_ABORT_L:
365 ftype = VM_PROT_EXECUTE;
366 break;
367 default:
368 /*
369 * If the exception was because of a read or cache operation
370 * pass a read fault type into the vm code. Cache operations
371 * need read permission but will set the WnR flag when the
372 * memory is unmapped.
373 */
374 if ((esr & ISS_DATA_WnR) == 0 || (esr & ISS_DATA_CM) != 0)
375 ftype = VM_PROT_READ;
376 else
377 ftype = VM_PROT_WRITE;
378 break;
379 }
380
381 /* Fault in the page. */
382 error = vm_fault_trap(map, far, ftype, VM_FAULT_NORMAL, &sig, &ucode);
383 if (error != KERN_SUCCESS) {
384 if (lower) {
385 call_trapsignal(td, sig, ucode, (void *)far,
386 ESR_ELx_EXCEPTION(esr));
387 } else {
388 bad_far:
389 if (td->td_intr_nesting_level == 0 &&
390 pcb->pcb_onfault != 0) {
391 frame->tf_elr = pcb->pcb_onfault;
392 return;
393 }
394
395 printf("Fatal data abort:\n");
396 print_registers(frame);
397 print_gp_register("far", far);
398 printf(" esr: 0x%.16lx\n", esr);
399
400 #ifdef KDB
401 if (debugger_on_trap) {
402 kdb_why = KDB_WHY_TRAP;
403 handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0,
404 frame);
405 kdb_why = KDB_WHY_UNSET;
406 if (handled)
407 return;
408 }
409 #endif
410 panic("vm_fault failed: 0x%lx error %d",
411 frame->tf_elr, error);
412 }
413 }
414
415 if (lower)
416 userret(td, frame);
417 }
418
419 static void
print_gp_register(const char * name,uint64_t value)420 print_gp_register(const char *name, uint64_t value)
421 {
422 #if defined(DDB)
423 c_db_sym_t sym;
424 const char *sym_name;
425 db_expr_t sym_value;
426 db_expr_t offset;
427 #endif
428
429 printf(" %s: 0x%.16lx", name, value);
430 #if defined(DDB)
431 /* If this looks like a kernel address try to find the symbol */
432 if (value >= VM_MIN_KERNEL_ADDRESS) {
433 sym = db_search_symbol(value, DB_STGY_ANY, &offset);
434 if (sym != C_DB_SYM_NULL) {
435 db_symbol_values(sym, &sym_name, &sym_value);
436 printf(" (%s + 0x%lx)", sym_name, offset);
437 }
438 }
439 #endif
440 printf("\n");
441 }
442
443 static void
print_registers(struct trapframe * frame)444 print_registers(struct trapframe *frame)
445 {
446 char name[4];
447 u_int reg;
448
449 for (reg = 0; reg < nitems(frame->tf_x); reg++) {
450 snprintf(name, sizeof(name), "%sx%d", (reg < 10) ? " " : "",
451 reg);
452 print_gp_register(name, frame->tf_x[reg]);
453 }
454 printf(" sp: 0x%.16lx\n", frame->tf_sp);
455 print_gp_register(" lr", frame->tf_lr);
456 print_gp_register("elr", frame->tf_elr);
457 printf("spsr: 0x%.16lx\n", frame->tf_spsr);
458 }
459
460 #ifdef VFP
461 static void
fpe_trap(struct thread * td,void * addr,uint32_t exception)462 fpe_trap(struct thread *td, void *addr, uint32_t exception)
463 {
464 int code;
465
466 code = FPE_FLTIDO;
467 if ((exception & ISS_FP_TFV) != 0) {
468 if ((exception & ISS_FP_IOF) != 0)
469 code = FPE_FLTINV;
470 else if ((exception & ISS_FP_DZF) != 0)
471 code = FPE_FLTDIV;
472 else if ((exception & ISS_FP_OFF) != 0)
473 code = FPE_FLTOVF;
474 else if ((exception & ISS_FP_UFF) != 0)
475 code = FPE_FLTUND;
476 else if ((exception & ISS_FP_IXF) != 0)
477 code = FPE_FLTRES;
478 }
479 call_trapsignal(td, SIGFPE, code, addr, exception);
480 }
481 #endif
482
483 /*
484 * See the comment above data_abort().
485 */
486 void NO_PERTHREAD_SSP
do_el1h_sync(struct thread * td,struct trapframe * frame)487 do_el1h_sync(struct thread *td, struct trapframe *frame)
488 {
489 uint32_t exception;
490 uint64_t esr, far;
491 int dfsc;
492
493 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
494 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
495
496 far = frame->tf_far;
497 /* Read the esr register to get the exception details */
498 esr = frame->tf_esr;
499 exception = ESR_ELx_EXCEPTION(esr);
500
501 #ifdef KDTRACE_HOOKS
502 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
503 return;
504 #endif
505
506 CTR4(KTR_TRAP, "%s: exception=%lu, elr=0x%lx, esr=0x%lx",
507 __func__, exception, frame->tf_elr, esr);
508
509 /*
510 * Enable debug exceptions if we aren't already handling one. They will
511 * be masked again in the exception handler's epilogue.
512 */
513 switch (exception) {
514 case EXCP_BRK:
515 case EXCP_BRKPT_EL1:
516 case EXCP_WATCHPT_EL1:
517 case EXCP_SOFTSTP_EL1:
518 break;
519 default:
520 dbg_enable();
521 break;
522 }
523
524 switch (exception) {
525 case EXCP_FP_SIMD:
526 case EXCP_TRAP_FP:
527 #ifdef VFP
528 if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) {
529 vfp_restore_state();
530 } else
531 #endif
532 {
533 print_registers(frame);
534 printf(" esr: 0x%.16lx\n", esr);
535 panic("VFP exception in the kernel");
536 }
537 break;
538 case EXCP_INSN_ABORT:
539 case EXCP_DATA_ABORT:
540 dfsc = esr & ISS_DATA_DFSC_MASK;
541 if (dfsc < nitems(abort_handlers) &&
542 abort_handlers[dfsc] != NULL) {
543 abort_handlers[dfsc](td, frame, esr, far, 0);
544 } else {
545 print_registers(frame);
546 print_gp_register("far", far);
547 printf(" esr: 0x%.16lx\n", esr);
548 panic("Unhandled EL1 %s abort: 0x%x",
549 exception == EXCP_INSN_ABORT ? "instruction" :
550 "data", dfsc);
551 }
552 break;
553 case EXCP_BRK:
554 #ifdef KDTRACE_HOOKS
555 if ((esr & ESR_ELx_ISS_MASK) == 0x40d /* BRK_IMM16_VAL */ &&
556 dtrace_invop_jump_addr != NULL &&
557 dtrace_invop_jump_addr(frame) == 0)
558 break;
559 #endif
560 #ifdef KDB
561 kdb_trap(exception, 0, frame);
562 #else
563 panic("No debugger in kernel.");
564 #endif
565 break;
566 case EXCP_BRKPT_EL1:
567 case EXCP_WATCHPT_EL1:
568 case EXCP_SOFTSTP_EL1:
569 #ifdef KDB
570 kdb_trap(exception, 0, frame);
571 #else
572 panic("No debugger in kernel.");
573 #endif
574 break;
575 case EXCP_FPAC:
576 /* We can see this if the authentication on PAC fails */
577 print_registers(frame);
578 print_gp_register("far", far);
579 panic("FPAC kernel exception");
580 break;
581 case EXCP_UNKNOWN:
582 print_registers(frame);
583 print_gp_register("far", far);
584 panic("Undefined instruction: %08x",
585 *(uint32_t *)frame->tf_elr);
586 break;
587 case EXCP_BTI:
588 print_registers(frame);
589 print_gp_register("far", far);
590 panic("Branch Target exception");
591 break;
592 default:
593 print_registers(frame);
594 print_gp_register("far", far);
595 panic("Unknown kernel exception 0x%x esr_el1 0x%lx", exception,
596 esr);
597 }
598 }
599
600 void
do_el0_sync(struct thread * td,struct trapframe * frame)601 do_el0_sync(struct thread *td, struct trapframe *frame)
602 {
603 pcpu_bp_harden bp_harden;
604 uint32_t exception;
605 uint64_t esr, far;
606 int dfsc;
607
608 /* Check we have a sane environment when entering from userland */
609 KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
610 ("Invalid pcpu address from userland: %p (tpidr 0x%lx)",
611 get_pcpu(), READ_SPECIALREG(tpidr_el1)));
612
613 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
614 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
615
616 far = frame->tf_far;
617 esr = frame->tf_esr;
618 exception = ESR_ELx_EXCEPTION(esr);
619 if (exception == EXCP_INSN_ABORT_L && far > VM_MAXUSER_ADDRESS) {
620 /*
621 * Userspace may be trying to train the branch predictor to
622 * attack the kernel. If we are on a CPU affected by this
623 * call the handler to clear the branch predictor state.
624 */
625 bp_harden = PCPU_GET(bp_harden);
626 if (bp_harden != NULL)
627 bp_harden();
628 }
629 intr_enable();
630
631 CTR4(KTR_TRAP, "%s: exception=%lu, elr=0x%lx, esr=0x%lx",
632 __func__, exception, frame->tf_elr, esr);
633
634 switch (exception) {
635 case EXCP_FP_SIMD:
636 #ifdef VFP
637 vfp_restore_state();
638 #else
639 panic("VFP exception in userland");
640 #endif
641 break;
642 case EXCP_TRAP_FP:
643 #ifdef VFP
644 fpe_trap(td, (void *)frame->tf_elr, esr);
645 userret(td, frame);
646 #else
647 panic("VFP exception in userland");
648 #endif
649 break;
650 case EXCP_SVE:
651 /* Returns true if this thread can use SVE */
652 if (!sve_restore_state(td))
653 call_trapsignal(td, SIGILL, ILL_ILLTRP,
654 (void *)frame->tf_elr, exception);
655 userret(td, frame);
656 break;
657 case EXCP_SVC32:
658 case EXCP_SVC64:
659 svc_handler(td, frame);
660 break;
661 case EXCP_INSN_ABORT_L:
662 case EXCP_DATA_ABORT_L:
663 case EXCP_DATA_ABORT:
664 dfsc = esr & ISS_DATA_DFSC_MASK;
665 if (dfsc < nitems(abort_handlers) &&
666 abort_handlers[dfsc] != NULL)
667 abort_handlers[dfsc](td, frame, esr, far, 1);
668 else {
669 print_registers(frame);
670 print_gp_register("far", far);
671 printf(" esr: 0x%.16lx\n", esr);
672 panic("Unhandled EL0 %s abort: 0x%x",
673 exception == EXCP_INSN_ABORT_L ? "instruction" :
674 "data", dfsc);
675 }
676 break;
677 case EXCP_UNKNOWN:
678 if (!undef_insn(frame))
679 call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far,
680 exception);
681 userret(td, frame);
682 break;
683 case EXCP_FPAC:
684 call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr,
685 exception);
686 userret(td, frame);
687 break;
688 case EXCP_SP_ALIGN:
689 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp,
690 exception);
691 userret(td, frame);
692 break;
693 case EXCP_PC_ALIGN:
694 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
695 exception);
696 userret(td, frame);
697 break;
698 case EXCP_BRKPT_EL0:
699 case EXCP_BRK:
700 #ifdef COMPAT_FREEBSD32
701 case EXCP_BRKPT_32:
702 #endif /* COMPAT_FREEBSD32 */
703 call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr,
704 exception);
705 userret(td, frame);
706 break;
707 case EXCP_WATCHPT_EL0:
708 call_trapsignal(td, SIGTRAP, TRAP_TRACE, (void *)far,
709 exception);
710 userret(td, frame);
711 break;
712 case EXCP_MSR:
713 /*
714 * The CPU can raise EXCP_MSR when userspace executes an mrs
715 * instruction to access a special register userspace doesn't
716 * have access to.
717 */
718 if (!undef_insn(frame))
719 call_trapsignal(td, SIGILL, ILL_PRVOPC,
720 (void *)frame->tf_elr, exception);
721 userret(td, frame);
722 break;
723 case EXCP_SOFTSTP_EL0:
724 PROC_LOCK(td->td_proc);
725 if ((td->td_dbgflags & TDB_STEP) != 0) {
726 td->td_frame->tf_spsr &= ~PSR_SS;
727 td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
728 WRITE_SPECIALREG(mdscr_el1,
729 READ_SPECIALREG(mdscr_el1) & ~MDSCR_SS);
730 }
731 PROC_UNLOCK(td->td_proc);
732 call_trapsignal(td, SIGTRAP, TRAP_TRACE,
733 (void *)frame->tf_elr, exception);
734 userret(td, frame);
735 break;
736 case EXCP_BTI:
737 call_trapsignal(td, SIGILL, ILL_ILLOPC, (void *)frame->tf_elr,
738 exception);
739 userret(td, frame);
740 break;
741 default:
742 call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr,
743 exception);
744 userret(td, frame);
745 break;
746 }
747
748 KASSERT(
749 (td->td_pcb->pcb_fpflags & ~(PCB_FP_USERMASK|PCB_FP_SVEVALID)) == 0,
750 ("Kernel VFP flags set while entering userspace"));
751 KASSERT(
752 td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate,
753 ("Kernel VFP state in use when entering userspace"));
754 }
755
756 /*
757 * TODO: We will need to handle these later when we support ARMv8.2 RAS.
758 */
759 void
do_serror(struct trapframe * frame)760 do_serror(struct trapframe *frame)
761 {
762 uint64_t esr, far;
763
764 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
765 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
766
767 far = frame->tf_far;
768 esr = frame->tf_esr;
769
770 print_registers(frame);
771 print_gp_register("far", far);
772 printf(" esr: 0x%.16lx\n", esr);
773 panic("Unhandled System Error");
774 }
775
776 void
unhandled_exception(struct trapframe * frame)777 unhandled_exception(struct trapframe *frame)
778 {
779 uint64_t esr, far;
780
781 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
782 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
783
784 far = frame->tf_far;
785 esr = frame->tf_esr;
786
787 print_registers(frame);
788 print_gp_register("far", far);
789 printf(" esr: 0x%.16lx\n", esr);
790 panic("Unhandled exception");
791 }
792