1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28 #include "opt_ddb.h"
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/asan.h>
33 #include <sys/kernel.h>
34 #include <sys/ktr.h>
35 #include <sys/lock.h>
36 #include <sys/msan.h>
37 #include <sys/mutex.h>
38 #include <sys/proc.h>
39 #include <sys/ptrace.h>
40 #include <sys/syscall.h>
41 #include <sys/sysent.h>
42 #ifdef KDB
43 #include <sys/kdb.h>
44 #endif
45
46 #include <vm/vm.h>
47 #include <vm/pmap.h>
48 #include <vm/vm_kern.h>
49 #include <vm/vm_map.h>
50 #include <vm/vm_param.h>
51 #include <vm/vm_extern.h>
52
53 #include <machine/frame.h>
54 #include <machine/md_var.h>
55 #include <machine/pcb.h>
56 #include <machine/pcpu.h>
57 #include <machine/undefined.h>
58
59 #ifdef KDTRACE_HOOKS
60 #include <sys/dtrace_bsd.h>
61 #endif
62
63 #ifdef VFP
64 #include <machine/vfp.h>
65 #endif
66
67 #ifdef KDB
68 #include <machine/db_machdep.h>
69 #endif
70
71 #ifdef DDB
72 #include <ddb/ddb.h>
73 #include <ddb/db_sym.h>
74 #endif
75
76 /* Called from exception.S */
77 void do_el1h_sync(struct thread *, struct trapframe *);
78 void do_el0_sync(struct thread *, struct trapframe *);
79 void do_el0_error(struct trapframe *);
80 void do_serror(struct trapframe *);
81 void unhandled_exception(struct trapframe *);
82
83 static void print_gp_register(const char *name, uint64_t value);
84 static void print_registers(struct trapframe *frame);
85
86 int (*dtrace_invop_jump_addr)(struct trapframe *);
87
88 u_long cnt_efirt_faults;
89 int print_efirt_faults;
90
91 typedef void (abort_handler)(struct thread *, struct trapframe *, uint64_t,
92 uint64_t, int);
93
94 static abort_handler align_abort;
95 static abort_handler data_abort;
96 static abort_handler external_abort;
97 static abort_handler tag_check_abort;
98
99 static abort_handler *abort_handlers[] = {
100 [ISS_DATA_DFSC_TF_L0] = data_abort,
101 [ISS_DATA_DFSC_TF_L1] = data_abort,
102 [ISS_DATA_DFSC_TF_L2] = data_abort,
103 [ISS_DATA_DFSC_TF_L3] = data_abort,
104 [ISS_DATA_DFSC_AFF_L1] = data_abort,
105 [ISS_DATA_DFSC_AFF_L2] = data_abort,
106 [ISS_DATA_DFSC_AFF_L3] = data_abort,
107 [ISS_DATA_DFSC_PF_L1] = data_abort,
108 [ISS_DATA_DFSC_PF_L2] = data_abort,
109 [ISS_DATA_DFSC_PF_L3] = data_abort,
110 [ISS_DATA_DFSC_TAG] = tag_check_abort,
111 [ISS_DATA_DFSC_ALIGN] = align_abort,
112 [ISS_DATA_DFSC_EXT] = external_abort,
113 [ISS_DATA_DFSC_EXT_L0] = external_abort,
114 [ISS_DATA_DFSC_EXT_L1] = external_abort,
115 [ISS_DATA_DFSC_EXT_L2] = external_abort,
116 [ISS_DATA_DFSC_EXT_L3] = external_abort,
117 [ISS_DATA_DFSC_ECC] = external_abort,
118 [ISS_DATA_DFSC_ECC_L0] = external_abort,
119 [ISS_DATA_DFSC_ECC_L1] = external_abort,
120 [ISS_DATA_DFSC_ECC_L2] = external_abort,
121 [ISS_DATA_DFSC_ECC_L3] = external_abort,
122 };
123
124 static __inline void
call_trapsignal(struct thread * td,int sig,int code,void * addr,int trapno)125 call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno)
126 {
127 ksiginfo_t ksi;
128
129 ksiginfo_init_trap(&ksi);
130 ksi.ksi_signo = sig;
131 ksi.ksi_code = code;
132 ksi.ksi_addr = addr;
133 ksi.ksi_trapno = trapno;
134 ksi.ksi_flags |= KSI_EXCEPT;
135 trapsignal(td, &ksi);
136 }
137
138 int
cpu_fetch_syscall_args(struct thread * td)139 cpu_fetch_syscall_args(struct thread *td)
140 {
141 struct proc *p;
142 syscallarg_t *ap, *dst_ap;
143 struct syscall_args *sa;
144
145 p = td->td_proc;
146 sa = &td->td_sa;
147 ap = td->td_frame->tf_x;
148 dst_ap = &sa->args[0];
149
150 sa->code = td->td_frame->tf_x[8];
151 sa->original_code = sa->code;
152
153 if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) {
154 sa->code = *ap++;
155 } else {
156 *dst_ap++ = *ap++;
157 }
158
159 if (__predict_false(sa->code >= p->p_sysent->sv_size))
160 sa->callp = &nosys_sysent;
161 else
162 sa->callp = &p->p_sysent->sv_table[sa->code];
163
164 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
165 ("Syscall %d takes too many arguments", sa->code));
166
167 memcpy(dst_ap, ap, (nitems(sa->args) - 1) * sizeof(*dst_ap));
168
169 td->td_retval[0] = 0;
170 td->td_retval[1] = 0;
171
172 return (0);
173 }
174
175 #include "../../kern/subr_syscall.c"
176
177 /*
178 * Test for fault generated by given access instruction in
179 * bus_peek_<foo> or bus_poke_<foo> bus function.
180 */
181 extern uint32_t generic_bs_peek_1f, generic_bs_peek_2f;
182 extern uint32_t generic_bs_peek_4f, generic_bs_peek_8f;
183 extern uint32_t generic_bs_poke_1f, generic_bs_poke_2f;
184 extern uint32_t generic_bs_poke_4f, generic_bs_poke_8f;
185
186 static bool
test_bs_fault(void * addr)187 test_bs_fault(void *addr)
188 {
189 return (addr == &generic_bs_peek_1f ||
190 addr == &generic_bs_peek_2f ||
191 addr == &generic_bs_peek_4f ||
192 addr == &generic_bs_peek_8f ||
193 addr == &generic_bs_poke_1f ||
194 addr == &generic_bs_poke_2f ||
195 addr == &generic_bs_poke_4f ||
196 addr == &generic_bs_poke_8f);
197 }
198
199 static bool
svc_handler(struct thread * td,struct trapframe * frame)200 svc_handler(struct thread *td, struct trapframe *frame)
201 {
202
203 if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) {
204 syscallenter(td);
205 syscallret(td);
206 /* Skip userret as syscallret already called it */
207 return (true);
208 } else {
209 call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr,
210 ESR_ELx_EXCEPTION(frame->tf_esr));
211 return (false);
212 }
213 }
214
215 static void
align_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)216 align_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
217 uint64_t far, int lower)
218 {
219 if (!lower) {
220 /*
221 * Accessing unaligned memory may fault when using atomics.
222 * Make sure we don't panic if we're doing an unaligned
223 * access to userland memory, as can happen with _umtx_op()
224 */
225 if (td->td_intr_nesting_level == 0 &&
226 td->td_pcb->pcb_onfault != 0) {
227 frame->tf_elr = td->td_pcb->pcb_onfault;
228 return;
229 }
230
231 print_registers(frame);
232 print_gp_register("far", far);
233 printf(" esr: 0x%.16lx\n", esr);
234 panic("Misaligned access from kernel space!");
235 }
236
237 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
238 ESR_ELx_EXCEPTION(frame->tf_esr));
239 }
240
241
242 static void
external_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)243 external_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
244 uint64_t far, int lower)
245 {
246 if (lower) {
247 call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)far,
248 ESR_ELx_EXCEPTION(frame->tf_esr));
249 return;
250 }
251
252 /*
253 * Try to handle synchronous external aborts caused by
254 * bus_space_peek() and/or bus_space_poke() functions.
255 */
256 if (test_bs_fault((void *)frame->tf_elr)) {
257 frame->tf_elr = (uint64_t)generic_bs_fault;
258 return;
259 }
260
261 print_registers(frame);
262 print_gp_register("far", far);
263 printf(" esr: 0x%.16lx\n", esr);
264 panic("Unhandled external data abort");
265 }
266
267 static void
tag_check_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)268 tag_check_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
269 uint64_t far, int lower)
270 {
271 /*
272 * A Tag Check Fault should be handled as a SIGSEGV if it occurs
273 * at EL0 and a kernel panic if at EL1.
274 */
275 if (!lower) {
276 /*
277 * If we have a fault handler, let it decide what to do.
278 */
279 if (td->td_intr_nesting_level == 0 &&
280 td->td_pcb->pcb_onfault != 0) {
281 frame->tf_elr = td->td_pcb->pcb_onfault;
282 return;
283 }
284 print_registers(frame);
285 print_gp_register("far", far);
286 printf(" esr: 0x%.16lx\n", esr);
287 panic("Tag Check Fault");
288 }
289
290 call_trapsignal(td, SIGSEGV, SEGV_MTESERR, (void *)far,
291 ESR_ELx_EXCEPTION(frame->tf_esr));
292 userret(td, frame);
293 }
294
295 /*
296 * It is unsafe to access the stack canary value stored in "td" until
297 * kernel map translation faults are handled, see the pmap_klookup() call below.
298 * Thus, stack-smashing detection with per-thread canaries must be disabled in
299 * this function.
300 */
301 static void NO_PERTHREAD_SSP
data_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)302 data_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
303 uint64_t far, int lower)
304 {
305 struct vm_map *map;
306 struct pcb *pcb;
307 vm_offset_t fault_va;
308 vm_prot_t ftype;
309 int error, sig, ucode;
310 #ifdef KDB
311 bool handled;
312 #endif
313
314 /*
315 * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive
316 * and Store-Exclusive instruction usage restrictions", state
317 * of the exclusive monitors after data abort exception is unknown.
318 */
319 clrex();
320
321 #ifdef KDB
322 if (kdb_active) {
323 kdb_reenter();
324 return;
325 }
326 #endif
327
328 fault_va = far;
329 if (lower) {
330 map = &td->td_proc->p_vmspace->vm_map;
331 if ((td->td_proc->p_md.md_tcr & TCR_TBI0) != 0)
332 fault_va = ADDR_MAKE_CANONICAL(far);
333 } else if (!ADDR_IS_CANONICAL(far)) {
334 /* We received a TBI/PAC/etc. fault from the kernel */
335 error = KERN_INVALID_ADDRESS;
336 pcb = td->td_pcb;
337 goto bad_far;
338 } else if (ADDR_IS_KERNEL(far)) {
339 /*
340 * Handle a special case: the data abort was caused by accessing
341 * a thread structure while its mapping was being promoted or
342 * demoted, as a consequence of the break-before-make rule. It
343 * is not safe to enable interrupts or dereference "td" before
344 * this case is handled.
345 *
346 * In principle, if pmap_klookup() fails, there is no need to
347 * call pmap_fault() below, but avoiding that call is not worth
348 * the effort.
349 */
350 if (ESR_ELx_EXCEPTION(esr) == EXCP_DATA_ABORT) {
351 switch (esr & ISS_DATA_DFSC_MASK) {
352 case ISS_DATA_DFSC_TF_L0:
353 case ISS_DATA_DFSC_TF_L1:
354 case ISS_DATA_DFSC_TF_L2:
355 case ISS_DATA_DFSC_TF_L3:
356 if (pmap_klookup(far, NULL))
357 return;
358 break;
359 }
360 }
361 if (td->td_md.md_spinlock_count == 0 &&
362 (frame->tf_spsr & PSR_DAIF_INTR) != PSR_DAIF_INTR) {
363 MPASS((frame->tf_spsr & PSR_DAIF_INTR) == 0);
364 intr_enable();
365 }
366 map = kernel_map;
367 } else {
368 if (td->td_md.md_spinlock_count == 0 &&
369 (frame->tf_spsr & PSR_DAIF_INTR) != PSR_DAIF_INTR) {
370 MPASS((frame->tf_spsr & PSR_DAIF_INTR) == 0);
371 intr_enable();
372 }
373 map = &td->td_proc->p_vmspace->vm_map;
374 if (map == NULL)
375 map = kernel_map;
376 }
377 pcb = td->td_pcb;
378
379 /*
380 * Try to handle translation, access flag, and permission faults.
381 * Translation faults may occur as a result of the required
382 * break-before-make sequence used when promoting or demoting
383 * superpages. Such faults must not occur while holding the pmap lock,
384 * or pmap_fault() will recurse on that lock.
385 */
386 if ((lower || map == kernel_map || pcb->pcb_onfault != 0) &&
387 pmap_fault(map->pmap, esr, fault_va) == KERN_SUCCESS)
388 return;
389
390 #ifdef INVARIANTS
391 if (td->td_md.md_spinlock_count != 0) {
392 print_registers(frame);
393 print_gp_register("far", far);
394 printf(" esr: 0x%.16lx\n", esr);
395 panic("data abort with spinlock held (spinlock count %d != 0)",
396 td->td_md.md_spinlock_count);
397 }
398 #endif
399 if ((td->td_pflags & TDP_NOFAULTING) == 0 &&
400 (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
401 WARN_GIANTOK, NULL, "Kernel page fault") != 0)) {
402 print_registers(frame);
403 print_gp_register("far", far);
404 printf(" esr: 0x%.16lx\n", esr);
405 panic("data abort in critical section or under mutex");
406 }
407
408 switch (ESR_ELx_EXCEPTION(esr)) {
409 case EXCP_INSN_ABORT:
410 case EXCP_INSN_ABORT_L:
411 ftype = VM_PROT_EXECUTE;
412 break;
413 default:
414 /*
415 * If the exception was because of a read or cache operation
416 * pass a read fault type into the vm code. Cache operations
417 * need read permission but will set the WnR flag when the
418 * memory is unmapped.
419 */
420 if ((esr & ISS_DATA_WnR) == 0 || (esr & ISS_DATA_CM) != 0)
421 ftype = VM_PROT_READ;
422 else
423 ftype = VM_PROT_WRITE;
424 break;
425 }
426
427 /* Fault in the page. */
428 error = vm_fault_trap(map, fault_va, ftype, VM_FAULT_NORMAL, &sig,
429 &ucode);
430 if (error != KERN_SUCCESS) {
431 if (lower) {
432 call_trapsignal(td, sig, ucode, (void *)far,
433 ESR_ELx_EXCEPTION(esr));
434 } else {
435 bad_far:
436 if (td->td_intr_nesting_level == 0 &&
437 pcb->pcb_onfault != 0) {
438 frame->tf_elr = pcb->pcb_onfault;
439 return;
440 }
441
442 printf("Fatal data abort:\n");
443 print_registers(frame);
444 print_gp_register("far", far);
445 printf(" esr: 0x%.16lx\n", esr);
446
447 #ifdef KDB
448 if (debugger_on_trap) {
449 kdb_why = KDB_WHY_TRAP;
450 handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0,
451 frame);
452 kdb_why = KDB_WHY_UNSET;
453 if (handled)
454 return;
455 }
456 #endif
457 panic("vm_fault failed: 0x%lx error %d",
458 frame->tf_elr, error);
459 }
460 }
461 }
462
463 static void
print_gp_register(const char * name,uint64_t value)464 print_gp_register(const char *name, uint64_t value)
465 {
466 #if defined(DDB)
467 c_db_sym_t sym;
468 const char *sym_name;
469 db_expr_t sym_value;
470 db_expr_t offset;
471 #endif
472
473 printf(" %s: 0x%.16lx", name, value);
474 #if defined(DDB)
475 /* If this looks like a kernel address try to find the symbol */
476 if (value >= VM_MIN_KERNEL_ADDRESS) {
477 sym = db_search_symbol(value, DB_STGY_ANY, &offset);
478 if (sym != C_DB_SYM_NULL) {
479 db_symbol_values(sym, &sym_name, &sym_value);
480 printf(" (%s + 0x%lx)", sym_name, offset);
481 }
482 }
483 #endif
484 printf("\n");
485 }
486
487 static void
print_registers(struct trapframe * frame)488 print_registers(struct trapframe *frame)
489 {
490 char name[4];
491 u_int reg;
492
493 for (reg = 0; reg < nitems(frame->tf_x); reg++) {
494 snprintf(name, sizeof(name), "%sx%d", (reg < 10) ? " " : "",
495 reg);
496 print_gp_register(name, frame->tf_x[reg]);
497 }
498 printf(" sp: 0x%.16lx\n", frame->tf_sp);
499 print_gp_register(" lr", frame->tf_lr);
500 print_gp_register("elr", frame->tf_elr);
501 printf("spsr: 0x%.16lx\n", frame->tf_spsr);
502 }
503
504 #ifdef VFP
505 static void
fpe_trap(struct thread * td,void * addr,uint32_t exception)506 fpe_trap(struct thread *td, void *addr, uint32_t exception)
507 {
508 int code;
509
510 code = FPE_FLTIDO;
511 if ((exception & ISS_FP_TFV) != 0) {
512 if ((exception & ISS_FP_IOF) != 0)
513 code = FPE_FLTINV;
514 else if ((exception & ISS_FP_DZF) != 0)
515 code = FPE_FLTDIV;
516 else if ((exception & ISS_FP_OFF) != 0)
517 code = FPE_FLTOVF;
518 else if ((exception & ISS_FP_UFF) != 0)
519 code = FPE_FLTUND;
520 else if ((exception & ISS_FP_IXF) != 0)
521 code = FPE_FLTRES;
522 }
523 call_trapsignal(td, SIGFPE, code, addr, exception);
524 }
525 #endif
526
527 static void
handle_moe(struct thread * td,struct trapframe * frame,uint64_t esr)528 handle_moe(struct thread *td, struct trapframe *frame, uint64_t esr)
529 {
530 uint64_t src;
531 uint64_t dest;
532 uint64_t size;
533 int src_reg;
534 int dest_reg;
535 int size_reg;
536 int format_option;
537
538 format_option = esr & ISS_MOE_FORMAT_OPTION_MASK;
539 dest_reg = (esr & ISS_MOE_DESTREG_MASK) >> ISS_MOE_DESTREG_SHIFT;
540 size_reg = (esr & ISS_MOE_SIZEREG_MASK) >> ISS_MOE_SIZEREG_SHIFT;
541 dest = frame->tf_x[dest_reg];
542 size = frame->tf_x[size_reg];
543
544 /*
545 * Put the registers back in the original format suitable for a
546 * prologue instruction, using the generic return routine from the
547 * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH.
548 */
549 if (esr & ISS_MOE_MEMINST) {
550 /* SET* instruction */
551 if (format_option == ISS_MOE_FORMAT_OPTION_A ||
552 format_option == ISS_MOE_FORMAT_OPTION_A2) {
553 /* Format is from Option A; forward set */
554 frame->tf_x[dest_reg] = dest + size;
555 frame->tf_x[size_reg] = -size;
556 }
557 } else {
558 /* CPY* instruction */
559 src_reg = (esr & ISS_MOE_SRCREG_MASK) >> ISS_MOE_SRCREG_SHIFT;
560 src = frame->tf_x[src_reg];
561
562 if (format_option == ISS_MOE_FORMAT_OPTION_B ||
563 format_option == ISS_MOE_FORMAT_OPTION_B2) {
564 /* Format is from Option B */
565 if (frame->tf_spsr & PSR_N) {
566 /* Backward copy */
567 frame->tf_x[dest_reg] = dest - size;
568 frame->tf_x[src_reg] = src + size;
569 }
570 } else {
571 /* Format is from Option A */
572 if (frame->tf_x[size_reg] & (1UL << 63)) {
573 /* Forward copy */
574 frame->tf_x[dest_reg] = dest + size;
575 frame->tf_x[src_reg] = src + size;
576 frame->tf_x[size_reg] = -size;
577 }
578 }
579 }
580
581 if (esr & ISS_MOE_FROM_EPILOGUE)
582 frame->tf_elr -= 8;
583 else
584 frame->tf_elr -= 4;
585 }
586
587 /*
588 * See the comment above data_abort().
589 */
590 void NO_PERTHREAD_SSP
do_el1h_sync(struct thread * td,struct trapframe * frame)591 do_el1h_sync(struct thread *td, struct trapframe *frame)
592 {
593 uint32_t exception;
594 uint64_t esr, far;
595 int dfsc;
596
597 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
598 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
599
600 far = frame->tf_far;
601 /* Read the esr register to get the exception details */
602 esr = frame->tf_esr;
603 exception = ESR_ELx_EXCEPTION(esr);
604
605 #ifdef KDTRACE_HOOKS
606 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
607 return;
608 #endif
609
610 CTR4(KTR_TRAP, "%s: exception=%lu, elr=0x%lx, esr=0x%lx",
611 __func__, exception, frame->tf_elr, esr);
612
613 /*
614 * Enable debug exceptions if we aren't already handling one. They will
615 * be masked again in the exception handler's epilogue.
616 */
617 switch (exception) {
618 case EXCP_BRK:
619 case EXCP_BRKPT_EL1:
620 case EXCP_WATCHPT_EL1:
621 case EXCP_SOFTSTP_EL1:
622 break;
623 default:
624 dbg_enable();
625 break;
626 }
627
628 switch (exception) {
629 case EXCP_FP_SIMD:
630 case EXCP_TRAP_FP:
631 #ifdef VFP
632 if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) {
633 vfp_restore_state();
634 } else
635 #endif
636 {
637 print_registers(frame);
638 printf(" esr: 0x%.16lx\n", esr);
639 panic("VFP exception in the kernel");
640 }
641 break;
642 case EXCP_INSN_ABORT:
643 case EXCP_DATA_ABORT:
644 dfsc = esr & ISS_DATA_DFSC_MASK;
645 if (dfsc < nitems(abort_handlers) &&
646 abort_handlers[dfsc] != NULL) {
647 abort_handlers[dfsc](td, frame, esr, far, 0);
648 } else {
649 print_registers(frame);
650 print_gp_register("far", far);
651 printf(" esr: 0x%.16lx\n", esr);
652 panic("Unhandled EL1 %s abort: 0x%x",
653 exception == EXCP_INSN_ABORT ? "instruction" :
654 "data", dfsc);
655 }
656 break;
657 case EXCP_BRK:
658 #ifdef KDTRACE_HOOKS
659 if ((esr & ESR_ELx_ISS_MASK) == 0x40d /* BRK_IMM16_VAL */ &&
660 dtrace_invop_jump_addr != NULL &&
661 dtrace_invop_jump_addr(frame) == 0)
662 break;
663 #endif
664 #ifdef KDB
665 kdb_trap(exception, 0, frame);
666 #else
667 panic("No debugger in kernel.");
668 #endif
669 break;
670 case EXCP_BRKPT_EL1:
671 case EXCP_WATCHPT_EL1:
672 case EXCP_SOFTSTP_EL1:
673 #ifdef KDB
674 kdb_trap(exception, 0, frame);
675 #else
676 panic("No debugger in kernel.");
677 #endif
678 break;
679 case EXCP_FPAC:
680 /* We can see this if the authentication on PAC fails */
681 print_registers(frame);
682 print_gp_register("far", far);
683 panic("FPAC kernel exception");
684 break;
685 case EXCP_UNKNOWN:
686 print_registers(frame);
687 print_gp_register("far", far);
688 panic("Undefined instruction: %08x",
689 *(uint32_t *)frame->tf_elr);
690 break;
691 case EXCP_BTI:
692 print_registers(frame);
693 print_gp_register("far", far);
694 panic("Branch Target exception");
695 break;
696 case EXCP_MOE:
697 handle_moe(td, frame, esr);
698 break;
699 default:
700 print_registers(frame);
701 print_gp_register("far", far);
702 panic("Unknown kernel exception 0x%x esr_el1 0x%lx", exception,
703 esr);
704 }
705 }
706
707 void
do_el0_sync(struct thread * td,struct trapframe * frame)708 do_el0_sync(struct thread *td, struct trapframe *frame)
709 {
710 pcpu_bp_harden bp_harden;
711 uint32_t exception;
712 uint64_t esr, far;
713 int dfsc;
714 bool skip_userret;
715
716 /* Check we have a sane environment when entering from userland */
717 KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
718 ("Invalid pcpu address from userland: %p (tpidr 0x%lx)",
719 get_pcpu(), READ_SPECIALREG(tpidr_el1)));
720
721 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
722 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
723
724 far = frame->tf_far;
725 esr = frame->tf_esr;
726 exception = ESR_ELx_EXCEPTION(esr);
727 if (exception == EXCP_INSN_ABORT_L && far > VM_MAXUSER_ADDRESS) {
728 /*
729 * Userspace may be trying to train the branch predictor to
730 * attack the kernel. If we are on a CPU affected by this
731 * call the handler to clear the branch predictor state.
732 */
733 bp_harden = PCPU_GET(bp_harden);
734 if (bp_harden != NULL)
735 bp_harden();
736 }
737 intr_enable();
738
739 CTR4(KTR_TRAP, "%s: exception=%lu, elr=0x%lx, esr=0x%lx",
740 __func__, exception, frame->tf_elr, esr);
741
742 skip_userret = false;
743 switch (exception) {
744 case EXCP_FP_SIMD:
745 #ifdef VFP
746 vfp_restore_state();
747 #else
748 panic("VFP exception in userland");
749 #endif
750 break;
751 case EXCP_TRAP_FP:
752 #ifdef VFP
753 fpe_trap(td, (void *)frame->tf_elr, esr);
754 #else
755 panic("VFP exception in userland");
756 #endif
757 break;
758 case EXCP_SVE:
759 /* Returns true if this thread can use SVE */
760 if (!sve_restore_state(td))
761 call_trapsignal(td, SIGILL, ILL_ILLTRP,
762 (void *)frame->tf_elr, exception);
763 break;
764 case EXCP_SVC32:
765 case EXCP_SVC64:
766 skip_userret = svc_handler(td, frame);
767 break;
768 case EXCP_INSN_ABORT_L:
769 case EXCP_DATA_ABORT_L:
770 case EXCP_DATA_ABORT:
771 dfsc = esr & ISS_DATA_DFSC_MASK;
772 if (dfsc < nitems(abort_handlers) &&
773 abort_handlers[dfsc] != NULL)
774 abort_handlers[dfsc](td, frame, esr, far, 1);
775 else {
776 print_registers(frame);
777 print_gp_register("far", far);
778 printf(" esr: 0x%.16lx\n", esr);
779 panic("Unhandled EL0 %s abort: 0x%x",
780 exception == EXCP_INSN_ABORT_L ? "instruction" :
781 "data", dfsc);
782 }
783 break;
784 case EXCP_UNKNOWN:
785 if (!undef_insn(frame))
786 call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far,
787 exception);
788 break;
789 case EXCP_FPAC:
790 call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr,
791 exception);
792 break;
793 case EXCP_SP_ALIGN:
794 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp,
795 exception);
796 break;
797 case EXCP_PC_ALIGN:
798 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
799 exception);
800 break;
801 case EXCP_BRKPT_EL0:
802 case EXCP_BRK:
803 #ifdef COMPAT_FREEBSD32
804 case EXCP_BRKPT_32:
805 #endif /* COMPAT_FREEBSD32 */
806 call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr,
807 exception);
808 break;
809 case EXCP_WATCHPT_EL0:
810 call_trapsignal(td, SIGTRAP, TRAP_TRACE, (void *)far,
811 exception);
812 break;
813 case EXCP_MSR:
814 /*
815 * The CPU can raise EXCP_MSR when userspace executes an mrs
816 * instruction to access a special register userspace doesn't
817 * have access to.
818 */
819 if (!undef_insn(frame))
820 call_trapsignal(td, SIGILL, ILL_PRVOPC,
821 (void *)frame->tf_elr, exception);
822 break;
823 case EXCP_SOFTSTP_EL0:
824 PROC_LOCK(td->td_proc);
825 if ((td->td_dbgflags & TDB_STEP) != 0) {
826 td->td_frame->tf_spsr &= ~PSR_SS;
827 td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
828 WRITE_SPECIALREG(mdscr_el1,
829 READ_SPECIALREG(mdscr_el1) & ~MDSCR_SS);
830 }
831 PROC_UNLOCK(td->td_proc);
832 call_trapsignal(td, SIGTRAP, TRAP_TRACE,
833 (void *)frame->tf_elr, exception);
834 break;
835 case EXCP_BTI:
836 call_trapsignal(td, SIGILL, ILL_ILLOPC, (void *)frame->tf_elr,
837 exception);
838 break;
839 case EXCP_MOE:
840 handle_moe(td, frame, esr);
841 break;
842 default:
843 call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr,
844 exception);
845 break;
846 }
847
848 if (!skip_userret)
849 userret(td, frame);
850 KASSERT(
851 (td->td_pcb->pcb_fpflags & ~(PCB_FP_USERMASK|PCB_FP_SVEVALID)) == 0,
852 ("Kernel VFP flags set while entering userspace"));
853 KASSERT(
854 td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate,
855 ("Kernel VFP state in use when entering userspace"));
856 }
857
858 /*
859 * TODO: We will need to handle these later when we support ARMv8.2 RAS.
860 */
861 void
do_serror(struct trapframe * frame)862 do_serror(struct trapframe *frame)
863 {
864 uint64_t esr, far;
865
866 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
867 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
868
869 far = frame->tf_far;
870 esr = frame->tf_esr;
871
872 print_registers(frame);
873 print_gp_register("far", far);
874 printf(" esr: 0x%.16lx\n", esr);
875 panic("Unhandled System Error");
876 }
877
878 void
unhandled_exception(struct trapframe * frame)879 unhandled_exception(struct trapframe *frame)
880 {
881 uint64_t esr, far;
882
883 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
884 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
885
886 far = frame->tf_far;
887 esr = frame->tf_esr;
888
889 print_registers(frame);
890 print_gp_register("far", far);
891 printf(" esr: 0x%.16lx\n", esr);
892 panic("Unhandled exception");
893 }
894