1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (C) 1994, David Greenman
5 * Copyright (c) 1990, 1993
6 * The Regents of the University of California. All rights reserved.
7 *
8 * This code is derived from software contributed to Berkeley by
9 * the University of Utah, and William Jolitz.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by the University of
22 * California, Berkeley and its contributors.
23 * 4. Neither the name of the University nor the names of its contributors
24 * may be used to endorse or promote products derived from this software
25 * without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE.
38 */
39
40 /*
41 * AMD64 Trap and System call handling
42 */
43
44 #include "opt_clock.h"
45 #include "opt_cpu.h"
46 #include "opt_hwpmc_hooks.h"
47 #include "opt_isa.h"
48 #include "opt_kdb.h"
49
50 #include <sys/param.h>
51 #include <sys/asan.h>
52 #include <sys/bus.h>
53 #include <sys/systm.h>
54 #include <sys/proc.h>
55 #include <sys/ptrace.h>
56 #include <sys/kdb.h>
57 #include <sys/kernel.h>
58 #include <sys/ktr.h>
59 #include <sys/lock.h>
60 #include <sys/msan.h>
61 #include <sys/mutex.h>
62 #include <sys/resourcevar.h>
63 #include <sys/signalvar.h>
64 #include <sys/syscall.h>
65 #include <sys/sysctl.h>
66 #include <sys/sysent.h>
67 #include <sys/uio.h>
68 #include <sys/vmmeter.h>
69 #ifdef HWPMC_HOOKS
70 #include <sys/pmckern.h>
71 PMC_SOFT_DEFINE( , , page_fault, all);
72 PMC_SOFT_DEFINE( , , page_fault, read);
73 PMC_SOFT_DEFINE( , , page_fault, write);
74 #endif
75
76 #include <vm/vm.h>
77 #include <vm/vm_param.h>
78 #include <vm/pmap.h>
79 #include <vm/vm_kern.h>
80 #include <vm/vm_map.h>
81 #include <vm/vm_page.h>
82 #include <vm/vm_extern.h>
83
84 #include <machine/cpu.h>
85 #include <machine/intr_machdep.h>
86 #include <x86/mca.h>
87 #include <machine/md_var.h>
88 #include <machine/pcb.h>
89 #include <machine/smp.h>
90 #include <machine/stack.h>
91 #include <machine/trap.h>
92 #include <machine/tss.h>
93
94 #ifdef KDTRACE_HOOKS
95 #include <sys/dtrace_bsd.h>
96 #endif
97
98 extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg),
99 IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32),
100 IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall);
101
102 void __noinline trap(struct trapframe *frame);
103 void trap_check(struct trapframe *frame);
104 void dblfault_handler(struct trapframe *frame);
105
106 static int trap_pfault(struct trapframe *, bool, int *, int *);
107 static void trap_diag(struct trapframe *, vm_offset_t);
108 static void trap_fatal(struct trapframe *, vm_offset_t);
109 #ifdef KDTRACE_HOOKS
110 static bool trap_user_dtrace(struct trapframe *,
111 int (**hook)(struct trapframe *));
112 #endif
113
114 static const char UNKNOWN[] = "unknown";
115 static const char *const trap_msg[] = {
116 [0] = UNKNOWN, /* unused */
117 [T_PRIVINFLT] = "privileged instruction fault",
118 [2] = UNKNOWN, /* unused */
119 [T_BPTFLT] = "breakpoint instruction fault",
120 [4] = UNKNOWN, /* unused */
121 [5] = UNKNOWN, /* unused */
122 [T_ARITHTRAP] = "arithmetic trap",
123 [7] = UNKNOWN, /* unused */
124 [8] = UNKNOWN, /* unused */
125 [T_PROTFLT] = "general protection fault",
126 [T_TRCTRAP] = "debug exception",
127 [11] = UNKNOWN, /* unused */
128 [T_PAGEFLT] = "page fault",
129 [13] = UNKNOWN, /* unused */
130 [T_ALIGNFLT] = "alignment fault",
131 [15] = UNKNOWN, /* unused */
132 [16] = UNKNOWN, /* unused */
133 [17] = UNKNOWN, /* unused */
134 [T_DIVIDE] = "integer divide fault",
135 [T_NMI] = "non-maskable interrupt trap",
136 [T_OFLOW] = "overflow trap",
137 [T_BOUND] = "FPU bounds check fault",
138 [T_DNA] = "FPU device not available",
139 [T_DOUBLEFLT] = "double fault",
140 [T_FPOPFLT] = "FPU operand fetch fault",
141 [T_TSSFLT] = "invalid TSS fault",
142 [T_SEGNPFLT] = "segment not present fault",
143 [T_STKFLT] = "stack fault",
144 [T_MCHK] = "machine check trap",
145 [T_XMMFLT] = "SIMD floating-point exception",
146 [T_RESERVED] = "reserved (unknown) fault",
147 [31] = UNKNOWN, /* reserved */
148 [T_DTRACE_RET] = "DTrace pid return trap",
149 };
150
151 static const char *
traptype_to_msg(u_int type)152 traptype_to_msg(u_int type)
153 {
154 return (type < nitems(trap_msg) ? trap_msg[type] :
155 "unknown/reserved trap");
156 }
157
158 static int uprintf_signal;
159 SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN,
160 &uprintf_signal, 0,
161 "Print debugging information on trap signal to ctty");
162
163 u_long cnt_efirt_faults;
164 int print_efirt_faults = 1;
165
166 /*
167 * Control L1D flush on return from NMI.
168 *
169 * Tunable can be set to the following values:
170 * 0 - only enable flush on return from NMI if required by vmm.ko (default)
171 * >1 - always flush on return from NMI.
172 *
173 * Post-boot, the sysctl indicates if flushing is currently enabled.
174 */
175 int nmi_flush_l1d_sw;
176 SYSCTL_INT(_machdep, OID_AUTO, nmi_flush_l1d_sw, CTLFLAG_RWTUN,
177 &nmi_flush_l1d_sw, 0,
178 "Flush L1 Data Cache on NMI exit, software bhyve L1TF mitigation assist");
179
180 /*
181 * Table of handlers for various segment load faults.
182 */
183 static const struct {
184 uintptr_t faddr;
185 uintptr_t fhandler;
186 } sfhandlers[] = {
187 {
188 .faddr = (uintptr_t)ld_ds,
189 .fhandler = (uintptr_t)ds_load_fault,
190 },
191 {
192 .faddr = (uintptr_t)ld_es,
193 .fhandler = (uintptr_t)es_load_fault,
194 },
195 {
196 .faddr = (uintptr_t)ld_fs,
197 .fhandler = (uintptr_t)fs_load_fault,
198 },
199 {
200 .faddr = (uintptr_t)ld_gs,
201 .fhandler = (uintptr_t)gs_load_fault,
202 },
203 {
204 .faddr = (uintptr_t)ld_gsbase,
205 .fhandler = (uintptr_t)gsbase_load_fault
206 },
207 {
208 .faddr = (uintptr_t)ld_fsbase,
209 .fhandler = (uintptr_t)fsbase_load_fault,
210 },
211 };
212
213 /*
214 * Exception, fault, and trap interface to the FreeBSD kernel.
215 * This common code is called from assembly language IDT gate entry
216 * routines that prepare a suitable stack frame, and restore this
217 * frame after the exception has been processed.
218 */
219
220 void
trap(struct trapframe * frame)221 trap(struct trapframe *frame)
222 {
223 ksiginfo_t ksi;
224 struct thread *td;
225 struct proc *p;
226 register_t addr, dr6;
227 size_t i;
228 int pf, signo, ucode;
229 u_int type;
230
231 td = curthread;
232 p = td->td_proc;
233 dr6 = 0;
234
235 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
236 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
237
238 VM_CNT_INC(v_trap);
239 type = frame->tf_trapno;
240
241 #ifdef KDB
242 if (kdb_active) {
243 kdb_reenter();
244 return;
245 }
246 #endif
247 if (type == T_NMI) {
248 nmi_handle_intr(frame);
249 return;
250 }
251
252 if (type == T_RESERVED) {
253 trap_fatal(frame, 0);
254 return;
255 }
256
257 if ((frame->tf_rflags & PSL_I) == 0) {
258 /*
259 * Buggy application or kernel code has disabled
260 * interrupts and then trapped. Enabling interrupts
261 * now is wrong, but it is better than running with
262 * interrupts disabled until they are accidentally
263 * enabled later.
264 */
265 if (TRAPF_USERMODE(frame)) {
266 uprintf(
267 "pid %ld (%s): trap %d (%s) "
268 "with interrupts disabled\n",
269 (long)curproc->p_pid, curthread->td_name, type,
270 trap_msg[type]);
271 } else {
272 switch (type) {
273 case T_NMI:
274 case T_BPTFLT:
275 case T_TRCTRAP:
276 case T_PROTFLT:
277 case T_SEGNPFLT:
278 case T_STKFLT:
279 break;
280 default:
281 printf(
282 "kernel trap %d with interrupts disabled\n",
283 type);
284
285 /*
286 * We shouldn't enable interrupts while holding a
287 * spin lock.
288 */
289 if (td->td_md.md_spinlock_count == 0)
290 enable_intr();
291 }
292 }
293 }
294
295 if (TRAPF_USERMODE(frame)) {
296 /* user trap */
297
298 td->td_pticks = 0;
299 td->td_frame = frame;
300 addr = frame->tf_rip;
301 if (td->td_cowgen != atomic_load_int(&p->p_cowgen))
302 thread_cow_update(td);
303
304 switch (type) {
305 case T_PRIVINFLT: /* privileged instruction fault */
306 signo = SIGILL;
307 ucode = ILL_PRVOPC;
308 break;
309
310 case T_BPTFLT: /* bpt instruction fault */
311 #ifdef KDTRACE_HOOKS
312 if (trap_user_dtrace(frame, &dtrace_pid_probe_ptr))
313 return;
314 #else
315 enable_intr();
316 #endif
317 signo = SIGTRAP;
318 ucode = TRAP_BRKPT;
319 break;
320
321 case T_TRCTRAP: /* debug exception */
322 enable_intr();
323 signo = SIGTRAP;
324 ucode = TRAP_TRACE;
325 dr6 = rdr6();
326 if ((dr6 & DBREG_DR6_BS) != 0) {
327 PROC_LOCK(td->td_proc);
328 if ((td->td_dbgflags & TDB_STEP) != 0) {
329 td->td_frame->tf_rflags &= ~PSL_T;
330 td->td_dbgflags &= ~TDB_STEP;
331 }
332 PROC_UNLOCK(td->td_proc);
333 }
334 break;
335
336 case T_ARITHTRAP: /* arithmetic trap */
337 ucode = fputrap_x87();
338 if (ucode == -1)
339 return;
340 signo = SIGFPE;
341 break;
342
343 case T_PROTFLT: /* general protection fault */
344 signo = SIGBUS;
345 ucode = BUS_OBJERR;
346 break;
347 case T_STKFLT: /* stack fault */
348 case T_SEGNPFLT: /* segment not present fault */
349 signo = SIGBUS;
350 ucode = BUS_ADRERR;
351 break;
352 case T_TSSFLT: /* invalid TSS fault */
353 signo = SIGBUS;
354 ucode = BUS_OBJERR;
355 break;
356 case T_ALIGNFLT:
357 signo = SIGBUS;
358 ucode = BUS_ADRALN;
359 break;
360 case T_DOUBLEFLT: /* double fault */
361 default:
362 signo = SIGBUS;
363 ucode = BUS_OBJERR;
364 break;
365
366 case T_PAGEFLT: /* page fault */
367 /*
368 * Can emulator handle this trap?
369 */
370 if (*p->p_sysent->sv_trap != NULL &&
371 (*p->p_sysent->sv_trap)(td) == 0)
372 return;
373
374 pf = trap_pfault(frame, true, &signo, &ucode);
375 if (pf == -1)
376 return;
377 if (pf == 0)
378 goto userret;
379 addr = frame->tf_addr;
380 break;
381
382 case T_DIVIDE: /* integer divide fault */
383 ucode = FPE_INTDIV;
384 signo = SIGFPE;
385 break;
386
387 case T_OFLOW: /* integer overflow fault */
388 ucode = FPE_INTOVF;
389 signo = SIGFPE;
390 break;
391
392 case T_BOUND: /* bounds check fault */
393 ucode = FPE_FLTSUB;
394 signo = SIGFPE;
395 break;
396
397 case T_DNA:
398 /* transparent fault (due to context switch "late") */
399 KASSERT(PCB_USER_FPU(td->td_pcb),
400 ("kernel FPU ctx has leaked"));
401 fpudna();
402 return;
403
404 case T_FPOPFLT: /* FPU operand fetch fault */
405 ucode = ILL_COPROC;
406 signo = SIGILL;
407 break;
408
409 case T_XMMFLT: /* SIMD floating-point exception */
410 ucode = fputrap_sse();
411 if (ucode == -1)
412 return;
413 signo = SIGFPE;
414 break;
415 #ifdef KDTRACE_HOOKS
416 case T_DTRACE_RET:
417 (void)trap_user_dtrace(frame, &dtrace_return_probe_ptr);
418 return;
419 #endif
420 }
421 } else {
422 /* kernel trap */
423
424 KASSERT(cold || td->td_ucred != NULL,
425 ("kernel trap doesn't have ucred"));
426
427 /*
428 * Most likely, EFI RT faulted. This check prevents
429 * kdb from handling breakpoints set on the BIOS text,
430 * if such option is ever needed.
431 */
432 if ((td->td_pflags & TDP_EFIRT) != 0 &&
433 curpcb->pcb_onfault != NULL && type != T_PAGEFLT) {
434 u_long cnt = atomic_fetchadd_long(&cnt_efirt_faults, 1);
435
436 if ((print_efirt_faults == 1 && cnt == 0) ||
437 print_efirt_faults == 2) {
438 trap_diag(frame, 0);
439 printf("EFI RT fault %s\n",
440 traptype_to_msg(type));
441 }
442 frame->tf_rip = (long)curpcb->pcb_onfault;
443 return;
444 }
445
446 switch (type) {
447 case T_PAGEFLT: /* page fault */
448 (void)trap_pfault(frame, false, NULL, NULL);
449 return;
450
451 case T_DNA:
452 if (PCB_USER_FPU(td->td_pcb))
453 panic("Unregistered use of FPU in kernel");
454 fpudna();
455 return;
456
457 case T_ARITHTRAP: /* arithmetic trap */
458 case T_XMMFLT: /* SIMD floating-point exception */
459 case T_FPOPFLT: /* FPU operand fetch fault */
460 /*
461 * For now, supporting kernel handler
462 * registration for FPU traps is overkill.
463 */
464 trap_fatal(frame, 0);
465 return;
466
467 case T_STKFLT: /* stack fault */
468 case T_PROTFLT: /* general protection fault */
469 case T_SEGNPFLT: /* segment not present fault */
470 if (td->td_intr_nesting_level != 0)
471 break;
472
473 /*
474 * Invalid segment selectors and out of bounds
475 * %rip's and %rsp's can be set up in user mode.
476 * This causes a fault in kernel mode when the
477 * kernel tries to return to user mode. We want
478 * to get this fault so that we can fix the
479 * problem here and not have to check all the
480 * selectors and pointers when the user changes
481 * them.
482 *
483 * In case of PTI, the IRETQ faulted while the
484 * kernel used the pti stack, and exception
485 * frame records %rsp value pointing to that
486 * stack. If we return normally to
487 * doreti_iret_fault, the trapframe is
488 * reconstructed on pti stack, and calltrap()
489 * called on it as well. Due to the very
490 * limited pti stack size, kernel does not
491 * survive for too long. Switch to the normal
492 * thread stack for the trap handling.
493 *
494 * Magic '5' is the number of qwords occupied by
495 * the hardware trap frame.
496 */
497 if (frame->tf_rip == (long)doreti_iret) {
498 KASSERT((read_rflags() & PSL_I) == 0,
499 ("interrupts enabled"));
500 frame->tf_rip = (long)doreti_iret_fault;
501 if ((PCPU_GET(curpmap)->pm_ucr3 !=
502 PMAP_NO_CR3) &&
503 (frame->tf_rsp == (uintptr_t)PCPU_GET(
504 pti_rsp0) - 5 * sizeof(register_t))) {
505 frame->tf_rsp = PCPU_GET(rsp0) - 5 *
506 sizeof(register_t);
507 }
508 return;
509 }
510
511 for (i = 0; i < nitems(sfhandlers); i++) {
512 if (frame->tf_rip == sfhandlers[i].faddr) {
513 KASSERT((read_rflags() & PSL_I) == 0,
514 ("interrupts enabled"));
515 frame->tf_rip = sfhandlers[i].fhandler;
516 return;
517 }
518 }
519
520 if (curpcb->pcb_onfault != NULL) {
521 frame->tf_rip = (long)curpcb->pcb_onfault;
522 return;
523 }
524 break;
525
526 case T_TSSFLT:
527 /*
528 * PSL_NT can be set in user mode and isn't cleared
529 * automatically when the kernel is entered. This
530 * causes a TSS fault when the kernel attempts to
531 * `iret' because the TSS link is uninitialized. We
532 * want to get this fault so that we can fix the
533 * problem here and not every time the kernel is
534 * entered.
535 */
536 if (frame->tf_rflags & PSL_NT) {
537 frame->tf_rflags &= ~PSL_NT;
538 return;
539 }
540 break;
541
542 case T_TRCTRAP: /* debug exception */
543 /* Clear any pending debug events. */
544 dr6 = rdr6();
545 load_dr6(0);
546
547 /*
548 * Ignore debug register exceptions due to
549 * accesses in the user's address space, which
550 * can happen under several conditions such as
551 * if a user sets a watchpoint on a buffer and
552 * then passes that buffer to a system call.
553 * We still want to get TRCTRAPS for addresses
554 * in kernel space because that is useful when
555 * debugging the kernel.
556 */
557 if (user_dbreg_trap(dr6))
558 return;
559
560 /*
561 * Malicious user code can configure a debug
562 * register watchpoint to trap on data access
563 * to the top of stack and then execute 'pop
564 * %ss; int 3'. Due to exception deferral for
565 * 'pop %ss', the CPU will not interrupt 'int
566 * 3' to raise the DB# exception for the debug
567 * register but will postpone the DB# until
568 * execution of the first instruction of the
569 * BP# handler (in kernel mode). Normally the
570 * previous check would ignore DB# exceptions
571 * for watchpoints on user addresses raised in
572 * kernel mode. However, some CPU errata
573 * include cases where DB# exceptions do not
574 * properly set bits in %dr6, e.g. Haswell
575 * HSD23 and Skylake-X SKZ24.
576 *
577 * A deferred DB# can also be raised on the
578 * first instructions of system call entry
579 * points or single-step traps via similar use
580 * of 'pop %ss' or 'mov xxx, %ss'.
581 */
582 if (pti) {
583 if (frame->tf_rip ==
584 (uintptr_t)IDTVEC(fast_syscall_pti) ||
585 #ifdef COMPAT_FREEBSD32
586 frame->tf_rip ==
587 (uintptr_t)IDTVEC(int0x80_syscall_pti) ||
588 #endif
589 frame->tf_rip == (uintptr_t)IDTVEC(bpt_pti))
590 return;
591 } else {
592 if (frame->tf_rip ==
593 (uintptr_t)IDTVEC(fast_syscall) ||
594 #ifdef COMPAT_FREEBSD32
595 frame->tf_rip ==
596 (uintptr_t)IDTVEC(int0x80_syscall) ||
597 #endif
598 frame->tf_rip == (uintptr_t)IDTVEC(bpt))
599 return;
600 }
601 if (frame->tf_rip == (uintptr_t)IDTVEC(dbg) ||
602 /* Needed for AMD. */
603 frame->tf_rip == (uintptr_t)IDTVEC(fast_syscall32))
604 return;
605 /*
606 * FALLTHROUGH (TRCTRAP kernel mode, kernel address)
607 */
608 case T_BPTFLT:
609 /*
610 * If KDB is enabled, let it handle the debugger trap.
611 * Otherwise, debugger traps "can't happen".
612 */
613 #ifdef KDB
614 if (kdb_trap(type, dr6, frame))
615 return;
616 #endif
617 break;
618 }
619
620 trap_fatal(frame, 0);
621 return;
622 }
623
624 ksiginfo_init_trap(&ksi);
625 ksi.ksi_signo = signo;
626 ksi.ksi_code = ucode;
627 ksi.ksi_trapno = type;
628 ksi.ksi_addr = (void *)addr;
629 if (uprintf_signal) {
630 uprintf("pid %d comm %s: signal %d err %#lx code %d type %d "
631 "addr %#lx rsp %#lx rip %#lx rax %#lx "
632 "<%02x %02x %02x %02x %02x %02x %02x %02x>\n",
633 p->p_pid, p->p_comm, signo, frame->tf_err, ucode, type,
634 addr, frame->tf_rsp, frame->tf_rip, frame->tf_rax,
635 fubyte((void *)(frame->tf_rip + 0)),
636 fubyte((void *)(frame->tf_rip + 1)),
637 fubyte((void *)(frame->tf_rip + 2)),
638 fubyte((void *)(frame->tf_rip + 3)),
639 fubyte((void *)(frame->tf_rip + 4)),
640 fubyte((void *)(frame->tf_rip + 5)),
641 fubyte((void *)(frame->tf_rip + 6)),
642 fubyte((void *)(frame->tf_rip + 7)));
643 }
644 KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
645 trapsignal(td, &ksi);
646
647 userret:
648 userret(td, frame);
649 KASSERT(PCB_USER_FPU(td->td_pcb),
650 ("Return from trap with kernel FPU ctx leaked"));
651 }
652
653 /*
654 * Ensure that we ignore any DTrace-induced faults. This function cannot
655 * be instrumented, so it cannot generate such faults itself.
656 */
657 void
trap_check(struct trapframe * frame)658 trap_check(struct trapframe *frame)
659 {
660
661 #ifdef KDTRACE_HOOKS
662 if (dtrace_trap_func != NULL &&
663 (*dtrace_trap_func)(frame, frame->tf_trapno) != 0)
664 return;
665 #endif
666 trap(frame);
667 }
668
669 static bool
trap_is_smap(struct trapframe * frame)670 trap_is_smap(struct trapframe *frame)
671 {
672
673 /*
674 * A page fault on a userspace address is classified as
675 * SMAP-induced if:
676 * - SMAP is supported;
677 * - kernel mode accessed present data page;
678 * - rflags.AC was cleared.
679 * Kernel must never access user space with rflags.AC cleared
680 * if SMAP is enabled.
681 */
682 return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 &&
683 (frame->tf_err & (PGEX_P | PGEX_U | PGEX_I | PGEX_RSV)) ==
684 PGEX_P && (frame->tf_rflags & PSL_AC) == 0);
685 }
686
687 static bool
trap_is_pti(struct trapframe * frame)688 trap_is_pti(struct trapframe *frame)
689 {
690
691 return (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 &&
692 pg_nx != 0 && (frame->tf_err & (PGEX_P | PGEX_W |
693 PGEX_U | PGEX_I)) == (PGEX_P | PGEX_U | PGEX_I) &&
694 (curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK) ==
695 (PCPU_GET(curpmap)->pm_cr3 & ~CR3_PCID_MASK));
696 }
697
698 /*
699 * Handle all details of a page fault.
700 * Returns:
701 * -1 if this fault was fatal, typically from kernel mode
702 * (cannot happen, but we need to return something).
703 * 0 if this fault was handled by updating either the user or kernel
704 * page table, execution can continue.
705 * 1 if this fault was from usermode and it was not handled, a synchronous
706 * signal should be delivered to the thread. *signo returns the signal
707 * number, *ucode gives si_code.
708 */
709 static int
trap_pfault(struct trapframe * frame,bool usermode,int * signo,int * ucode)710 trap_pfault(struct trapframe *frame, bool usermode, int *signo, int *ucode)
711 {
712 struct thread *td;
713 struct proc *p;
714 vm_map_t map;
715 vm_offset_t eva;
716 int rv;
717 vm_prot_t ftype;
718
719 MPASS(!usermode || (signo != NULL && ucode != NULL));
720
721 td = curthread;
722 p = td->td_proc;
723 eva = frame->tf_addr;
724
725 if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
726 /*
727 * Due to both processor errata and lazy TLB invalidation when
728 * access restrictions are removed from virtual pages, memory
729 * accesses that are allowed by the physical mapping layer may
730 * nonetheless cause one spurious page fault per virtual page.
731 * When the thread is executing a "no faulting" section that
732 * is bracketed by vm_fault_{disable,enable}_pagefaults(),
733 * every page fault is treated as a spurious page fault,
734 * unless it accesses the same virtual address as the most
735 * recent page fault within the same "no faulting" section.
736 */
737 if (td->td_md.md_spurflt_addr != eva ||
738 (td->td_pflags & TDP_RESETSPUR) != 0) {
739 /*
740 * Do nothing to the TLB. A stale TLB entry is
741 * flushed automatically by a page fault.
742 */
743 td->td_md.md_spurflt_addr = eva;
744 td->td_pflags &= ~TDP_RESETSPUR;
745 return (0);
746 }
747 } else {
748 /*
749 * If we get a page fault while in a critical section, then
750 * it is most likely a fatal kernel page fault. The kernel
751 * is already going to panic trying to get a sleep lock to
752 * do the VM lookup, so just consider it a fatal trap so the
753 * kernel can print out a useful trap message and even get
754 * to the debugger.
755 *
756 * If we get a page fault while holding a non-sleepable
757 * lock, then it is most likely a fatal kernel page fault.
758 * If WITNESS is enabled, then it's going to whine about
759 * bogus LORs with various VM locks, so just skip to the
760 * fatal trap handling directly.
761 */
762 if (td->td_critnest != 0 ||
763 WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
764 "Kernel page fault") != 0) {
765 trap_fatal(frame, eva);
766 return (-1);
767 }
768 }
769 if (eva >= kva_layout.km_low) {
770 /*
771 * Don't allow user-mode faults in kernel address space.
772 */
773 if (usermode) {
774 *signo = SIGSEGV;
775 *ucode = SEGV_MAPERR;
776 return (1);
777 }
778
779 map = kernel_map;
780 } else {
781 map = &p->p_vmspace->vm_map;
782
783 /*
784 * When accessing a usermode address, kernel must be
785 * ready to accept the page fault, and provide a
786 * handling routine. Since accessing the address
787 * without the handler is a bug, do not try to handle
788 * it normally, and panic immediately.
789 *
790 * If SMAP is enabled, filter SMAP faults also,
791 * because illegal access might occur to the mapped
792 * user address, causing infinite loop.
793 */
794 if (!usermode && (td->td_intr_nesting_level != 0 ||
795 trap_is_smap(frame) || curpcb->pcb_onfault == NULL)) {
796 trap_fatal(frame, eva);
797 return (-1);
798 }
799 }
800
801 /*
802 * If the trap was caused by errant bits in the PTE then panic.
803 */
804 if (frame->tf_err & PGEX_RSV) {
805 trap_fatal(frame, eva);
806 return (-1);
807 }
808
809 /*
810 * User-mode protection key violation (PKU). May happen
811 * either from usermode or from kernel if copyin accessed
812 * key-protected mapping.
813 */
814 if ((frame->tf_err & PGEX_PK) != 0) {
815 if (eva > VM_MAXUSER_ADDRESS) {
816 trap_fatal(frame, eva);
817 return (-1);
818 }
819 if (usermode) {
820 *signo = SIGSEGV;
821 *ucode = SEGV_PKUERR;
822 return (1);
823 }
824 goto after_vmfault;
825 }
826
827 /*
828 * If nx protection of the usermode portion of kernel page
829 * tables caused trap, panic.
830 */
831 if (usermode && trap_is_pti(frame))
832 panic("PTI: pid %d comm %s tf_err %#lx", p->p_pid,
833 p->p_comm, frame->tf_err);
834
835 /*
836 * PGEX_I is defined only if the execute disable bit capability is
837 * supported and enabled.
838 */
839 if (frame->tf_err & PGEX_W)
840 ftype = VM_PROT_WRITE;
841 else if ((frame->tf_err & PGEX_I) && pg_nx != 0)
842 ftype = VM_PROT_EXECUTE;
843 else
844 ftype = VM_PROT_READ;
845
846 /* Fault in the page. */
847 rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode);
848 if (rv == KERN_SUCCESS) {
849 #ifdef HWPMC_HOOKS
850 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
851 PMC_SOFT_CALL_TF( , , page_fault, all, frame);
852 if (ftype == VM_PROT_READ)
853 PMC_SOFT_CALL_TF( , , page_fault, read,
854 frame);
855 else
856 PMC_SOFT_CALL_TF( , , page_fault, write,
857 frame);
858 }
859 #endif
860 return (0);
861 }
862
863 if (usermode)
864 return (1);
865 after_vmfault:
866 if (td->td_intr_nesting_level == 0 &&
867 curpcb->pcb_onfault != NULL) {
868 if ((td->td_pflags & TDP_EFIRT) != 0) {
869 u_long cnt = atomic_fetchadd_long(&cnt_efirt_faults, 1);
870
871 if ((print_efirt_faults == 1 && cnt == 0) ||
872 print_efirt_faults == 2) {
873 trap_diag(frame, eva);
874 printf("EFI RT page fault\n");
875 }
876 }
877 frame->tf_rip = (long)curpcb->pcb_onfault;
878 return (0);
879 }
880 trap_fatal(frame, eva);
881 return (-1);
882 }
883
884 static void
trap_diag(struct trapframe * frame,vm_offset_t eva)885 trap_diag(struct trapframe *frame, vm_offset_t eva)
886 {
887 int code, ss;
888 u_int type;
889 struct soft_segment_descriptor softseg;
890 struct user_segment_descriptor *gdt;
891
892 code = frame->tf_err;
893 type = frame->tf_trapno;
894 gdt = *PCPU_PTR(gdt);
895 sdtossd(&gdt[IDXSEL(frame->tf_cs & 0xffff)], &softseg);
896
897 printf("\n\nFatal trap %d: %s while in %s mode\n", type,
898 type < nitems(trap_msg) ? trap_msg[type] : UNKNOWN,
899 TRAPF_USERMODE(frame) ? "user" : "kernel");
900 /* Print these separately in case pcpu accesses trap. */
901 printf("cpuid = %d; apic id = %02x\n", PCPU_GET(cpuid),
902 PCPU_GET(apic_id));
903 if (type == T_PAGEFLT) {
904 printf("fault virtual address = 0x%lx\n", eva);
905 printf("fault code = %s %s %s%s%s, %s\n",
906 code & PGEX_U ? "user" : "supervisor",
907 code & PGEX_W ? "write" : "read",
908 code & PGEX_I ? "instruction" : "data",
909 code & PGEX_PK ? " prot key" : "",
910 code & PGEX_SGX ? " SGX" : "",
911 code & PGEX_RSV ? "reserved bits in PTE" :
912 code & PGEX_P ? "protection violation" : "page not present");
913 }
914 printf("instruction pointer = 0x%lx:0x%lx\n",
915 frame->tf_cs & 0xffff, frame->tf_rip);
916 ss = frame->tf_ss & 0xffff;
917 printf("stack pointer = 0x%x:0x%lx\n", ss, frame->tf_rsp);
918 printf("frame pointer = 0x%x:0x%lx\n", ss, frame->tf_rbp);
919 printf("code segment = base 0x%lx, limit 0x%lx, type 0x%x\n",
920 softseg.ssd_base, softseg.ssd_limit, softseg.ssd_type);
921 printf(" = DPL %d, pres %d, long %d, def32 %d, gran %d\n",
922 softseg.ssd_dpl, softseg.ssd_p, softseg.ssd_long, softseg.ssd_def32,
923 softseg.ssd_gran);
924 printf("processor eflags = ");
925 if (frame->tf_rflags & PSL_T)
926 printf("trace trap, ");
927 if (frame->tf_rflags & PSL_I)
928 printf("interrupt enabled, ");
929 if (frame->tf_rflags & PSL_NT)
930 printf("nested task, ");
931 if (frame->tf_rflags & PSL_RF)
932 printf("resume, ");
933 printf("IOPL = %ld\n", (frame->tf_rflags & PSL_IOPL) >> 12);
934 printf("current process = %d (%s)\n",
935 curproc->p_pid, curthread->td_name);
936
937 printf("rdi: %016lx rsi: %016lx rdx: %016lx\n", frame->tf_rdi,
938 frame->tf_rsi, frame->tf_rdx);
939 printf("rcx: %016lx r8: %016lx r9: %016lx\n", frame->tf_rcx,
940 frame->tf_r8, frame->tf_r9);
941 printf("rax: %016lx rbx: %016lx rbp: %016lx\n", frame->tf_rax,
942 frame->tf_rbx, frame->tf_rbp);
943 printf("r10: %016lx r11: %016lx r12: %016lx\n", frame->tf_r10,
944 frame->tf_r11, frame->tf_r12);
945 printf("r13: %016lx r14: %016lx r15: %016lx\n", frame->tf_r13,
946 frame->tf_r14, frame->tf_r15);
947
948 printf("trap number = %d\n", type);
949 }
950
951 static void
trap_fatal(struct trapframe * frame,vm_offset_t eva)952 trap_fatal(struct trapframe *frame, vm_offset_t eva)
953 {
954 u_int type;
955
956 type = frame->tf_trapno;
957 trap_diag(frame, eva);
958 #ifdef KDB
959 if (debugger_on_trap) {
960 bool handled;
961
962 kdb_why = KDB_WHY_TRAP;
963 handled = kdb_trap(type, 0, frame);
964 kdb_why = KDB_WHY_UNSET;
965 if (handled)
966 return;
967 }
968 #endif
969 panic("%s", traptype_to_msg(type));
970 }
971
972 #ifdef KDTRACE_HOOKS
973 /*
974 * Invoke a userspace DTrace hook. The hook pointer is cleared when no
975 * userspace probes are enabled, so we must synchronize with DTrace to ensure
976 * that a trapping thread is able to call the hook before it is cleared.
977 */
978 static bool
trap_user_dtrace(struct trapframe * frame,int (** hookp)(struct trapframe *))979 trap_user_dtrace(struct trapframe *frame, int (**hookp)(struct trapframe *))
980 {
981 int (*hook)(struct trapframe *);
982
983 hook = atomic_load_ptr(hookp);
984 enable_intr();
985 if (hook != NULL)
986 return ((hook)(frame) == 0);
987 return (false);
988 }
989 #endif
990
991 /*
992 * Double fault handler. Called when a fault occurs while writing
993 * a frame for a trap/exception onto the stack. This usually occurs
994 * when the stack overflows (such is the case with infinite recursion,
995 * for example).
996 */
997 void
dblfault_handler(struct trapframe * frame)998 dblfault_handler(struct trapframe *frame)
999 {
1000 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
1001 #ifdef KDTRACE_HOOKS
1002 if (dtrace_doubletrap_func != NULL)
1003 (*dtrace_doubletrap_func)();
1004 #endif
1005 printf("\nFatal double fault\n"
1006 "rip %#lx rsp %#lx rbp %#lx\n"
1007 "rax %#lx rdx %#lx rbx %#lx\n"
1008 "rcx %#lx rsi %#lx rdi %#lx\n"
1009 "r8 %#lx r9 %#lx r10 %#lx\n"
1010 "r11 %#lx r12 %#lx r13 %#lx\n"
1011 "r14 %#lx r15 %#lx rflags %#lx\n"
1012 "cs %#lx ss %#lx ds %#hx es %#hx fs %#hx gs %#hx\n"
1013 "fsbase %#lx gsbase %#lx kgsbase %#lx\n",
1014 frame->tf_rip, frame->tf_rsp, frame->tf_rbp,
1015 frame->tf_rax, frame->tf_rdx, frame->tf_rbx,
1016 frame->tf_rcx, frame->tf_rdi, frame->tf_rsi,
1017 frame->tf_r8, frame->tf_r9, frame->tf_r10,
1018 frame->tf_r11, frame->tf_r12, frame->tf_r13,
1019 frame->tf_r14, frame->tf_r15, frame->tf_rflags,
1020 frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
1021 frame->tf_fs, frame->tf_gs,
1022 rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
1023 /* Print these separately in case pcpu accesses trap. */
1024 printf("cpuid = %d; apic id = %02x\n", PCPU_GET(cpuid),
1025 PCPU_GET(apic_id));
1026 panic("double fault");
1027 }
1028
1029 static int __noinline
cpu_fetch_syscall_args_fallback(struct thread * td,struct syscall_args * sa)1030 cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa)
1031 {
1032 struct proc *p;
1033 struct trapframe *frame;
1034 syscallarg_t *argp;
1035 caddr_t params;
1036 int reg, regcnt, error;
1037
1038 p = td->td_proc;
1039 frame = td->td_frame;
1040 reg = 0;
1041 regcnt = NARGREGS;
1042
1043 if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
1044 sa->code = frame->tf_rdi;
1045 reg++;
1046 regcnt--;
1047 }
1048
1049 if (sa->code >= p->p_sysent->sv_size)
1050 sa->callp = &nosys_sysent;
1051 else
1052 sa->callp = &p->p_sysent->sv_table[sa->code];
1053
1054 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
1055 ("Too many syscall arguments!"));
1056 argp = &frame->tf_rdi;
1057 argp += reg;
1058 memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS);
1059 if (sa->callp->sy_narg > regcnt) {
1060 params = (caddr_t)frame->tf_rsp + sizeof(register_t);
1061 error = copyin(params, &sa->args[regcnt],
1062 (sa->callp->sy_narg - regcnt) * sizeof(sa->args[0]));
1063 if (__predict_false(error != 0))
1064 return (error);
1065 }
1066
1067 td->td_retval[0] = 0;
1068 td->td_retval[1] = frame->tf_rdx;
1069
1070 return (0);
1071 }
1072
1073 int
cpu_fetch_syscall_args(struct thread * td)1074 cpu_fetch_syscall_args(struct thread *td)
1075 {
1076 struct proc *p;
1077 struct trapframe *frame;
1078 struct syscall_args *sa;
1079
1080 p = td->td_proc;
1081 frame = td->td_frame;
1082 sa = &td->td_sa;
1083
1084 sa->code = frame->tf_rax;
1085 sa->original_code = sa->code;
1086
1087 if (__predict_false(sa->code == SYS_syscall ||
1088 sa->code == SYS___syscall ||
1089 sa->code >= p->p_sysent->sv_size))
1090 return (cpu_fetch_syscall_args_fallback(td, sa));
1091
1092 sa->callp = &p->p_sysent->sv_table[sa->code];
1093 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
1094 ("Too many syscall arguments!"));
1095
1096 if (__predict_false(sa->callp->sy_narg > NARGREGS))
1097 return (cpu_fetch_syscall_args_fallback(td, sa));
1098
1099 memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS);
1100
1101 td->td_retval[0] = 0;
1102 td->td_retval[1] = frame->tf_rdx;
1103
1104 return (0);
1105 }
1106
1107 #include "../../kern/subr_syscall.c"
1108
1109 static void (*syscall_ret_l1d_flush)(void);
1110 int syscall_ret_l1d_flush_mode;
1111
1112 static void
flush_l1d_hw(void)1113 flush_l1d_hw(void)
1114 {
1115
1116 wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
1117 }
1118
1119 static void __noinline
amd64_syscall_ret_flush_l1d_check(int error)1120 amd64_syscall_ret_flush_l1d_check(int error)
1121 {
1122 void (*p)(void);
1123
1124 if (error != EEXIST && error != EAGAIN && error != EXDEV &&
1125 error != ENOENT && error != ENOTCONN && error != EINPROGRESS) {
1126 p = atomic_load_ptr(&syscall_ret_l1d_flush);
1127 if (p != NULL)
1128 p();
1129 }
1130 }
1131
1132 static void __inline
amd64_syscall_ret_flush_l1d_check_inline(int error)1133 amd64_syscall_ret_flush_l1d_check_inline(int error)
1134 {
1135
1136 if (__predict_false(error != 0))
1137 amd64_syscall_ret_flush_l1d_check(error);
1138 }
1139
1140 void
amd64_syscall_ret_flush_l1d(int error)1141 amd64_syscall_ret_flush_l1d(int error)
1142 {
1143
1144 amd64_syscall_ret_flush_l1d_check_inline(error);
1145 }
1146
1147 void
amd64_syscall_ret_flush_l1d_recalc(void)1148 amd64_syscall_ret_flush_l1d_recalc(void)
1149 {
1150 bool l1d_hw;
1151
1152 l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
1153 again:
1154 switch (syscall_ret_l1d_flush_mode) {
1155 case 0:
1156 syscall_ret_l1d_flush = NULL;
1157 break;
1158 case 1:
1159 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
1160 flush_l1d_sw_abi;
1161 break;
1162 case 2:
1163 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
1164 break;
1165 case 3:
1166 syscall_ret_l1d_flush = flush_l1d_sw_abi;
1167 break;
1168 default:
1169 syscall_ret_l1d_flush_mode = 1;
1170 goto again;
1171 }
1172 }
1173
1174 static int
machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)1175 machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
1176 {
1177 int error, val;
1178
1179 val = syscall_ret_l1d_flush_mode;
1180 error = sysctl_handle_int(oidp, &val, 0, req);
1181 if (error != 0 || req->newptr == NULL)
1182 return (error);
1183 syscall_ret_l1d_flush_mode = val;
1184 amd64_syscall_ret_flush_l1d_recalc();
1185 return (0);
1186 }
1187 SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
1188 CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
1189 machdep_syscall_ret_flush_l1d, "I",
1190 "Flush L1D on syscall return with error (0 - off, 1 - on, "
1191 "2 - use hw only, 3 - use sw only)");
1192
1193 /*
1194 * System call handler for native binaries. The trap frame is already
1195 * set up by the assembler trampoline and a pointer to it is saved in
1196 * td_frame.
1197 */
1198 void
amd64_syscall(struct thread * td,int traced)1199 amd64_syscall(struct thread *td, int traced)
1200 {
1201 ksiginfo_t ksi;
1202
1203 kmsan_mark(td->td_frame, sizeof(*td->td_frame), KMSAN_STATE_INITED);
1204
1205 KASSERT(TRAPF_USERMODE(td->td_frame),
1206 ("%s: not from user mode", __func__));
1207
1208 syscallenter(td);
1209
1210 /*
1211 * Traced syscall.
1212 */
1213 if (__predict_false(traced)) {
1214 td->td_frame->tf_rflags &= ~PSL_T;
1215 ksiginfo_init_trap(&ksi);
1216 ksi.ksi_signo = SIGTRAP;
1217 ksi.ksi_code = TRAP_TRACE;
1218 ksi.ksi_addr = (void *)td->td_frame->tf_rip;
1219 trapsignal(td, &ksi);
1220 }
1221
1222 KASSERT(PCB_USER_FPU(td->td_pcb),
1223 ("System call %s returning with kernel FPU ctx leaked",
1224 syscallname(td->td_proc, td->td_sa.code)));
1225 KASSERT(td->td_pcb->pcb_save == get_pcb_user_save_td(td),
1226 ("System call %s returning with mangled pcb_save",
1227 syscallname(td->td_proc, td->td_sa.code)));
1228 KASSERT(pmap_not_in_di(),
1229 ("System call %s returning with leaked invl_gen %lu",
1230 syscallname(td->td_proc, td->td_sa.code),
1231 td->td_md.md_invl_gen.gen));
1232
1233 syscallret(td);
1234
1235 /*
1236 * If the user-supplied value of %rip is not a canonical
1237 * address, then some CPUs will trigger a ring 0 #GP during
1238 * the sysret instruction. However, the fault handler would
1239 * execute in ring 0 with the user's %gs and %rsp which would
1240 * not be safe. Instead, use the full return path which
1241 * catches the problem safely.
1242 */
1243 if (__predict_false(td->td_frame->tf_rip >= (la57 ?
1244 VM_MAXUSER_ADDRESS_LA57 : VM_MAXUSER_ADDRESS_LA48)))
1245 set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
1246
1247 amd64_syscall_ret_flush_l1d_check_inline(td->td_errno);
1248 }
1249