1 /*-
2  * SPDX-License-Identifier: BSD-4-Clause
127 	[T_PRIVINFLT] =	{ .ei = true,	.msg = "privileged instruction fault" },
128 	[T_BPTFLT] =	{ .ei = false,	.msg = "breakpoint instruction fault" },
130 	[T_PROTFLT] =	{ .ei = true,	.msg = "general protection fault" },
132 	[T_PAGEFLT] =	{ .ei = true,	.msg = "page fault" },
133 	[T_ALIGNFLT] = 	{ .ei = true,	.msg = "alignment fault" },
134 	[T_DIVIDE] =	{ .ei = true,	.msg = "integer divide fault" },
135 	[T_NMI] =	{ .ei = false,	.msg = "non-maskable interrupt trap" },
137 	[T_BOUND] =	{ .ei = true,	.msg = "FPU bounds check fault" },
139 	[T_DOUBLEFLT] =	{ .ei = false,	.msg = "double fault" },
140 	[T_FPOPFLT] =	{ .ei = true,	.msg = "FPU operand fetch fault" },
141 	[T_TSSFLT] =	{ .ei = true,	.msg = "invalid TSS fault" },
142 	[T_SEGNPFLT] =	{ .ei = true,	.msg = "segment not present fault" },
143 	[T_STKFLT] =	{ .ei = true,	.msg = "stack fault" },
145 	[T_XMMFLT] =	{ .ei = true,	.msg = "SIMD floating-point exception" },
205 		    td, stk, td->td_kstack, td->td_kstack_pages);  in trap_check_kstack()
210  * Exception, fault, and trap interface to the FreeBSD kernel.
231 	p = td->td_proc;  in trap()
235 	type = frame->tf_trapno;  in trap()
266 	 * executing the probe, DTrace blocks re-scheduling and sets  in trap()
267 	 * a flag in its per-cpu flags to indicate that it doesn't  in trap()
268 	 * want to fault. On returning from the probe, the no-fault  in trap()
269 	 * flag is cleared and finally re-scheduling is enabled.  in trap()
280 	 * now, before optional enable of the interrupts below.  in trap()
291 	if ((frame->tf_eflags & PSL_I) == 0 && TRAPF_USERMODE(frame) &&  in trap()
292 	    (curpcb->pcb_flags & PCB_VM86CALL) == 0)  in trap()
295 		    (long)curproc->p_pid, curthread->td_name, type,  in trap()
301 	 * spurious page fault.  in trap()
303 	if (trap_enable_intr(type) && td->td_md.md_spinlock_count == 0 &&  in trap()
304 	    frame->tf_eip != (int)cpu_switch_load_gs)  in trap()
307         if (TRAPF_USERMODE(frame) && (curpcb->pcb_flags & PCB_VM86CALL) == 0) {  in trap()
310 		td->td_pticks = 0;  in trap()
311 		td->td_frame = frame;  in trap()
312 		addr = frame->tf_eip;  in trap()
313 		if (td->td_cowgen != atomic_load_int(&p->p_cowgen))  in trap()
317 		case T_PRIVINFLT:	/* privileged instruction fault */  in trap()
322 		case T_BPTFLT:		/* bpt instruction fault */  in trap()
340 				PROC_LOCK(td->td_proc);  in trap()
341 				if ((td->td_dbgflags & TDB_STEP) != 0) {  in trap()
342 					td->td_frame->tf_eflags &= ~PSL_T;  in trap()
343 					td->td_dbgflags &= ~TDB_STEP;  in trap()
345 				PROC_UNLOCK(td->td_proc);  in trap()
351 			if (ucode == -1)  in trap()
360 		case T_PROTFLT:		/* general protection fault */  in trap()
361 		case T_STKFLT:		/* stack fault */  in trap()
362 			if (frame->tf_eflags & PSL_VM) {  in trap()
376 		case T_SEGNPFLT:	/* segment not present fault */  in trap()
380 		case T_TSSFLT:		/* invalid TSS fault */  in trap()
388 		case T_DOUBLEFLT:	/* double fault */  in trap()
394 		case T_PAGEFLT:		/* page fault */  in trap()
398 			if (pf == -2) {  in trap()
401 				 * treat the fault as an illegal instruction   in trap()
402 				 * (T_PRIVINFLT) instead of a page fault.  in trap()
404 				type = frame->tf_trapno = T_PRIVINFLT;  in trap()
408 			if (pf == -1)  in trap()
414 		case T_DIVIDE:		/* integer divide fault */  in trap()
424 			if (time_second - lastalert > 10) {  in trap()
425 				log(LOG_WARNING, "NMI: power fail\n");  in trap()
435 		case T_OFLOW:		/* integer overflow fault */  in trap()
440 		case T_BOUND:		/* bounds check fault */  in trap()
446 			KASSERT(PCB_USER_FPU(td->td_pcb),  in trap()
448 			/* transparent fault (due to context switch "late") */  in trap()
452 				p->p_pid);  in trap()
457 		case T_FPOPFLT:		/* FPU operand fetch fault */  in trap()
462 		case T_XMMFLT:		/* SIMD floating-point exception */  in trap()
464 			if (ucode == -1)  in trap()
477 		KASSERT(cold || td->td_ucred != NULL,  in trap()
480 		case T_PAGEFLT:			/* page fault */  in trap()
485 			if (PCB_USER_FPU(td->td_pcb))  in trap()
492 		case T_XMMFLT:		/* SIMD floating-point exception */  in trap()
493 		case T_FPOPFLT:		/* FPU operand fetch fault */  in trap()
506 		case T_PROTFLT:		/* general protection fault */  in trap()
507 		case T_STKFLT:		/* stack fault */  in trap()
508 			if (frame->tf_eflags & PSL_VM) {  in trap()
523 		case T_SEGNPFLT:	/* segment not present fault */  in trap()
524 			if (curpcb->pcb_flags & PCB_VM86CALL)  in trap()
530 			 * underlying LDT entry.  This causes a fault  in trap()
536 			if (frame->tf_eip == (int)cpu_switch_load_gs) {  in trap()
537 				curpcb->pcb_gs = 0;  in trap()
546 			if (td->td_intr_nesting_level != 0)  in trap()
552 			 * This causes a fault in kernel mode when the  in trap()
554 			 * to get this fault so that we can fix the  in trap()
559 			 * N.B. Comparing to long mode, 32-bit mode  in trap()
568 			if (frame->tf_eip == (int)doreti_iret + setidt_disp) {  in trap()
569 				frame->tf_eip = (int)doreti_iret_fault +  in trap()
576 			if (frame->tf_eip == (int)doreti_popl_ds +  in trap()
578 				frame->tf_eip = (int)doreti_popl_ds_fault +  in trap()
582 			if (frame->tf_eip == (int)doreti_popl_es +  in trap()
584 				frame->tf_eip = (int)doreti_popl_es_fault +  in trap()
588 			if (frame->tf_eip == (int)doreti_popl_fs +  in trap()
590 				frame->tf_eip = (int)doreti_popl_fs_fault +  in trap()
594 			if (curpcb->pcb_onfault != NULL) {  in trap()
595 				frame->tf_eip = (int)curpcb->pcb_onfault;  in trap()
604 			 * causes a TSS fault when the kernel attempts to  in trap()
606 			 * want to get this fault so that we can fix the  in trap()
610 			if (frame->tf_eflags & PSL_NT) {  in trap()
611 				frame->tf_eflags &= ~PSL_NT;  in trap()
633 			   !(curpcb->pcb_flags & PCB_VM86CALL))  in trap()
651 			 * HSD23 and Skylake-X SKZ24.  in trap()
655 			 * points or single-step traps via similar use  in trap()
658 			if (frame->tf_eip ==  in trap()
660 			    frame->tf_eip == (uintptr_t)IDTVEC(bpt) +  in trap()
662 			    frame->tf_eip == (uintptr_t)IDTVEC(dbg) +  in trap()
681 			if (time_second - lastalert > 10) {  in trap()
682 				log(LOG_WARNING, "NMI: power fail\n");  in trap()
706 		    p->p_pid, p->p_comm, signo, frame->tf_err, ucode, type,  in trap()
707 		    addr, frame->tf_ss, frame->tf_esp, frame->tf_cs,  in trap()
708 		    frame->tf_eip, frame->tf_eax,  in trap()
709 		    fubyte((void *)(frame->tf_eip + 0)),  in trap()
710 		    fubyte((void *)(frame->tf_eip + 1)),  in trap()
711 		    fubyte((void *)(frame->tf_eip + 2)),  in trap()
712 		    fubyte((void *)(frame->tf_eip + 3)),  in trap()
713 		    fubyte((void *)(frame->tf_eip + 4)),  in trap()
714 		    fubyte((void *)(frame->tf_eip + 5)),  in trap()
715 		    fubyte((void *)(frame->tf_eip + 6)),  in trap()
716 		    fubyte((void *)(frame->tf_eip + 7)));  in trap()
723 	KASSERT(PCB_USER_FPU(td->td_pcb),  in trap()
728  * Handle all details of a page fault.
730  * -2 if the fault was caused by triggered workaround for Intel Pentium
732  * -1 if this fault was fatal, typically from kernel mode
734  * 0  if this fault was handled by updating either the user or kernel
736  * 1  if this fault was from usermode and it was not handled, a synchronous
753 	p = td->td_proc;  in trap_pfault()
755 	if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {  in trap_pfault()
760 		 * nonetheless cause one spurious page fault per virtual page.   in trap_pfault()
762 		 * is bracketed by vm_fault_{disable,enable}_pagefaults(),  in trap_pfault()
763 		 * every page fault is treated as a spurious page fault,  in trap_pfault()
765 		 * recent page fault within the same "no faulting" section.  in trap_pfault()
767 		if (td->td_md.md_spurflt_addr != eva ||  in trap_pfault()
768 		    (td->td_pflags & TDP_RESETSPUR) != 0) {  in trap_pfault()
771 			 * flushed automatically by a page fault.  in trap_pfault()
773 			td->td_md.md_spurflt_addr = eva;  in trap_pfault()
774 			td->td_pflags &= ~TDP_RESETSPUR;  in trap_pfault()
779 		 * If we get a page fault while in a critical section, then  in trap_pfault()
780 		 * it is most likely a fatal kernel page fault.  The kernel  in trap_pfault()
786 		 * If we get a page fault while holding a non-sleepable  in trap_pfault()
787 		 * lock, then it is most likely a fatal kernel page fault.  in trap_pfault()
792 		if (td->td_critnest != 0 ||  in trap_pfault()
794 		    "Kernel page fault") != 0) {  in trap_pfault()
796 			return (-1);  in trap_pfault()
801 		 * Don't allow user-mode faults in kernel address space.  in trap_pfault()
806 		 * fault.  in trap_pfault()
812 			return (-2);  in trap_pfault()
821 		return (-1);  in trap_pfault()
823 		map = usermode ? &p->p_vmspace->vm_map : kernel_map;  in trap_pfault()
826 		 * Kernel cannot access a user-space address directly  in trap_pfault()
830 		if (!usermode && td->td_intr_nesting_level != 0) {  in trap_pfault()
832 			return (-1);  in trap_pfault()
839 	if (frame->tf_err & PGEX_RSV) {  in trap_pfault()
841 		return (-1);  in trap_pfault()
848 	if (frame->tf_err & PGEX_W)  in trap_pfault()
850 	else if ((frame->tf_err & PGEX_I) && pg_nx != 0)  in trap_pfault()
855 	/* Fault in the page. */  in trap_pfault()
873 	if (td->td_intr_nesting_level == 0 &&  in trap_pfault()
874 	    curpcb->pcb_onfault != NULL) {  in trap_pfault()
875 		frame->tf_eip = (int)curpcb->pcb_onfault;  in trap_pfault()
879 	return (-1);  in trap_pfault()
892 	code = frame->tf_err;  in trap_fatal()
893 	type = frame->tf_trapno;  in trap_fatal()
894 	sdtossd(&gdt[IDXSEL(frame->tf_cs & 0xffff)].sd, &softseg);  in trap_fatal()
897 	    frame->tf_eflags & PSL_VM ? "vm86" :  in trap_fatal()
898 	    ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel");  in trap_fatal()
905 		printf("fault virtual address	= 0x%x\n", eva);  in trap_fatal()
906 		printf("fault code		= %s %s%s, %s\n",  in trap_fatal()
918 	       frame->tf_cs & 0xffff, frame->tf_eip);  in trap_fatal()
920 		ss = frame->tf_ss & 0xffff;  in trap_fatal()
921 		esp = frame->tf_esp;  in trap_fatal()
924 		esp = (int)&frame->tf_esp;  in trap_fatal()
927 	printf("frame pointer	        = 0x%x:0x%x\n", ss, frame->tf_ebp);  in trap_fatal()
934 	if (frame->tf_eflags & PSL_T)  in trap_fatal()
936 	if (frame->tf_eflags & PSL_I)  in trap_fatal()
938 	if (frame->tf_eflags & PSL_NT)  in trap_fatal()
940 	if (frame->tf_eflags & PSL_RF)  in trap_fatal()
942 	if (frame->tf_eflags & PSL_VM)  in trap_fatal()
944 	printf("IOPL = %d\n", (frame->tf_eflags & PSL_IOPL) >> 12);  in trap_fatal()
946 	    curproc->p_pid, curthread->td_name);  in trap_fatal()
951 		frame->tf_err = eva;	/* smuggle fault address to ddb */  in trap_fatal()
953 		frame->tf_err = code;	/* restore error code */  in trap_fatal()
986  * Double fault handler. Called when a fault occurs while writing
993  * the time of the double fault is not available at <kstack> unless
994  * the machine was idle when the double fault occurred. The downside
1006 	printf("\nFatal double fault:\n");  in dblfault_handler()
1014 	    t->tss_eip, t->tss_esp, t->tss_ebp, t->tss_eax,  in dblfault_handler()
1015 	    t->tss_edx, t->tss_ecx, t->tss_edi, t->tss_esi,  in dblfault_handler()
1016 	    t->tss_ebx,  in dblfault_handler()
1017 	    t->tss_eflags, t->tss_cs, t->tss_ss, t->tss_ds,  in dblfault_handler()
1018 	    t->tss_es, t->tss_fs, t->tss_gs, t->tss_cr3);  in dblfault_handler()
1023 	panic("double fault");  in dblfault_handler()
1040 	p = td->td_proc;  in cpu_fetch_syscall_args()
1041 	frame = td->td_frame;  in cpu_fetch_syscall_args()
1042 	sa = &td->td_sa;  in cpu_fetch_syscall_args()
1045 	if (__predict_false(frame->tf_cs == 7 && frame->tf_eip == 2)) {  in cpu_fetch_syscall_args()
1051 		error = fueword32((void *)frame->tf_esp, &eip);  in cpu_fetch_syscall_args()
1052 		if (error == -1)  in cpu_fetch_syscall_args()
1054 		cs = fuword16((void *)(frame->tf_esp + sizeof(u_int32_t)));  in cpu_fetch_syscall_args()
1055 		if (cs == -1)  in cpu_fetch_syscall_args()
1059 		 * Unwind in-kernel frame after all stack frame pieces  in cpu_fetch_syscall_args()
1062 		frame->tf_eip = eip;  in cpu_fetch_syscall_args()
1063 		frame->tf_cs = cs;  in cpu_fetch_syscall_args()
1064 		frame->tf_esp += 2 * sizeof(u_int32_t);  in cpu_fetch_syscall_args()
1065 		frame->tf_err = 7;	/* size of lcall $7,$0 */  in cpu_fetch_syscall_args()
1069 	sa->code = frame->tf_eax;  in cpu_fetch_syscall_args()
1070 	sa->original_code = sa->code;  in cpu_fetch_syscall_args()
1071 	params = (caddr_t)frame->tf_esp + sizeof(uint32_t);  in cpu_fetch_syscall_args()
1076 	if (sa->code == SYS_syscall) {  in cpu_fetch_syscall_args()
1081 		if (error == -1)  in cpu_fetch_syscall_args()
1083 		sa->code = tmp;  in cpu_fetch_syscall_args()
1085 	} else if (sa->code == SYS___syscall) {  in cpu_fetch_syscall_args()
1091 		if (error == -1)  in cpu_fetch_syscall_args()
1093 		sa->code = tmp;  in cpu_fetch_syscall_args()
1097  	if (sa->code >= p->p_sysent->sv_size)  in cpu_fetch_syscall_args()
1098 		sa->callp = &nosys_sysent;  in cpu_fetch_syscall_args()
1100  		sa->callp = &p->p_sysent->sv_table[sa->code];  in cpu_fetch_syscall_args()
1102 	if (params != NULL && sa->callp->sy_narg != 0)  in cpu_fetch_syscall_args()
1103 		error = copyin(params, (caddr_t)sa->args,  in cpu_fetch_syscall_args()
1104 		    (u_int)(sa->callp->sy_narg * sizeof(uint32_t)));  in cpu_fetch_syscall_args()
1109 		td->td_retval[0] = 0;  in cpu_fetch_syscall_args()
1110 		td->td_retval[1] = frame->tf_edx;  in cpu_fetch_syscall_args()
1119  * syscall - system call request C handler.  A system call is
1131 	    (curpcb->pcb_flags & PCB_VM86CALL) == 0)) {  in syscall()
1137 	orig_tf_eflags = frame->tf_eflags;  in syscall()
1140 	td->td_frame = frame;  in syscall()
1148 		frame->tf_eflags &= ~PSL_T;  in syscall()
1152 		ksi.ksi_addr = (void *)frame->tf_eip;  in syscall()
1156 	KASSERT(PCB_USER_FPU(td->td_pcb),  in syscall()
1158 	     syscallname(td->td_proc, td->td_sa.code)));  in syscall()
1159 	KASSERT(td->td_pcb->pcb_save == get_pcb_user_save_td(td),  in syscall()
1161 	     syscallname(td->td_proc, td->td_sa.code)));  in syscall()