/* * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013, 2014 by Delphix. All rights reserved. */ /* * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.113 2003/10/15 02:04:52 peter Exp $ */ #include #include #include #include #include #include #include #include #include #include /* * only one routine in this file is interesting to lint */ #if defined(__lint) void ndptrap_frstor(void) {} #else #include "assym.h" /* * push $0 on stack for traps that do not * generate an error code. This is so the rest * of the kernel can expect a consistent stack * from from any exception. * * Note that for all exceptions for amd64 * %r11 and %rcx are on the stack. Just pop * them back into their appropriate registers and let * it get saved as is running native. */ #if defined(__xpv) && defined(__amd64) #define NPTRAP_NOERR(trapno) \ pushq $0; \ pushq $trapno #define TRAP_NOERR(trapno) \ XPV_TRAP_POP; \ NPTRAP_NOERR(trapno) /* * error code already pushed by hw * onto stack. */ #define TRAP_ERR(trapno) \ XPV_TRAP_POP; \ pushq $trapno #else /* __xpv && __amd64 */ #define TRAP_NOERR(trapno) \ push $0; \ push $trapno #define NPTRAP_NOERR(trapno) TRAP_NOERR(trapno) /* * error code already pushed by hw * onto stack. */ #define TRAP_ERR(trapno) \ push $trapno #endif /* __xpv && __amd64 */ /* * #DE */ ENTRY_NP(div0trap) TRAP_NOERR(T_ZERODIV) /* $0 */ jmp cmntrap SET_SIZE(div0trap) /* * #DB * * Fetch %dr6 and clear it, handing off the value to the * cmntrap code in %r15/%esi */ ENTRY_NP(dbgtrap) TRAP_NOERR(T_SGLSTP) /* $1 */ #if defined(__amd64) #if !defined(__xpv) /* no sysenter support yet */ /* * If we get here as a result of single-stepping a sysenter * instruction, we suddenly find ourselves taking a #db * in kernel mode -before- we've swapgs'ed. So before we can * take the trap, we do the swapgs here, and fix the return * %rip in trap() so that we return immediately after the * swapgs in the sysenter handler to avoid doing the swapgs again. * * Nobody said that the design of sysenter was particularly * elegant, did they? */ pushq %r11 /* * At this point the stack looks like this: * * (high address) r_ss * r_rsp * r_rfl * r_cs * r_rip <-- %rsp + 24 * r_err <-- %rsp + 16 * r_trapno <-- %rsp + 8 * (low address) %r11 <-- %rsp */ leaq sys_sysenter(%rip), %r11 cmpq %r11, 24(%rsp) /* Compare to saved r_rip on the stack */ je 1f leaq brand_sys_sysenter(%rip), %r11 cmpq %r11, 24(%rsp) /* Compare to saved r_rip on the stack */ jne 2f 1: SWAPGS 2: popq %r11 #endif /* !__xpv */ INTR_PUSH #if defined(__xpv) movl $6, %edi call kdi_dreg_get movq %rax, %r15 /* %db6 -> %r15 */ movl $6, %edi movl $0, %esi call kdi_dreg_set /* 0 -> %db6 */ #else movq %db6, %r15 xorl %eax, %eax movq %rax, %db6 #endif #elif defined(__i386) INTR_PUSH #if defined(__xpv) pushl $6 call kdi_dreg_get addl $4, %esp movl %eax, %esi /* %dr6 -> %esi */ pushl $0 pushl $6 call kdi_dreg_set /* 0 -> %dr6 */ addl $8, %esp #else movl %db6, %esi xorl %eax, %eax movl %eax, %db6 #endif #endif /* __i386 */ jmp cmntrap_pushed SET_SIZE(dbgtrap) #if defined(__amd64) #if !defined(__xpv) /* * Macro to set the gsbase or kgsbase to the address of the struct cpu * for this processor. If we came from userland, set kgsbase else * set gsbase. We find the proper cpu struct by looping through * the cpu structs for all processors till we find a match for the gdt * of the trapping processor. The stack is expected to be pointing at * the standard regs pushed by hardware on a trap (plus error code and trapno). */ #define SET_CPU_GSBASE \ subq $REGOFF_TRAPNO, %rsp; /* save regs */ \ movq %rax, REGOFF_RAX(%rsp); \ movq %rbx, REGOFF_RBX(%rsp); \ movq %rcx, REGOFF_RCX(%rsp); \ movq %rdx, REGOFF_RDX(%rsp); \ movq %rbp, REGOFF_RBP(%rsp); \ movq %rsp, %rbp; \ subq $16, %rsp; /* space for gdt */ \ sgdt 6(%rsp); \ movq 8(%rsp), %rcx; /* %rcx has gdt to match */ \ xorl %ebx, %ebx; /* loop index */ \ leaq cpu(%rip), %rdx; /* cpu pointer array */ \ 1: \ movq (%rdx, %rbx, CLONGSIZE), %rax; /* get cpu[i] */ \ cmpq $0x0, %rax; /* cpu[i] == NULL ? */ \ je 2f; /* yes, continue */ \ cmpq %rcx, CPU_GDT(%rax); /* gdt == cpu[i]->cpu_gdt ? */ \ je 3f; /* yes, go set gsbase */ \ 2: \ incl %ebx; /* i++ */ \ cmpl $NCPU, %ebx; /* i < NCPU ? */ \ jb 1b; /* yes, loop */ \ /* XXX BIG trouble if we fall thru here. We didn't find a gdt match */ \ 3: \ movl $MSR_AMD_KGSBASE, %ecx; \ cmpw $KCS_SEL, REGOFF_CS(%rbp); /* trap from kernel? */ \ jne 4f; /* no, go set KGSBASE */ \ movl $MSR_AMD_GSBASE, %ecx; /* yes, set GSBASE */ \ mfence; /* OPTERON_ERRATUM_88 */ \ 4: \ movq %rax, %rdx; /* write base register */ \ shrq $32, %rdx; \ wrmsr; \ movq REGOFF_RDX(%rbp), %rdx; /* restore regs */ \ movq REGOFF_RCX(%rbp), %rcx; \ movq REGOFF_RBX(%rbp), %rbx; \ movq REGOFF_RAX(%rbp), %rax; \ movq %rbp, %rsp; \ movq REGOFF_RBP(%rsp), %rbp; \ addq $REGOFF_TRAPNO, %rsp /* pop stack */ #else /* __xpv */ #define SET_CPU_GSBASE /* noop on the hypervisor */ #endif /* __xpv */ #endif /* __amd64 */ #if defined(__amd64) /* * #NMI * * XXPV: See 6532669. */ ENTRY_NP(nmiint) TRAP_NOERR(T_NMIFLT) /* $2 */ SET_CPU_GSBASE /* * Save all registers and setup segment registers * with kernel selectors. */ INTR_PUSH INTGATE_INIT_KERNEL_FLAGS TRACE_PTR(%r12, %rax, %eax, %rdx, $TT_TRAP) TRACE_REGS(%r12, %rsp, %rax, %rbx) TRACE_STAMP(%r12) movq %rsp, %rbp movq %rbp, %rdi call av_dispatch_nmivect INTR_POP IRET /*NOTREACHED*/ SET_SIZE(nmiint) #elif defined(__i386) /* * #NMI */ ENTRY_NP(nmiint) TRAP_NOERR(T_NMIFLT) /* $2 */ /* * Save all registers and setup segment registers * with kernel selectors. */ INTR_PUSH INTGATE_INIT_KERNEL_FLAGS TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) TRACE_REGS(%edi, %esp, %ebx, %ecx) TRACE_STAMP(%edi) movl %esp, %ebp pushl %ebp call av_dispatch_nmivect addl $4, %esp INTR_POP_USER IRET SET_SIZE(nmiint) #endif /* __i386 */ /* * #BP */ ENTRY_NP(brktrap) #if defined(__amd64) XPV_TRAP_POP cmpw $KCS_SEL, 8(%rsp) jne bp_user /* * This is a breakpoint in the kernel -- it is very likely that this * is DTrace-induced. To unify DTrace handling, we spoof this as an * invalid opcode (#UD) fault. Note that #BP is a trap, not a fault -- * we must decrement the trapping %rip to make it appear as a fault. * We then push a non-zero error code to indicate that this is coming * from #BP. */ decq (%rsp) push $1 /* error code -- non-zero for #BP */ jmp ud_kernel bp_user: #endif /* __amd64 */ NPTRAP_NOERR(T_BPTFLT) /* $3 */ jmp dtrace_trap SET_SIZE(brktrap) /* * #OF */ ENTRY_NP(ovflotrap) TRAP_NOERR(T_OVFLW) /* $4 */ jmp cmntrap SET_SIZE(ovflotrap) /* * #BR */ ENTRY_NP(boundstrap) TRAP_NOERR(T_BOUNDFLT) /* $5 */ jmp cmntrap SET_SIZE(boundstrap) #if defined(__amd64) ENTRY_NP(invoptrap) XPV_TRAP_POP cmpw $KCS_SEL, 8(%rsp) jne ud_user #if defined(__xpv) movb $0, 12(%rsp) /* clear saved upcall_mask from %cs */ #endif push $0 /* error code -- zero for #UD */ ud_kernel: push $0xdddd /* a dummy trap number */ INTR_PUSH movq REGOFF_RIP(%rsp), %rdi movq REGOFF_RSP(%rsp), %rsi movq REGOFF_RAX(%rsp), %rdx pushq (%rsi) movq %rsp, %rsi subq $8, %rsp call dtrace_invop ALTENTRY(dtrace_invop_callsite) addq $16, %rsp cmpl $DTRACE_INVOP_PUSHL_EBP, %eax je ud_push cmpl $DTRACE_INVOP_LEAVE, %eax je ud_leave cmpl $DTRACE_INVOP_NOP, %eax je ud_nop cmpl $DTRACE_INVOP_RET, %eax je ud_ret jmp ud_trap ud_push: /* * We must emulate a "pushq %rbp". To do this, we pull the stack * down 8 bytes, and then store the base pointer. */ INTR_POP subq $16, %rsp /* make room for %rbp */ pushq %rax /* push temp */ movq 24(%rsp), %rax /* load calling RIP */ addq $1, %rax /* increment over trapping instr */ movq %rax, 8(%rsp) /* store calling RIP */ movq 32(%rsp), %rax /* load calling CS */ movq %rax, 16(%rsp) /* store calling CS */ movq 40(%rsp), %rax /* load calling RFLAGS */ movq %rax, 24(%rsp) /* store calling RFLAGS */ movq 48(%rsp), %rax /* load calling RSP */ subq $8, %rax /* make room for %rbp */ movq %rax, 32(%rsp) /* store calling RSP */ movq 56(%rsp), %rax /* load calling SS */ movq %rax, 40(%rsp) /* store calling SS */ movq 32(%rsp), %rax /* reload calling RSP */ movq %rbp, (%rax) /* store %rbp there */ popq %rax /* pop off temp */ IRET /* return from interrupt */ /*NOTREACHED*/ ud_leave: /* * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" * followed by a "popq %rbp". This is quite a bit simpler on amd64 * than it is on i386 -- we can exploit the fact that the %rsp is * explicitly saved to effect the pop without having to reshuffle * the other data pushed for the trap. */ INTR_POP pushq %rax /* push temp */ movq 8(%rsp), %rax /* load calling RIP */ addq $1, %rax /* increment over trapping instr */ movq %rax, 8(%rsp) /* store calling RIP */ movq (%rbp), %rax /* get new %rbp */ addq $8, %rbp /* adjust new %rsp */ movq %rbp, 32(%rsp) /* store new %rsp */ movq %rax, %rbp /* set new %rbp */ popq %rax /* pop off temp */ IRET /* return from interrupt */ /*NOTREACHED*/ ud_nop: /* * We must emulate a "nop". This is obviously not hard: we need only * advance the %rip by one. */ INTR_POP incq (%rsp) IRET /*NOTREACHED*/ ud_ret: INTR_POP pushq %rax /* push temp */ movq 32(%rsp), %rax /* load %rsp */ movq (%rax), %rax /* load calling RIP */ movq %rax, 8(%rsp) /* store calling RIP */ addq $8, 32(%rsp) /* adjust new %rsp */ popq %rax /* pop off temp */ IRET /* return from interrupt */ /*NOTREACHED*/ ud_trap: /* * We're going to let the kernel handle this as a normal #UD. If, * however, we came through #BP and are spoofing #UD (in this case, * the stored error value will be non-zero), we need to de-spoof * the trap by incrementing %rip and pushing T_BPTFLT. */ cmpq $0, REGOFF_ERR(%rsp) je ud_ud incq REGOFF_RIP(%rsp) addq $REGOFF_RIP, %rsp NPTRAP_NOERR(T_BPTFLT) /* $3 */ jmp cmntrap ud_ud: addq $REGOFF_RIP, %rsp ud_user: NPTRAP_NOERR(T_ILLINST) jmp cmntrap SET_SIZE(invoptrap) #elif defined(__i386) /* * #UD */ ENTRY_NP(invoptrap) /* * If we are taking an invalid opcode trap while in the kernel, this * is likely an FBT probe point. */ pushl %gs cmpw $KGS_SEL, (%esp) jne 8f addl $4, %esp #if defined(__xpv) movb $0, 6(%esp) /* clear saved upcall_mask from %cs */ #endif /* __xpv */ pusha pushl %eax /* push %eax -- may be return value */ pushl %esp /* push stack pointer */ addl $48, (%esp) /* adjust to incoming args */ pushl 40(%esp) /* push calling EIP */ call dtrace_invop ALTENTRY(dtrace_invop_callsite) addl $12, %esp cmpl $DTRACE_INVOP_PUSHL_EBP, %eax je 1f cmpl $DTRACE_INVOP_POPL_EBP, %eax je 2f cmpl $DTRACE_INVOP_LEAVE, %eax je 3f cmpl $DTRACE_INVOP_NOP, %eax je 4f jmp 7f 1: /* * We must emulate a "pushl %ebp". To do this, we pull the stack * down 4 bytes, and then store the base pointer. */ popa subl $4, %esp /* make room for %ebp */ pushl %eax /* push temp */ movl 8(%esp), %eax /* load calling EIP */ incl %eax /* increment over LOCK prefix */ movl %eax, 4(%esp) /* store calling EIP */ movl 12(%esp), %eax /* load calling CS */ movl %eax, 8(%esp) /* store calling CS */ movl 16(%esp), %eax /* load calling EFLAGS */ movl %eax, 12(%esp) /* store calling EFLAGS */ movl %ebp, 16(%esp) /* push %ebp */ popl %eax /* pop off temp */ jmp _emul_done 2: /* * We must emulate a "popl %ebp". To do this, we do the opposite of * the above: we remove the %ebp from the stack, and squeeze up the * saved state from the trap. */ popa pushl %eax /* push temp */ movl 16(%esp), %ebp /* pop %ebp */ movl 12(%esp), %eax /* load calling EFLAGS */ movl %eax, 16(%esp) /* store calling EFLAGS */ movl 8(%esp), %eax /* load calling CS */ movl %eax, 12(%esp) /* store calling CS */ movl 4(%esp), %eax /* load calling EIP */ incl %eax /* increment over LOCK prefix */ movl %eax, 8(%esp) /* store calling EIP */ popl %eax /* pop off temp */ addl $4, %esp /* adjust stack pointer */ jmp _emul_done 3: /* * We must emulate a "leave", which is the same as a "movl %ebp, %esp" * followed by a "popl %ebp". This looks similar to the above, but * requires two temporaries: one for the new base pointer, and one * for the staging register. */ popa pushl %eax /* push temp */ pushl %ebx /* push temp */ movl %ebp, %ebx /* set temp to old %ebp */ movl (%ebx), %ebp /* pop %ebp */ movl 16(%esp), %eax /* load calling EFLAGS */ movl %eax, (%ebx) /* store calling EFLAGS */ movl 12(%esp), %eax /* load calling CS */ movl %eax, -4(%ebx) /* store calling CS */ movl 8(%esp), %eax /* load calling EIP */ incl %eax /* increment over LOCK prefix */ movl %eax, -8(%ebx) /* store calling EIP */ movl %ebx, -4(%esp) /* temporarily store new %esp */ popl %ebx /* pop off temp */ popl %eax /* pop off temp */ movl -12(%esp), %esp /* set stack pointer */ subl $8, %esp /* adjust for three pushes, one pop */ jmp _emul_done 4: /* * We must emulate a "nop". This is obviously not hard: we need only * advance the %eip by one. */ popa incl (%esp) _emul_done: IRET /* return from interrupt */ 7: popa pushl $0 pushl $T_ILLINST /* $6 */ jmp cmntrap 8: addl $4, %esp pushl $0 pushl $T_ILLINST /* $6 */ jmp cmntrap SET_SIZE(invoptrap) #endif /* __i386 */ #if defined(__amd64) /* * #NM */ #if defined(__xpv) ENTRY_NP(ndptrap) /* * (On the hypervisor we must make a hypercall so we might as well * save everything and handle as in a normal trap.) */ TRAP_NOERR(T_NOEXTFLT) /* $7 */ INTR_PUSH /* * We want to do this quickly as every lwp using fp will take this * after a context switch -- we do the frequent path in ndptrap_frstor * below; for all other cases, we let the trap code handle it */ LOADCPU(%rax) /* swapgs handled in hypervisor */ cmpl $0, fpu_exists(%rip) je .handle_in_trap /* let trap handle no fp case */ movq CPU_THREAD(%rax), %rbx /* %rbx = curthread */ movl $FPU_EN, %eax movq T_LWP(%rbx), %rbx /* %rbx = lwp */ testq %rbx, %rbx jz .handle_in_trap /* should not happen? */ #if LWP_PCB_FPU != 0 addq $LWP_PCB_FPU, %rbx /* &lwp->lwp_pcb.pcb_fpu */ #endif testl %eax, PCB_FPU_FLAGS(%rbx) jz .handle_in_trap /* must be the first fault */ CLTS andl $_BITNOT(FPU_VALID), PCB_FPU_FLAGS(%rbx) #if FPU_CTX_FPU_REGS != 0 addq $FPU_CTX_FPU_REGS, %rbx #endif movl FPU_CTX_FPU_XSAVE_MASK(%rbx), %eax /* for xrstor */ movl FPU_CTX_FPU_XSAVE_MASK+4(%rbx), %edx /* for xrstor */ /* * the label below is used in trap.c to detect FP faults in * kernel due to user fault. */ ALTENTRY(ndptrap_frstor) .globl _patch_xrstorq_rbx _patch_xrstorq_rbx: FXRSTORQ ((%rbx)) cmpw $KCS_SEL, REGOFF_CS(%rsp) je .return_to_kernel ASSERT_UPCALL_MASK_IS_SET USER_POP IRET /* return to user mode */ /*NOTREACHED*/ .return_to_kernel: INTR_POP IRET /*NOTREACHED*/ .handle_in_trap: INTR_POP pushq $0 /* can not use TRAP_NOERR */ pushq $T_NOEXTFLT jmp cmninttrap SET_SIZE(ndptrap_frstor) SET_SIZE(ndptrap) #else /* __xpv */ ENTRY_NP(ndptrap) /* * We want to do this quickly as every lwp using fp will take this * after a context switch -- we do the frequent path in ndptrap_frstor * below; for all other cases, we let the trap code handle it */ pushq %rax pushq %rbx cmpw $KCS_SEL, 24(%rsp) /* did we come from kernel mode? */ jne 1f LOADCPU(%rax) /* if yes, don't swapgs */ jmp 2f 1: SWAPGS /* if from user, need swapgs */ LOADCPU(%rax) SWAPGS 2: /* * Xrstor needs to use edx as part of its flag. * NOTE: have to push rdx after "cmpw ...24(%rsp)", otherwise rsp+$24 * will not point to CS. */ pushq %rdx cmpl $0, fpu_exists(%rip) je .handle_in_trap /* let trap handle no fp case */ movq CPU_THREAD(%rax), %rbx /* %rbx = curthread */ movl $FPU_EN, %eax movq T_LWP(%rbx), %rbx /* %rbx = lwp */ testq %rbx, %rbx jz .handle_in_trap /* should not happen? */ #if LWP_PCB_FPU != 0 addq $LWP_PCB_FPU, %rbx /* &lwp->lwp_pcb.pcb_fpu */ #endif testl %eax, PCB_FPU_FLAGS(%rbx) jz .handle_in_trap /* must be the first fault */ clts andl $_BITNOT(FPU_VALID), PCB_FPU_FLAGS(%rbx) #if FPU_CTX_FPU_REGS != 0 addq $FPU_CTX_FPU_REGS, %rbx #endif movl FPU_CTX_FPU_XSAVE_MASK(%rbx), %eax /* for xrstor */ movl FPU_CTX_FPU_XSAVE_MASK+4(%rbx), %edx /* for xrstor */ /* * the label below is used in trap.c to detect FP faults in * kernel due to user fault. */ ALTENTRY(ndptrap_frstor) .globl _patch_xrstorq_rbx _patch_xrstorq_rbx: FXRSTORQ ((%rbx)) popq %rdx popq %rbx popq %rax IRET /*NOTREACHED*/ .handle_in_trap: popq %rdx popq %rbx popq %rax TRAP_NOERR(T_NOEXTFLT) /* $7 */ jmp cmninttrap SET_SIZE(ndptrap_frstor) SET_SIZE(ndptrap) #endif /* __xpv */ #elif defined(__i386) ENTRY_NP(ndptrap) /* * We want to do this quickly as every lwp using fp will take this * after a context switch -- we do the frequent path in fpnoextflt * below; for all other cases, we let the trap code handle it */ pushl %eax pushl %ebx pushl %edx /* for xrstor */ pushl %ds pushl %gs movl $KDS_SEL, %ebx movw %bx, %ds movl $KGS_SEL, %eax movw %ax, %gs LOADCPU(%eax) cmpl $0, fpu_exists je .handle_in_trap /* let trap handle no fp case */ movl CPU_THREAD(%eax), %ebx /* %ebx = curthread */ movl $FPU_EN, %eax movl T_LWP(%ebx), %ebx /* %ebx = lwp */ testl %ebx, %ebx jz .handle_in_trap /* should not happen? */ #if LWP_PCB_FPU != 0 addl $LWP_PCB_FPU, %ebx /* &lwp->lwp_pcb.pcb_fpu */ #endif testl %eax, PCB_FPU_FLAGS(%ebx) jz .handle_in_trap /* must be the first fault */ CLTS andl $_BITNOT(FPU_VALID), PCB_FPU_FLAGS(%ebx) #if FPU_CTX_FPU_REGS != 0 addl $FPU_CTX_FPU_REGS, %ebx #endif movl FPU_CTX_FPU_XSAVE_MASK(%ebx), %eax /* for xrstor */ movl FPU_CTX_FPU_XSAVE_MASK+4(%ebx), %edx /* for xrstor */ /* * the label below is used in trap.c to detect FP faults in kernel * due to user fault. */ ALTENTRY(ndptrap_frstor) .globl _patch_fxrstor_ebx _patch_fxrstor_ebx: .globl _patch_xrstor_ebx _patch_xrstor_ebx: frstor (%ebx) /* may be patched to fxrstor */ nop /* (including this byte) */ popl %gs popl %ds popl %edx popl %ebx popl %eax IRET .handle_in_trap: popl %gs popl %ds popl %edx popl %ebx popl %eax TRAP_NOERR(T_NOEXTFLT) /* $7 */ jmp cmninttrap SET_SIZE(ndptrap_frstor) SET_SIZE(ndptrap) #endif /* __i386 */ #if !defined(__xpv) #if defined(__amd64) /* * #DF */ ENTRY_NP(syserrtrap) pushq $T_DBLFLT SET_CPU_GSBASE /* * We share this handler with kmdb (if kmdb is loaded). As such, we * may have reached this point after encountering a #df in kmdb. If * that happens, we'll still be on kmdb's IDT. We need to switch back * to this CPU's IDT before proceeding. Furthermore, if we did arrive * here from kmdb, kmdb is probably in a very sickly state, and * shouldn't be entered from the panic flow. We'll suppress that * entry by setting nopanicdebug. */ pushq %rax subq $DESCTBR_SIZE, %rsp sidt (%rsp) movq %gs:CPU_IDT, %rax cmpq %rax, DTR_BASE(%rsp) je 1f movq %rax, DTR_BASE(%rsp) movw $_MUL(NIDT, GATE_DESC_SIZE), DTR_LIMIT(%rsp) lidt (%rsp) movl $1, nopanicdebug 1: addq $DESCTBR_SIZE, %rsp popq %rax DFTRAP_PUSH /* * freeze trap trace. */ #ifdef TRAPTRACE leaq trap_trace_freeze(%rip), %r11 incl (%r11) #endif ENABLE_INTR_FLAGS movq %rsp, %rdi /* ®s */ xorl %esi, %esi /* clear address */ xorl %edx, %edx /* cpuid = 0 */ call trap SET_SIZE(syserrtrap) #elif defined(__i386) /* * #DF */ ENTRY_NP(syserrtrap) cli /* disable interrupts */ /* * We share this handler with kmdb (if kmdb is loaded). As such, we * may have reached this point after encountering a #df in kmdb. If * that happens, we'll still be on kmdb's IDT. We need to switch back * to this CPU's IDT before proceeding. Furthermore, if we did arrive * here from kmdb, kmdb is probably in a very sickly state, and * shouldn't be entered from the panic flow. We'll suppress that * entry by setting nopanicdebug. */ subl $DESCTBR_SIZE, %esp movl %gs:CPU_IDT, %eax sidt (%esp) cmpl DTR_BASE(%esp), %eax je 1f movl %eax, DTR_BASE(%esp) movw $_MUL(NIDT, GATE_DESC_SIZE), DTR_LIMIT(%esp) lidt (%esp) movl $1, nopanicdebug 1: addl $DESCTBR_SIZE, %esp /* * Check the CPL in the TSS to see what mode * (user or kernel) we took the fault in. At this * point we are running in the context of the double * fault task (dftss) but the CPU's task points to * the previous task (ktss) where the process context * has been saved as the result of the task switch. */ movl %gs:CPU_TSS, %eax /* get the TSS */ movl TSS_SS(%eax), %ebx /* save the fault SS */ movl TSS_ESP(%eax), %edx /* save the fault ESP */ testw $CPL_MASK, TSS_CS(%eax) /* user mode ? */ jz make_frame movw TSS_SS0(%eax), %ss /* get on the kernel stack */ movl TSS_ESP0(%eax), %esp /* * Clear the NT flag to avoid a task switch when the process * finally pops the EFL off the stack via an iret. Clear * the TF flag since that is what the processor does for * a normal exception. Clear the IE flag so that interrupts * remain disabled. */ movl TSS_EFL(%eax), %ecx andl $_BITNOT(PS_NT|PS_T|PS_IE), %ecx pushl %ecx popfl /* restore the EFL */ movw TSS_LDT(%eax), %cx /* restore the LDT */ lldt %cx /* * Restore process segment selectors. */ movw TSS_DS(%eax), %ds movw TSS_ES(%eax), %es movw TSS_FS(%eax), %fs movw TSS_GS(%eax), %gs /* * Restore task segment selectors. */ movl $KDS_SEL, TSS_DS(%eax) movl $KDS_SEL, TSS_ES(%eax) movl $KDS_SEL, TSS_SS(%eax) movl $KFS_SEL, TSS_FS(%eax) movl $KGS_SEL, TSS_GS(%eax) /* * Clear the TS bit, the busy bits in both task * descriptors, and switch tasks. */ clts leal gdt0, %ecx movl DFTSS_SEL+4(%ecx), %esi andl $_BITNOT(0x200), %esi movl %esi, DFTSS_SEL+4(%ecx) movl KTSS_SEL+4(%ecx), %esi andl $_BITNOT(0x200), %esi movl %esi, KTSS_SEL+4(%ecx) movw $KTSS_SEL, %cx ltr %cx /* * Restore part of the process registers. */ movl TSS_EBP(%eax), %ebp movl TSS_ECX(%eax), %ecx movl TSS_ESI(%eax), %esi movl TSS_EDI(%eax), %edi make_frame: /* * Make a trap frame. Leave the error code (0) on * the stack since the first word on a trap stack is * unused anyway. */ pushl %ebx / fault SS pushl %edx / fault ESP pushl TSS_EFL(%eax) / fault EFL pushl TSS_CS(%eax) / fault CS pushl TSS_EIP(%eax) / fault EIP pushl $0 / error code pushl $T_DBLFLT / trap number 8 movl TSS_EBX(%eax), %ebx / restore EBX movl TSS_EDX(%eax), %edx / restore EDX movl TSS_EAX(%eax), %eax / restore EAX sti / enable interrupts jmp cmntrap SET_SIZE(syserrtrap) #endif /* __i386 */ #endif /* !__xpv */ ENTRY_NP(overrun) push $0 TRAP_NOERR(T_EXTOVRFLT) /* $9 i386 only - not generated */ jmp cmninttrap SET_SIZE(overrun) /* * #TS */ ENTRY_NP(invtsstrap) TRAP_ERR(T_TSSFLT) /* $10 already have error code on stack */ jmp cmntrap SET_SIZE(invtsstrap) /* * #NP */ ENTRY_NP(segnptrap) TRAP_ERR(T_SEGFLT) /* $11 already have error code on stack */ #if defined(__amd64) SET_CPU_GSBASE #endif jmp cmntrap SET_SIZE(segnptrap) /* * #SS */ ENTRY_NP(stktrap) TRAP_ERR(T_STKFLT) /* $12 already have error code on stack */ jmp cmntrap SET_SIZE(stktrap) /* * #GP */ ENTRY_NP(gptrap) TRAP_ERR(T_GPFLT) /* $13 already have error code on stack */ #if defined(__amd64) SET_CPU_GSBASE #endif jmp cmntrap SET_SIZE(gptrap) /* * #PF */ ENTRY_NP(pftrap) TRAP_ERR(T_PGFLT) /* $14 already have error code on stack */ INTR_PUSH #if defined(__xpv) #if defined(__amd64) movq %gs:CPU_VCPU_INFO, %r15 movq VCPU_INFO_ARCH_CR2(%r15), %r15 /* vcpu[].arch.cr2 */ #elif defined(__i386) movl %gs:CPU_VCPU_INFO, %esi movl VCPU_INFO_ARCH_CR2(%esi), %esi /* vcpu[].arch.cr2 */ #endif /* __i386 */ #else /* __xpv */ #if defined(__amd64) movq %cr2, %r15 #elif defined(__i386) movl %cr2, %esi #endif /* __i386 */ #endif /* __xpv */ jmp cmntrap_pushed SET_SIZE(pftrap) #if !defined(__amd64) .globl idt0_default_r /* * #PF pentium bug workaround */ ENTRY_NP(pentium_pftrap) pushl %eax movl %cr2, %eax andl $MMU_STD_PAGEMASK, %eax cmpl %eax, %cs:idt0_default_r+2 /* fixme */ je check_for_user_address user_mode: popl %eax pushl $T_PGFLT /* $14 */ jmp cmntrap check_for_user_address: /* * Before we assume that we have an unmapped trap on our hands, * check to see if this is a fault from user mode. If it is, * we'll kick back into the page fault handler. */ movl 4(%esp), %eax /* error code */ andl $PF_ERR_USER, %eax jnz user_mode /* * We now know that this is the invalid opcode trap. */ popl %eax addl $4, %esp /* pop error code */ jmp invoptrap SET_SIZE(pentium_pftrap) #endif /* !__amd64 */ ENTRY_NP(resvtrap) TRAP_NOERR(15) /* (reserved) */ jmp cmntrap SET_SIZE(resvtrap) /* * #MF */ ENTRY_NP(ndperr) TRAP_NOERR(T_EXTERRFLT) /* $16 */ jmp cmninttrap SET_SIZE(ndperr) /* * #AC */ ENTRY_NP(achktrap) TRAP_ERR(T_ALIGNMENT) /* $17 */ jmp cmntrap SET_SIZE(achktrap) /* * #MC */ .globl cmi_mca_trap /* see uts/i86pc/os/cmi.c */ #if defined(__amd64) ENTRY_NP(mcetrap) TRAP_NOERR(T_MCE) /* $18 */ SET_CPU_GSBASE INTR_PUSH INTGATE_INIT_KERNEL_FLAGS TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) TRACE_REGS(%rdi, %rsp, %rbx, %rcx) TRACE_STAMP(%rdi) movq %rsp, %rbp movq %rsp, %rdi /* arg0 = struct regs *rp */ call cmi_mca_trap /* cmi_mca_trap(rp); */ jmp _sys_rtt SET_SIZE(mcetrap) #else ENTRY_NP(mcetrap) TRAP_NOERR(T_MCE) /* $18 */ INTR_PUSH INTGATE_INIT_KERNEL_FLAGS TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) TRACE_REGS(%edi, %esp, %ebx, %ecx) TRACE_STAMP(%edi) movl %esp, %ebp movl %esp, %ecx pushl %ecx /* arg0 = struct regs *rp */ call cmi_mca_trap /* cmi_mca_trap(rp) */ addl $4, %esp /* pop arg0 */ jmp _sys_rtt SET_SIZE(mcetrap) #endif /* * #XF */ ENTRY_NP(xmtrap) TRAP_NOERR(T_SIMDFPE) /* $19 */ jmp cmninttrap SET_SIZE(xmtrap) ENTRY_NP(invaltrap) TRAP_NOERR(30) /* very invalid */ jmp cmntrap SET_SIZE(invaltrap) ENTRY_NP(invalint) TRAP_NOERR(31) /* even more so */ jmp cmnint SET_SIZE(invalint) .globl fasttable #if defined(__amd64) ENTRY_NP(fasttrap) cmpl $T_LASTFAST, %eax ja 1f orl %eax, %eax /* (zero extend top 32-bits) */ leaq fasttable(%rip), %r11 leaq (%r11, %rax, CLONGSIZE), %r11 jmp *(%r11) 1: /* * Fast syscall number was illegal. Make it look * as if the INT failed. Modify %rip to point before the * INT, push the expected error code and fake a GP fault. * * XXX Why make the error code be offset into idt + 1? * Instead we should push a real (soft?) error code * on the stack and #gp handler could know about fasttraps? */ XPV_TRAP_POP subq $2, (%rsp) /* XXX int insn 2-bytes */ pushq $_CONST(_MUL(T_FASTTRAP, GATE_DESC_SIZE) + 2) #if defined(__xpv) pushq %r11 pushq %rcx #endif jmp gptrap SET_SIZE(fasttrap) #elif defined(__i386) ENTRY_NP(fasttrap) cmpl $T_LASTFAST, %eax ja 1f jmp *%cs:fasttable(, %eax, CLONGSIZE) 1: /* * Fast syscall number was illegal. Make it look * as if the INT failed. Modify %eip to point before the * INT, push the expected error code and fake a GP fault. * * XXX Why make the error code be offset into idt + 1? * Instead we should push a real (soft?) error code * on the stack and #gp handler could know about fasttraps? */ subl $2, (%esp) /* XXX int insn 2-bytes */ pushl $_CONST(_MUL(T_FASTTRAP, GATE_DESC_SIZE) + 2) jmp gptrap SET_SIZE(fasttrap) #endif /* __i386 */ ENTRY_NP(dtrace_ret) TRAP_NOERR(T_DTRACE_RET) jmp dtrace_trap SET_SIZE(dtrace_ret) #if defined(__amd64) /* * RFLAGS 24 bytes up the stack from %rsp. * XXX a constant would be nicer. */ ENTRY_NP(fast_null) XPV_TRAP_POP orq $PS_C, 24(%rsp) /* set carry bit in user flags */ IRET /*NOTREACHED*/ SET_SIZE(fast_null) #elif defined(__i386) ENTRY_NP(fast_null) orw $PS_C, 8(%esp) /* set carry bit in user flags */ IRET SET_SIZE(fast_null) #endif /* __i386 */ /* * Interrupts start at 32 */ #define MKIVCT(n) \ ENTRY_NP(ivct/**/n) \ push $0; \ push $n - 0x20; \ jmp cmnint; \ SET_SIZE(ivct/**/n) MKIVCT(32) MKIVCT(33) MKIVCT(34) MKIVCT(35) MKIVCT(36) MKIVCT(37) MKIVCT(38) MKIVCT(39) MKIVCT(40) MKIVCT(41) MKIVCT(42) MKIVCT(43) MKIVCT(44) MKIVCT(45) MKIVCT(46) MKIVCT(47) MKIVCT(48) MKIVCT(49) MKIVCT(50) MKIVCT(51) MKIVCT(52) MKIVCT(53) MKIVCT(54) MKIVCT(55) MKIVCT(56) MKIVCT(57) MKIVCT(58) MKIVCT(59) MKIVCT(60) MKIVCT(61) MKIVCT(62) MKIVCT(63) MKIVCT(64) MKIVCT(65) MKIVCT(66) MKIVCT(67) MKIVCT(68) MKIVCT(69) MKIVCT(70) MKIVCT(71) MKIVCT(72) MKIVCT(73) MKIVCT(74) MKIVCT(75) MKIVCT(76) MKIVCT(77) MKIVCT(78) MKIVCT(79) MKIVCT(80) MKIVCT(81) MKIVCT(82) MKIVCT(83) MKIVCT(84) MKIVCT(85) MKIVCT(86) MKIVCT(87) MKIVCT(88) MKIVCT(89) MKIVCT(90) MKIVCT(91) MKIVCT(92) MKIVCT(93) MKIVCT(94) MKIVCT(95) MKIVCT(96) MKIVCT(97) MKIVCT(98) MKIVCT(99) MKIVCT(100) MKIVCT(101) MKIVCT(102) MKIVCT(103) MKIVCT(104) MKIVCT(105) MKIVCT(106) MKIVCT(107) MKIVCT(108) MKIVCT(109) MKIVCT(110) MKIVCT(111) MKIVCT(112) MKIVCT(113) MKIVCT(114) MKIVCT(115) MKIVCT(116) MKIVCT(117) MKIVCT(118) MKIVCT(119) MKIVCT(120) MKIVCT(121) MKIVCT(122) MKIVCT(123) MKIVCT(124) MKIVCT(125) MKIVCT(126) MKIVCT(127) MKIVCT(128) MKIVCT(129) MKIVCT(130) MKIVCT(131) MKIVCT(132) MKIVCT(133) MKIVCT(134) MKIVCT(135) MKIVCT(136) MKIVCT(137) MKIVCT(138) MKIVCT(139) MKIVCT(140) MKIVCT(141) MKIVCT(142) MKIVCT(143) MKIVCT(144) MKIVCT(145) MKIVCT(146) MKIVCT(147) MKIVCT(148) MKIVCT(149) MKIVCT(150) MKIVCT(151) MKIVCT(152) MKIVCT(153) MKIVCT(154) MKIVCT(155) MKIVCT(156) MKIVCT(157) MKIVCT(158) MKIVCT(159) MKIVCT(160) MKIVCT(161) MKIVCT(162) MKIVCT(163) MKIVCT(164) MKIVCT(165) MKIVCT(166) MKIVCT(167) MKIVCT(168) MKIVCT(169) MKIVCT(170) MKIVCT(171) MKIVCT(172) MKIVCT(173) MKIVCT(174) MKIVCT(175) MKIVCT(176) MKIVCT(177) MKIVCT(178) MKIVCT(179) MKIVCT(180) MKIVCT(181) MKIVCT(182) MKIVCT(183) MKIVCT(184) MKIVCT(185) MKIVCT(186) MKIVCT(187) MKIVCT(188) MKIVCT(189) MKIVCT(190) MKIVCT(191) MKIVCT(192) MKIVCT(193) MKIVCT(194) MKIVCT(195) MKIVCT(196) MKIVCT(197) MKIVCT(198) MKIVCT(199) MKIVCT(200) MKIVCT(201) MKIVCT(202) MKIVCT(203) MKIVCT(204) MKIVCT(205) MKIVCT(206) MKIVCT(207) MKIVCT(208) MKIVCT(209) MKIVCT(210) MKIVCT(211) MKIVCT(212) MKIVCT(213) MKIVCT(214) MKIVCT(215) MKIVCT(216) MKIVCT(217) MKIVCT(218) MKIVCT(219) MKIVCT(220) MKIVCT(221) MKIVCT(222) MKIVCT(223) MKIVCT(224) MKIVCT(225) MKIVCT(226) MKIVCT(227) MKIVCT(228) MKIVCT(229) MKIVCT(230) MKIVCT(231) MKIVCT(232) MKIVCT(233) MKIVCT(234) MKIVCT(235) MKIVCT(236) MKIVCT(237) MKIVCT(238) MKIVCT(239) MKIVCT(240) MKIVCT(241) MKIVCT(242) MKIVCT(243) MKIVCT(244) MKIVCT(245) MKIVCT(246) MKIVCT(247) MKIVCT(248) MKIVCT(249) MKIVCT(250) MKIVCT(251) MKIVCT(252) MKIVCT(253) MKIVCT(254) MKIVCT(255) #endif /* __lint */