1/* 2 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 3 * Use is subject to license terms. 4 */ 5 6/* 7 * Copyright (c) 1989, 1990 William F. Jolitz. 8 * Copyright (c) 1990 The Regents of the University of California. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 * 39 * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.113 2003/10/15 02:04:52 peter Exp $ 40 */ 41 42#pragma ident "%Z%%M% %I% %E% SMI" 43 44#include <sys/asm_linkage.h> 45#include <sys/asm_misc.h> 46#include <sys/trap.h> 47#include <sys/psw.h> 48#include <sys/regset.h> 49#include <sys/privregs.h> 50#include <sys/dtrace.h> 51#include <sys/traptrace.h> 52#include <sys/machparam.h> 53 54/* 55 * only one routine in this file is interesting to lint 56 */ 57 58#if defined(__lint) 59 60void 61ndptrap_frstor(void) 62{} 63 64#else 65 66#include "assym.h" 67 68/* 69 * push $0 on stack for traps that do not 70 * generate an error code. This is so the rest 71 * of the kernel can expect a consistent stack 72 * from from any exception. 73 */ 74#define TRAP_NOERR(trapno) \ 75 push $0; \ 76 push $trapno 77 78/* 79 * error code already pushed by hw 80 * onto stack. 81 */ 82#define TRAP_ERR(trapno) \ 83 push $trapno 84 85 /* 86 * #DE 87 */ 88 ENTRY_NP(div0trap) 89 TRAP_NOERR(T_ZERODIV) /* $0 */ 90 jmp cmntrap 91 SET_SIZE(div0trap) 92 93#if defined(__amd64) 94 /* 95 * #DB 96 * 97 * If we get here as a result of single-stepping a sysenter 98 * instruction, we suddenly find ourselves taking a #db 99 * in kernel mode -before- we've swapgs'ed. So before we can 100 * take the trap, we do the swapgs here, and fix the return 101 * %rip in trap() so that we return immediately after the 102 * swapgs in the sysenter handler to avoid doing the swapgs again. 103 * 104 * Nobody said that the design of sysenter was particularly 105 * elegant, did they? 106 */ 107 ENTRY_NP(dbgtrap) 108 pushq %r11 109 leaq sys_sysenter(%rip), %r11 110 cmpq %r11, 8(%rsp) 111 jne 1f 112 swapgs 1131: popq %r11 114 TRAP_NOERR(T_SGLSTP) /* $1 */ 115 jmp cmntrap 116 SET_SIZE(dbgtrap) 117 118#elif defined(__i386) 119 /* 120 * #DB 121 */ 122 ENTRY_NP(dbgtrap) 123 TRAP_NOERR(T_SGLSTP) /* $1 */ 124 jmp cmntrap 125 SET_SIZE(dbgtrap) 126#endif 127 128#if defined(__amd64) 129 130/* 131 * Macro to set the gsbase or kgsbase to the address of the struct cpu 132 * for this processor. If we came from userland, set kgsbase else clear 133 * gs and set gsbase. We find the proper cpu struct by looping through 134 * the cpu structs for all processors till we find a match for the gdt 135 * of the trapping processor. The stack is expected to be pointing at 136 * The standard regs pushed by hardware on a trap (plus error code and trapno). 137 */ 138#define SET_CPU_GSBASE \ 139 subq $REGOFF_TRAPNO, %rsp; /* save regs */ \ 140 movq %rax, REGOFF_RAX(%rsp); \ 141 movq %rbx, REGOFF_RBX(%rsp); \ 142 movq %rcx, REGOFF_RCX(%rsp); \ 143 movq %rdx, REGOFF_RDX(%rsp); \ 144 movq %rbp, REGOFF_RBP(%rsp); \ 145 movq %rsp, %rbp; \ 146 subq $16, %rsp; /* space for gdt */ \ 147 sgdt 6(%rsp); \ 148 movq 8(%rsp), %rcx; /* %rcx has gdt to match */ \ 149 xorl %ebx, %ebx; /* loop index */ \ 150 leaq cpu(%rip), %rdx; /* cpu pointer array */ \ 1511: \ 152 movq (%rdx, %rbx, CLONGSIZE), %rax; /* get cpu[i] */ \ 153 cmpq $0x0, %rax; /* cpu[i] == NULL ? */ \ 154 je 2f; /* yes, continue */ \ 155 cmpq %rcx, CPU_GDT(%rax); /* gdt == cpu[i]->cpu_gdt ? */ \ 156 je 3f; /* yes, go set gsbase */ \ 1572: \ 158 incl %ebx; /* i++ */ \ 159 cmpl $NCPU, %ebx; /* i < NCPU ? */ \ 160 jb 1b; /* yes, loop */ \ 161/* XXX BIG trouble if we fall thru here. We didn't find a gdt match */ \ 1623: \ 163 movl $MSR_AMD_KGSBASE, %ecx; \ 164 cmpw $KCS_SEL, REGOFF_CS(%rbp); /* trap from kernel? */ \ 165 jne 4f; /* no, go set KGSBASE */ \ 166 movl $MSR_AMD_GSBASE, %ecx; /* yes, set GSBASE */ \ 167 mfence; /* OPTERON_ERRATUM_88 */ \ 1684: \ 169 movq %rax, %rdx; /* write base register */ \ 170 shrq $32, %rdx; \ 171 wrmsr; \ 172 movq REGOFF_RDX(%rbp), %rdx; /* restore regs */ \ 173 movq REGOFF_RCX(%rbp), %rcx; \ 174 movq REGOFF_RBX(%rbp), %rbx; \ 175 movq REGOFF_RAX(%rbp), %rax; \ 176 movq %rbp, %rsp; \ 177 movq REGOFF_RBP(%rsp), %rbp; \ 178 addq $REGOFF_TRAPNO, %rsp /* pop stack */ 179#endif /* __amd64 */ 180 181 182 183 184 .globl nmivect 185 .globl idt0_default_r 186 187#if defined(__amd64) 188 189 /* 190 * #NMI 191 */ 192 ENTRY_NP(nmiint) 193 TRAP_NOERR(T_NMIFLT) /* $2 */ 194 195 SET_CPU_GSBASE 196 197 /* 198 * Save all registers and setup segment registers 199 * with kernel selectors. 200 */ 201 INTR_PUSH 202 203 DISABLE_INTR_FLAGS /* and set the kernel flags */ 204 205 TRACE_PTR(%r12, %rax, %eax, %rdx, $TT_TRAP) 206 207 TRACE_REGS(%r12, %rsp, %rax, %rbx) 208 TRACE_STAMP(%r12) 209 210 movq %rsp, %rbp 211 212 movq %rbp, %rdi 213 call av_dispatch_nmivect 214 215 INTR_POP 216 iretq 217 SET_SIZE(nmiint) 218 219#elif defined(__i386) 220 221 /* 222 * #NMI 223 */ 224 ENTRY_NP(nmiint) 225 TRAP_NOERR(T_NMIFLT) /* $2 */ 226 227 /* 228 * Save all registers and setup segment registers 229 * with kernel selectors. 230 */ 231 INTR_PUSH 232 233 /* 234 * setup pointer to reg struct as 2nd argument. 235 */ 236 movl %esp, %ebp 237 pushl %ebp 238 239 DISABLE_INTR_FLAGS 240 241 movl nmivect, %esi /* get autovect structure */ 242loop1: 243 cmpl $0, %esi /* if pointer is null */ 244 je .intr_ret /* we're done */ 245 movl AV_VECTOR(%esi), %edx /* get the interrupt routine */ 246 pushl AV_INTARG1(%esi) /* get argument to interrupt routine */ 247 call *%edx /* call interrupt routine with arg */ 248 addl $4, %esp 249 movl AV_LINK(%esi), %esi /* get next routine on list */ 250 jmp loop1 /* keep looping until end of list */ 251 252.intr_ret: 253 addl $4, %esp /* 'pop' %ebp */ 254 INTR_POP_USER 255 iret 256 SET_SIZE(nmiint) 257 258#endif /* __i386 */ 259 260 /* 261 * #BP 262 */ 263 ENTRY_NP(brktrap) 264#if defined(__amd64) 265 cmpw $KCS_SEL, 8(%rsp) 266 je bp_jmpud 267#endif 268 269 TRAP_NOERR(T_BPTFLT) /* $3 */ 270 jmp dtrace_trap 271 272#if defined(__amd64) 273bp_jmpud: 274 /* 275 * This is a breakpoint in the kernel -- it is very likely that this 276 * is DTrace-induced. To unify DTrace handling, we spoof this as an 277 * invalid opcode (#UD) fault. Note that #BP is a trap, not a fault -- 278 * we must decrement the trapping %rip to make it appear as a fault. 279 * We then push a non-zero error code to indicate that this is coming 280 * from #BP. 281 */ 282 decq (%rsp) 283 push $1 /* error code -- non-zero for #BP */ 284 jmp ud_kernel 285#endif 286 287 SET_SIZE(brktrap) 288 289 /* 290 * #OF 291 */ 292 ENTRY_NP(ovflotrap) 293 TRAP_NOERR(T_OVFLW) /* $4 */ 294 jmp cmntrap 295 SET_SIZE(ovflotrap) 296 297 /* 298 * #BR 299 */ 300 ENTRY_NP(boundstrap) 301 TRAP_NOERR(T_BOUNDFLT) /* $5 */ 302 jmp cmntrap 303 SET_SIZE(boundstrap) 304 305#if defined(__amd64) 306 307 ENTRY_NP(invoptrap) 308 cmpw $KCS_SEL, 8(%rsp) 309 jne ud_user 310 311 push $0 /* error code -- zero for #UD */ 312ud_kernel: 313 push $0xdddd /* a dummy trap number */ 314 TRAP_PUSH 315 movq REGOFF_RIP(%rsp), %rdi 316 movq REGOFF_RSP(%rsp), %rsi 317 movq REGOFF_RAX(%rsp), %rdx 318 pushq (%rsi) 319 movq %rsp, %rsi 320 call dtrace_invop 321 ALTENTRY(dtrace_invop_callsite) 322 addq $8, %rsp 323 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 324 je ud_push 325 cmpl $DTRACE_INVOP_LEAVE, %eax 326 je ud_leave 327 cmpl $DTRACE_INVOP_NOP, %eax 328 je ud_nop 329 cmpl $DTRACE_INVOP_RET, %eax 330 je ud_ret 331 jmp ud_trap 332 333ud_push: 334 /* 335 * We must emulate a "pushq %rbp". To do this, we pull the stack 336 * down 8 bytes, and then store the base pointer. 337 */ 338 INTR_POP 339 subq $16, %rsp /* make room for %rbp */ 340 pushq %rax /* push temp */ 341 movq 24(%rsp), %rax /* load calling RIP */ 342 addq $1, %rax /* increment over trapping instr */ 343 movq %rax, 8(%rsp) /* store calling RIP */ 344 movq 32(%rsp), %rax /* load calling CS */ 345 movq %rax, 16(%rsp) /* store calling CS */ 346 movq 40(%rsp), %rax /* load calling RFLAGS */ 347 movq %rax, 24(%rsp) /* store calling RFLAGS */ 348 movq 48(%rsp), %rax /* load calling RSP */ 349 subq $8, %rax /* make room for %rbp */ 350 movq %rax, 32(%rsp) /* store calling RSP */ 351 movq 56(%rsp), %rax /* load calling SS */ 352 movq %rax, 40(%rsp) /* store calling SS */ 353 movq 32(%rsp), %rax /* reload calling RSP */ 354 movq %rbp, (%rax) /* store %rbp there */ 355 popq %rax /* pop off temp */ 356 iretq /* return from interrupt */ 357 358ud_leave: 359 /* 360 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" 361 * followed by a "popq %rbp". This is quite a bit simpler on amd64 362 * than it is on i386 -- we can exploit the fact that the %rsp is 363 * explicitly saved to effect the pop without having to reshuffle 364 * the other data pushed for the trap. 365 */ 366 INTR_POP 367 pushq %rax /* push temp */ 368 movq 8(%rsp), %rax /* load calling RIP */ 369 addq $1, %rax /* increment over trapping instr */ 370 movq %rax, 8(%rsp) /* store calling RIP */ 371 movq (%rbp), %rax /* get new %rbp */ 372 addq $8, %rbp /* adjust new %rsp */ 373 movq %rbp, 32(%rsp) /* store new %rsp */ 374 movq %rax, %rbp /* set new %rbp */ 375 popq %rax /* pop off temp */ 376 iretq /* return from interrupt */ 377 378ud_nop: 379 /* 380 * We must emulate a "nop". This is obviously not hard: we need only 381 * advance the %rip by one. 382 */ 383 INTR_POP 384 incq (%rsp) 385 iretq 386 387ud_ret: 388 INTR_POP 389 pushq %rax /* push temp */ 390 movq 32(%rsp), %rax /* load %rsp */ 391 movq (%rax), %rax /* load calling RIP */ 392 movq %rax, 8(%rsp) /* store calling RIP */ 393 addq $8, 32(%rsp) /* adjust new %rsp */ 394 popq %rax /* pop off temp */ 395 iretq /* return from interrupt */ 396 397ud_trap: 398 /* 399 * We're going to let the kernel handle this as a normal #UD. If, 400 * however, we came through #BP and are spoofing #UD (in this case, 401 * the stored error value will be non-zero), we need to de-spoof 402 * the trap by incrementing %rip and pushing T_BPTFLT. 403 */ 404 cmpq $0, REGOFF_ERR(%rsp) 405 je ud_ud 406 incq REGOFF_RIP(%rsp) 407 addq $REGOFF_RIP, %rsp 408 TRAP_NOERR(T_BPTFLT) /* $3 */ 409 jmp cmntrap 410 411ud_ud: 412 addq $REGOFF_RIP, %rsp 413ud_user: 414 TRAP_NOERR(T_ILLINST) 415 jmp cmntrap 416 SET_SIZE(invoptrap) 417 418#elif defined(__i386) 419 420 /* 421 * #UD 422 */ 423 ENTRY_NP(invoptrap) 424 /* 425 * If we are taking an invalid opcode trap while in the kernel, this 426 * is likely an FBT probe point. 427 */ 428 pushl %gs 429 cmpw $KGS_SEL, (%esp) 430 jne 8f 431 addl $4, %esp 432 pusha 433 pushl %eax /* push %eax -- may be return value */ 434 pushl %esp /* push stack pointer */ 435 addl $48, (%esp) /* adjust to incoming args */ 436 pushl 40(%esp) /* push calling EIP */ 437 call dtrace_invop 438 ALTENTRY(dtrace_invop_callsite) 439 addl $12, %esp 440 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 441 je 1f 442 cmpl $DTRACE_INVOP_POPL_EBP, %eax 443 je 2f 444 cmpl $DTRACE_INVOP_LEAVE, %eax 445 je 3f 446 cmpl $DTRACE_INVOP_NOP, %eax 447 je 4f 448 jmp 7f 449 4501: 451 /* 452 * We must emulate a "pushl %ebp". To do this, we pull the stack 453 * down 4 bytes, and then store the base pointer. 454 */ 455 popa 456 subl $4, %esp /* make room for %ebp */ 457 pushl %eax /* push temp */ 458 movl 8(%esp), %eax /* load calling EIP */ 459 incl %eax /* increment over LOCK prefix */ 460 movl %eax, 4(%esp) /* store calling EIP */ 461 movl 12(%esp), %eax /* load calling CS */ 462 movl %eax, 8(%esp) /* store calling CS */ 463 movl 16(%esp), %eax /* load calling EFLAGS */ 464 movl %eax, 12(%esp) /* store calling EFLAGS */ 465 movl %ebp, 16(%esp) /* push %ebp */ 466 popl %eax /* pop off temp */ 467 iret /* return from interrupt */ 468 4692: 470 /* 471 * We must emulate a "popl %ebp". To do this, we do the opposite of 472 * the above: we remove the %ebp from the stack, and squeeze up the 473 * saved state from the trap. 474 */ 475 popa 476 pushl %eax /* push temp */ 477 movl 16(%esp), %ebp /* pop %ebp */ 478 movl 12(%esp), %eax /* load calling EFLAGS */ 479 movl %eax, 16(%esp) /* store calling EFLAGS */ 480 movl 8(%esp), %eax /* load calling CS */ 481 movl %eax, 12(%esp) /* store calling CS */ 482 movl 4(%esp), %eax /* load calling EIP */ 483 incl %eax /* increment over LOCK prefix */ 484 movl %eax, 8(%esp) /* store calling EIP */ 485 popl %eax /* pop off temp */ 486 addl $4, %esp /* adjust stack pointer */ 487 iret /* return from interrupt */ 488 4893: 490 /* 491 * We must emulate a "leave", which is the same as a "movl %ebp, %esp" 492 * followed by a "popl %ebp". This looks similar to the above, but 493 * requires two temporaries: one for the new base pointer, and one 494 * for the staging register. 495 */ 496 popa 497 pushl %eax /* push temp */ 498 pushl %ebx /* push temp */ 499 movl %ebp, %ebx /* set temp to old %ebp */ 500 movl (%ebx), %ebp /* pop %ebp */ 501 movl 16(%esp), %eax /* load calling EFLAGS */ 502 movl %eax, (%ebx) /* store calling EFLAGS */ 503 movl 12(%esp), %eax /* load calling CS */ 504 movl %eax, -4(%ebx) /* store calling CS */ 505 movl 8(%esp), %eax /* load calling EIP */ 506 incl %eax /* increment over LOCK prefix */ 507 movl %eax, -8(%ebx) /* store calling EIP */ 508 movl %ebx, -4(%esp) /* temporarily store new %esp */ 509 popl %ebx /* pop off temp */ 510 popl %eax /* pop off temp */ 511 movl -12(%esp), %esp /* set stack pointer */ 512 subl $8, %esp /* adjust for three pushes, one pop */ 513 iret /* return from interrupt */ 514 5154: 516 /* 517 * We must emulate a "nop". This is obviously not hard: we need only 518 * advance the %eip by one. 519 */ 520 popa 521 incl (%esp) 522 iret 523 5247: 525 popa 526 pushl $0 527 pushl $T_ILLINST /* $6 */ 528 jmp cmntrap 5298: 530 addl $4, %esp 531 pushl $0 532 pushl $T_ILLINST /* $6 */ 533 jmp cmntrap 534 SET_SIZE(invoptrap) 535 536#endif /* __i386 */ 537 538#if defined(__amd64) 539 540 /* 541 * #NM 542 */ 543 ENTRY_NP(ndptrap) 544 /* 545 * We want to do this quickly as every lwp using fp will take this 546 * after a context switch -- we do the frequent path in ndptrap_frstor 547 * below; for all other cases, we let the trap code handle it 548 */ 549 pushq %rax 550 pushq %rbx 551 cmpw $KCS_SEL, 24(%rsp) /* did we come from kernel mode? */ 552 jne 1f 553 LOADCPU(%rbx) /* if yes, don't swapgs */ 554 jmp 2f 5551: 556 swapgs /* if from user, need swapgs */ 557 LOADCPU(%rbx) 558 swapgs 5592: 560 cmpl $0, fpu_exists(%rip) 561 je .handle_in_trap /* let trap handle no fp case */ 562 movq CPU_THREAD(%rbx), %rax /* %rax = curthread */ 563 movl $FPU_EN, %ebx 564 movq T_LWP(%rax), %rax /* %rax = lwp */ 565 testq %rax, %rax 566 jz .handle_in_trap /* should not happen? */ 567#if LWP_PCB_FPU != 0 568 addq $LWP_PCB_FPU, %rax /* &lwp->lwp_pcb.pcb_fpu */ 569#endif 570 testl %ebx, PCB_FPU_FLAGS(%rax) 571 jz .handle_in_trap /* must be the first fault */ 572 clts 573 andl $_BITNOT(FPU_VALID), PCB_FPU_FLAGS(%rax) 574#if FPU_CTX_FPU_REGS != 0 575 addq $FPU_CTX_FPU_REGS, %rax 576#endif 577 /* 578 * the label below is used in trap.c to detect FP faults in 579 * kernel due to user fault. 580 */ 581 ALTENTRY(ndptrap_frstor) 582 fxrstor (%rax) 583 popq %rbx 584 popq %rax 585 iretq 586 587.handle_in_trap: 588 popq %rbx 589 popq %rax 590 TRAP_NOERR(T_NOEXTFLT) /* $7 */ 591 jmp cmninttrap 592 SET_SIZE(ndptrap_frstor) 593 SET_SIZE(ndptrap) 594 595#elif defined(__i386) 596 597 ENTRY_NP(ndptrap) 598 /* 599 * We want to do this quickly as every lwp using fp will take this 600 * after a context switch -- we do the frequent path in fpnoextflt 601 * below; for all other cases, we let the trap code handle it 602 */ 603 pushl %eax 604 pushl %ebx 605 pushl %ds 606 pushl %gs 607 movl $KDS_SEL, %ebx 608 movw %bx, %ds 609 movl $KGS_SEL, %eax 610 movw %ax, %gs 611 LOADCPU(%ebx) 612 cmpl $0, fpu_exists 613 je .handle_in_trap /* let trap handle no fp case */ 614 movl CPU_THREAD(%ebx), %eax /* %eax = curthread */ 615 movl $FPU_EN, %ebx 616 movl T_LWP(%eax), %eax /* %eax = lwp */ 617 testl %eax, %eax 618 jz .handle_in_trap /* should not happen? */ 619#if LWP_PCB_FPU != 0 620 addl $LWP_PCB_FPU, %eax /* &lwp->lwp_pcb.pcb_fpu */ 621#endif 622 testl %ebx, PCB_FPU_FLAGS(%eax) 623 jz .handle_in_trap /* must be the first fault */ 624 clts 625 andl $_BITNOT(FPU_VALID), PCB_FPU_FLAGS(%eax) 626#if FPU_CTX_FPU_REGS != 0 627 addl $FPU_CTX_FPU_REGS, %eax 628#endif 629 /* 630 * the label below is used in trap.c to detect FP faults in kernel 631 * due to user fault. 632 */ 633 ALTENTRY(ndptrap_frstor) 634 .globl _patch_fxrstor_eax 635_patch_fxrstor_eax: 636 frstor (%eax) /* may be patched to fxrstor */ 637 nop /* (including this byte) */ 638 popl %gs 639 popl %ds 640 popl %ebx 641 popl %eax 642 iret 643 644.handle_in_trap: 645 popl %gs 646 popl %ds 647 popl %ebx 648 popl %eax 649 pushl $0 650 pushl $T_NOEXTFLT /* $7 */ 651 jmp cmninttrap 652 SET_SIZE(ndptrap_frstor) 653 SET_SIZE(ndptrap) 654 655#endif /* __i386 */ 656 657#if defined(__amd64) 658 659 /* 660 * #DF 661 */ 662 ENTRY_NP(syserrtrap) 663 pushq $T_DBLFLT 664 665 SET_CPU_GSBASE 666 667 /* 668 * We share this handler with kmdb (if kmdb is loaded). As such, we may 669 * have reached this point after encountering a #df in kmdb. If that 670 * happens, we'll still be on kmdb's IDT. We need to switch back to this 671 * CPU's IDT before proceeding. Furthermore, if we did arrive here from 672 * kmdb, kmdb is probably in a very sickly state, and shouldn't be 673 * entered from the panic flow. We'll suppress that entry by setting 674 * nopanicdebug. 675 */ 676 pushq %rax 677 subq $DESCTBR_SIZE, %rsp 678 sidt (%rsp) 679 movq %gs:CPU_IDT, %rax 680 cmpq %rax, DTR_BASE(%rsp) 681 je 1f 682 683 movq %rax, DTR_BASE(%rsp) 684 movw $_MUL(NIDT, GATE_DESC_SIZE), DTR_LIMIT(%rsp) 685 lidt (%rsp) 686 687 movl $1, nopanicdebug 688 6891: addq $DESCTBR_SIZE, %rsp 690 popq %rax 691 692 DFTRAP_PUSH 693 694 /* 695 * freeze trap trace. 696 */ 697#ifdef TRAPTRACE 698 leaq trap_trace_freeze(%rip), %r11 699 incl (%r11) 700#endif 701 702 ENABLE_INTR_FLAGS 703 704 movq %rsp, %rdi /* ®s */ 705 xorl %esi, %esi /* clear address */ 706 xorl %edx, %edx /* cpuid = 0 */ 707 call trap 708 709 SET_SIZE(syserrtrap) 710 711#elif defined(__i386) 712 713 /* 714 * #DF 715 */ 716 ENTRY_NP(syserrtrap) 717 cli /* disable interrupts */ 718 719 /* 720 * We share this handler with kmdb (if kmdb is loaded). As such, we may 721 * have reached this point after encountering a #df in kmdb. If that 722 * happens, we'll still be on kmdb's IDT. We need to switch back to this 723 * CPU's IDT before proceeding. Furthermore, if we did arrive here from 724 * kmdb, kmdb is probably in a very sickly state, and shouldn't be 725 * entered from the panic flow. We'll suppress that entry by setting 726 * nopanicdebug. 727 */ 728 subl $DESCTBR_SIZE, %esp 729 movl %gs:CPU_IDT, %eax 730 sidt (%esp) 731 cmpl DTR_BASE(%esp), %eax 732 je 1f 733 734 movl %eax, DTR_BASE(%esp) 735 movw $_MUL(NIDT, GATE_DESC_SIZE), DTR_LIMIT(%esp) 736 lidt (%esp) 737 738 movl $1, nopanicdebug 739 7401: addl $DESCTBR_SIZE, %esp 741 742 /* 743 * Check the CPL in the TSS to see what mode 744 * (user or kernel) we took the fault in. At this 745 * point we are running in the context of the double 746 * fault task (dftss) but the CPU's task points to 747 * the previous task (ktss) where the process context 748 * has been saved as the result of the task switch. 749 */ 750 movl %gs:CPU_TSS, %eax /* get the TSS */ 751 movl TSS_SS(%eax), %ebx /* save the fault SS */ 752 movl TSS_ESP(%eax), %edx /* save the fault ESP */ 753 testw $CPL_MASK, TSS_CS(%eax) /* user mode ? */ 754 jz make_frame 755 movw TSS_SS0(%eax), %ss /* get on the kernel stack */ 756 movl TSS_ESP0(%eax), %esp 757 758 /* 759 * Clear the NT flag to avoid a task switch when the process 760 * finally pops the EFL off the stack via an iret. Clear 761 * the TF flag since that is what the processor does for 762 * a normal exception. Clear the IE flag so that interrupts 763 * remain disabled. 764 */ 765 movl TSS_EFL(%eax), %ecx 766 andl $_BITNOT(PS_NT|PS_T|PS_IE), %ecx 767 pushl %ecx 768 popfl /* restore the EFL */ 769 movw TSS_LDT(%eax), %cx /* restore the LDT */ 770 lldt %cx 771 772 /* 773 * Restore process segment selectors. 774 */ 775 movw TSS_DS(%eax), %ds 776 movw TSS_ES(%eax), %es 777 movw TSS_FS(%eax), %fs 778 movw TSS_GS(%eax), %gs 779 780 /* 781 * Restore task segment selectors. 782 */ 783 movl $KDS_SEL, TSS_DS(%eax) 784 movl $KDS_SEL, TSS_ES(%eax) 785 movl $KDS_SEL, TSS_SS(%eax) 786 movl $KFS_SEL, TSS_FS(%eax) 787 movl $KGS_SEL, TSS_GS(%eax) 788 789 /* 790 * Clear the TS bit, the busy bits in both task 791 * descriptors, and switch tasks. 792 */ 793 clts 794 leal gdt0, %ecx 795 movl DFTSS_SEL+4(%ecx), %esi 796 andl $_BITNOT(0x200), %esi 797 movl %esi, DFTSS_SEL+4(%ecx) 798 movl KTSS_SEL+4(%ecx), %esi 799 andl $_BITNOT(0x200), %esi 800 movl %esi, KTSS_SEL+4(%ecx) 801 movw $KTSS_SEL, %cx 802 ltr %cx 803 804 /* 805 * Restore part of the process registers. 806 */ 807 movl TSS_EBP(%eax), %ebp 808 movl TSS_ECX(%eax), %ecx 809 movl TSS_ESI(%eax), %esi 810 movl TSS_EDI(%eax), %edi 811 812make_frame: 813 /* 814 * Make a trap frame. Leave the error code (0) on 815 * the stack since the first word on a trap stack is 816 * unused anyway. 817 */ 818 pushl %ebx / fault SS 819 pushl %edx / fault ESP 820 pushl TSS_EFL(%eax) / fault EFL 821 pushl TSS_CS(%eax) / fault CS 822 pushl TSS_EIP(%eax) / fault EIP 823 pushl $0 / error code 824 pushl $T_DBLFLT / trap number 8 825 movl TSS_EBX(%eax), %ebx / restore EBX 826 movl TSS_EDX(%eax), %edx / restore EDX 827 movl TSS_EAX(%eax), %eax / restore EAX 828 sti / enable interrupts 829 jmp cmntrap 830 SET_SIZE(syserrtrap) 831 832#endif /* __i386 */ 833 834 ENTRY_NP(overrun) 835 push $0 836 TRAP_NOERR(T_EXTOVRFLT) /* $9 i386 only - not generated */ 837 jmp cmninttrap 838 SET_SIZE(overrun) 839 840 /* 841 * #TS 842 */ 843 ENTRY_NP(invtsstrap) 844 TRAP_ERR(T_TSSFLT) /* $10 already have error code on stack */ 845 jmp cmntrap 846 SET_SIZE(invtsstrap) 847 848 /* 849 * #NP 850 */ 851 ENTRY_NP(segnptrap) 852 TRAP_ERR(T_SEGFLT) /* $11 already have error code on stack */ 853#if defined(__amd64) 854 SET_CPU_GSBASE 855#endif 856 jmp cmntrap 857 SET_SIZE(segnptrap) 858 859 /* 860 * #SS 861 */ 862 ENTRY_NP(stktrap) 863 TRAP_ERR(T_STKFLT) /* $12 already have error code on stack */ 864 jmp cmntrap 865 SET_SIZE(stktrap) 866 867 /* 868 * #GP 869 */ 870 ENTRY_NP(gptrap) 871 TRAP_ERR(T_GPFLT) /* $13 already have error code on stack */ 872#if defined(__amd64) 873 SET_CPU_GSBASE 874#endif 875 jmp cmntrap 876 SET_SIZE(gptrap) 877 878 /* 879 * #PF 880 */ 881 ENTRY_NP(pftrap) 882 TRAP_ERR(T_PGFLT) /* $14 already have error code on stack */ 883 jmp cmntrap 884 SET_SIZE(pftrap) 885 886#if !defined(__amd64) 887 888 /* 889 * #PF pentium bug workaround 890 */ 891 ENTRY_NP(pentium_pftrap) 892 pushl %eax 893 movl %cr2, %eax 894 andl $MMU_STD_PAGEMASK, %eax 895 896 cmpl %eax, %cs:idt0_default_r+2 /* fixme */ 897 898 je check_for_user_address 899user_mode: 900 popl %eax 901 pushl $T_PGFLT /* $14 */ 902 jmp cmntrap 903check_for_user_address: 904 /* 905 * Before we assume that we have an unmapped trap on our hands, 906 * check to see if this is a fault from user mode. If it is, 907 * we'll kick back into the page fault handler. 908 */ 909 movl 4(%esp), %eax /* error code */ 910 andl $PF_ERR_USER, %eax 911 jnz user_mode 912 913 /* 914 * We now know that this is the invalid opcode trap. 915 */ 916 popl %eax 917 addl $4, %esp /* pop error code */ 918 jmp invoptrap 919 SET_SIZE(pentium_pftrap) 920 921#endif /* !__amd64 */ 922 923 ENTRY_NP(resvtrap) 924 TRAP_NOERR(15) /* (reserved) */ 925 jmp cmntrap 926 SET_SIZE(resvtrap) 927 928 /* 929 * #MF 930 */ 931 ENTRY_NP(ndperr) 932 TRAP_NOERR(T_EXTERRFLT) /* $16 */ 933 jmp cmninttrap 934 SET_SIZE(ndperr) 935 936 /* 937 * #AC 938 */ 939 ENTRY_NP(achktrap) 940 TRAP_ERR(T_ALIGNMENT) /* $17 */ 941 jmp cmntrap 942 SET_SIZE(achktrap) 943 944 /* 945 * #MC 946 */ 947 .globl cmi_mca_trap /* see uts/i86pc/os/cmi.c */ 948 949#if defined(__amd64) 950 951 ENTRY_NP(mcetrap) 952 TRAP_NOERR(T_MCE) /* $18 */ 953 SET_CPU_GSBASE 954 INTR_PUSH 955 956 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) 957 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) 958 TRACE_STAMP(%rdi) 959 960 DISABLE_INTR_FLAGS 961 movq %rsp, %rbp 962 963 movq %rsp, %rdi /* arg0 = struct regs *rp */ 964 call cmi_mca_trap /* cmi_mca_trap(rp); */ 965 966 jmp _sys_rtt 967 SET_SIZE(mcetrap) 968 969#else 970 971 ENTRY_NP(mcetrap) 972 TRAP_NOERR(T_MCE) /* $18 */ 973 INTR_PUSH 974 975 DISABLE_INTR_FLAGS 976 movl %esp, %ebp 977 978 movl %esp, %ecx 979 pushl %ecx /* arg0 = struct regs *rp */ 980 call cmi_mca_trap /* cmi_mca_trap(rp) */ 981 addl $4, %esp /* pop arg0 */ 982 983 jmp _sys_rtt 984 SET_SIZE(mcetrap) 985 986#endif 987 988 /* 989 * #XF 990 */ 991 ENTRY_NP(xmtrap) 992 TRAP_NOERR(T_SIMDFPE) /* $19 */ 993 jmp cmntrap 994 SET_SIZE(xmtrap) 995 996 ENTRY_NP(invaltrap) 997 TRAP_NOERR(30) /* very invalid */ 998 jmp cmntrap 999 SET_SIZE(invaltrap) 1000 1001 ENTRY_NP(invalint) 1002 TRAP_NOERR(31) /* even more so */ 1003 jmp cmnint 1004 SET_SIZE(invalint) 1005 1006 .globl fasttable 1007 1008#if defined(__amd64) 1009 1010 ENTRY_NP(fasttrap) 1011 cmpl $T_LASTFAST, %eax 1012 ja 1f 1013 orl %eax, %eax /* (zero extend top 32-bits) */ 1014 leaq fasttable(%rip), %r11 1015 leaq (%r11, %rax, CLONGSIZE), %r11 1016 jmp *(%r11) 10171: 1018 /* 1019 * Fast syscall number was illegal. Make it look 1020 * as if the INT failed. Modify %rip to point before the 1021 * INT, push the expected error code and fake a GP fault. 1022 * 1023 * XXX Why make the error code be offset into idt + 1? 1024 * Instead we should push a real (soft?) error code 1025 * on the stack and #gp handler could know about fasttraps? 1026 */ 1027 subq $2, (%rsp) /* XXX int insn 2-bytes */ 1028 pushq $_CONST(_MUL(T_FASTTRAP, GATE_DESC_SIZE) + 2) 1029 jmp gptrap 1030 SET_SIZE(fasttrap) 1031 1032#elif defined(__i386) 1033 1034 ENTRY_NP(fasttrap) 1035 cmpl $T_LASTFAST, %eax 1036 ja 1f 1037 jmp *%cs:fasttable(, %eax, CLONGSIZE) 10381: 1039 /* 1040 * Fast syscall number was illegal. Make it look 1041 * as if the INT failed. Modify %eip to point before the 1042 * INT, push the expected error code and fake a GP fault. 1043 * 1044 * XXX Why make the error code be offset into idt + 1? 1045 * Instead we should push a real (soft?) error code 1046 * on the stack and #gp handler could know about fasttraps? 1047 */ 1048 subl $2, (%esp) /* XXX int insn 2-bytes */ 1049 pushl $_CONST(_MUL(T_FASTTRAP, GATE_DESC_SIZE) + 2) 1050 jmp gptrap 1051 SET_SIZE(fasttrap) 1052 1053#endif /* __i386 */ 1054 1055 ENTRY_NP(dtrace_fasttrap) 1056 TRAP_NOERR(T_DTRACE_PROBE) 1057 jmp dtrace_trap 1058 SET_SIZE(dtrace_fasttrap) 1059 1060 ENTRY_NP(dtrace_ret) 1061 TRAP_NOERR(T_DTRACE_RET) 1062 jmp dtrace_trap 1063 SET_SIZE(dtrace_ret) 1064 1065#if defined(__amd64) 1066 1067 /* 1068 * RFLAGS 24 bytes up the stack from %rsp. 1069 * XXX a constant would be nicer. 1070 */ 1071 ENTRY_NP(fast_null) 1072 orq $PS_C, 24(%rsp) /* set carry bit in user flags */ 1073 iretq 1074 SET_SIZE(fast_null) 1075 1076#elif defined(__i386) 1077 1078 ENTRY_NP(fast_null) 1079 orw $PS_C, 8(%esp) /* set carry bit in user flags */ 1080 iret 1081 SET_SIZE(fast_null) 1082 1083#endif /* __i386 */ 1084 1085 /* 1086 * Interrupts start at 32 1087 */ 1088#define MKIVCT(n) \ 1089 ENTRY_NP(ivct/**/n) \ 1090 push $0; \ 1091 push $n - 0x20; \ 1092 jmp cmnint; \ 1093 SET_SIZE(ivct/**/n) 1094 1095 MKIVCT(32) 1096 MKIVCT(33) 1097 MKIVCT(34) 1098 MKIVCT(35) 1099 MKIVCT(36) 1100 MKIVCT(37) 1101 MKIVCT(38) 1102 MKIVCT(39) 1103 MKIVCT(40) 1104 MKIVCT(41) 1105 MKIVCT(42) 1106 MKIVCT(43) 1107 MKIVCT(44) 1108 MKIVCT(45) 1109 MKIVCT(46) 1110 MKIVCT(47) 1111 MKIVCT(48) 1112 MKIVCT(49) 1113 MKIVCT(50) 1114 MKIVCT(51) 1115 MKIVCT(52) 1116 MKIVCT(53) 1117 MKIVCT(54) 1118 MKIVCT(55) 1119 MKIVCT(56) 1120 MKIVCT(57) 1121 MKIVCT(58) 1122 MKIVCT(59) 1123 MKIVCT(60) 1124 MKIVCT(61) 1125 MKIVCT(62) 1126 MKIVCT(63) 1127 MKIVCT(64) 1128 MKIVCT(65) 1129 MKIVCT(66) 1130 MKIVCT(67) 1131 MKIVCT(68) 1132 MKIVCT(69) 1133 MKIVCT(70) 1134 MKIVCT(71) 1135 MKIVCT(72) 1136 MKIVCT(73) 1137 MKIVCT(74) 1138 MKIVCT(75) 1139 MKIVCT(76) 1140 MKIVCT(77) 1141 MKIVCT(78) 1142 MKIVCT(79) 1143 MKIVCT(80) 1144 MKIVCT(81) 1145 MKIVCT(82) 1146 MKIVCT(83) 1147 MKIVCT(84) 1148 MKIVCT(85) 1149 MKIVCT(86) 1150 MKIVCT(87) 1151 MKIVCT(88) 1152 MKIVCT(89) 1153 MKIVCT(90) 1154 MKIVCT(91) 1155 MKIVCT(92) 1156 MKIVCT(93) 1157 MKIVCT(94) 1158 MKIVCT(95) 1159 MKIVCT(96) 1160 MKIVCT(97) 1161 MKIVCT(98) 1162 MKIVCT(99) 1163 MKIVCT(100) 1164 MKIVCT(101) 1165 MKIVCT(102) 1166 MKIVCT(103) 1167 MKIVCT(104) 1168 MKIVCT(105) 1169 MKIVCT(106) 1170 MKIVCT(107) 1171 MKIVCT(108) 1172 MKIVCT(109) 1173 MKIVCT(110) 1174 MKIVCT(111) 1175 MKIVCT(112) 1176 MKIVCT(113) 1177 MKIVCT(114) 1178 MKIVCT(115) 1179 MKIVCT(116) 1180 MKIVCT(117) 1181 MKIVCT(118) 1182 MKIVCT(119) 1183 MKIVCT(120) 1184 MKIVCT(121) 1185 MKIVCT(122) 1186 MKIVCT(123) 1187 MKIVCT(124) 1188 MKIVCT(125) 1189 MKIVCT(126) 1190 MKIVCT(127) 1191 MKIVCT(128) 1192 MKIVCT(129) 1193 MKIVCT(130) 1194 MKIVCT(131) 1195 MKIVCT(132) 1196 MKIVCT(133) 1197 MKIVCT(134) 1198 MKIVCT(135) 1199 MKIVCT(136) 1200 MKIVCT(137) 1201 MKIVCT(138) 1202 MKIVCT(139) 1203 MKIVCT(140) 1204 MKIVCT(141) 1205 MKIVCT(142) 1206 MKIVCT(143) 1207 MKIVCT(144) 1208 MKIVCT(145) 1209 MKIVCT(146) 1210 MKIVCT(147) 1211 MKIVCT(148) 1212 MKIVCT(149) 1213 MKIVCT(150) 1214 MKIVCT(151) 1215 MKIVCT(152) 1216 MKIVCT(153) 1217 MKIVCT(154) 1218 MKIVCT(155) 1219 MKIVCT(156) 1220 MKIVCT(157) 1221 MKIVCT(158) 1222 MKIVCT(159) 1223 MKIVCT(160) 1224 MKIVCT(161) 1225 MKIVCT(162) 1226 MKIVCT(163) 1227 MKIVCT(164) 1228 MKIVCT(165) 1229 MKIVCT(166) 1230 MKIVCT(167) 1231 MKIVCT(168) 1232 MKIVCT(169) 1233 MKIVCT(170) 1234 MKIVCT(171) 1235 MKIVCT(172) 1236 MKIVCT(173) 1237 MKIVCT(174) 1238 MKIVCT(175) 1239 MKIVCT(176) 1240 MKIVCT(177) 1241 MKIVCT(178) 1242 MKIVCT(179) 1243 MKIVCT(180) 1244 MKIVCT(181) 1245 MKIVCT(182) 1246 MKIVCT(183) 1247 MKIVCT(184) 1248 MKIVCT(185) 1249 MKIVCT(186) 1250 MKIVCT(187) 1251 MKIVCT(188) 1252 MKIVCT(189) 1253 MKIVCT(190) 1254 MKIVCT(191) 1255 MKIVCT(192) 1256 MKIVCT(193) 1257 MKIVCT(194) 1258 MKIVCT(195) 1259 MKIVCT(196) 1260 MKIVCT(197) 1261 MKIVCT(198) 1262 MKIVCT(199) 1263 MKIVCT(200) 1264 MKIVCT(201) 1265 MKIVCT(202) 1266 MKIVCT(203) 1267 MKIVCT(204) 1268 MKIVCT(205) 1269 MKIVCT(206) 1270 MKIVCT(207) 1271 MKIVCT(208) 1272 MKIVCT(209) 1273 MKIVCT(210) 1274 MKIVCT(211) 1275 MKIVCT(212) 1276 MKIVCT(213) 1277 MKIVCT(214) 1278 MKIVCT(215) 1279 MKIVCT(216) 1280 MKIVCT(217) 1281 MKIVCT(218) 1282 MKIVCT(219) 1283 MKIVCT(220) 1284 MKIVCT(221) 1285 MKIVCT(222) 1286 MKIVCT(223) 1287 MKIVCT(224) 1288 MKIVCT(225) 1289 MKIVCT(226) 1290 MKIVCT(227) 1291 MKIVCT(228) 1292 MKIVCT(229) 1293 MKIVCT(230) 1294 MKIVCT(231) 1295 MKIVCT(232) 1296 MKIVCT(233) 1297 MKIVCT(234) 1298 MKIVCT(235) 1299 MKIVCT(236) 1300 MKIVCT(237) 1301 MKIVCT(238) 1302 MKIVCT(239) 1303 MKIVCT(240) 1304 MKIVCT(241) 1305 MKIVCT(242) 1306 MKIVCT(243) 1307 MKIVCT(244) 1308 MKIVCT(245) 1309 MKIVCT(246) 1310 MKIVCT(247) 1311 MKIVCT(248) 1312 MKIVCT(249) 1313 MKIVCT(250) 1314 MKIVCT(251) 1315 MKIVCT(252) 1316 MKIVCT(253) 1317 MKIVCT(254) 1318 MKIVCT(255) 1319 1320#endif /* __lint */ 1321