1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 1991,1992 Linus Torvalds 4 * 5 * entry_32.S contains the system-call and low-level fault and trap handling routines. 6 * 7 * Stack layout while running C code: 8 * ptrace needs to have all registers on the stack. 9 * If the order here is changed, it needs to be 10 * updated in fork.c:copy_process(), signal.c:do_signal(), 11 * ptrace.c and ptrace.h 12 * 13 * 0(%esp) - %ebx 14 * 4(%esp) - %ecx 15 * 8(%esp) - %edx 16 * C(%esp) - %esi 17 * 10(%esp) - %edi 18 * 14(%esp) - %ebp 19 * 18(%esp) - %eax 20 * 1C(%esp) - %ds 21 * 20(%esp) - %es 22 * 24(%esp) - %fs 23 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 24 * 2C(%esp) - orig_eax 25 * 30(%esp) - %eip 26 * 34(%esp) - %cs 27 * 38(%esp) - %eflags 28 * 3C(%esp) - %oldesp 29 * 40(%esp) - %oldss 30 */ 31 32#include <linux/linkage.h> 33#include <linux/err.h> 34#include <asm/thread_info.h> 35#include <asm/irqflags.h> 36#include <asm/errno.h> 37#include <asm/segment.h> 38#include <asm/smp.h> 39#include <asm/percpu.h> 40#include <asm/processor-flags.h> 41#include <asm/irq_vectors.h> 42#include <asm/cpufeatures.h> 43#include <asm/alternative-asm.h> 44#include <asm/asm.h> 45#include <asm/smap.h> 46#include <asm/frame.h> 47#include <asm/nospec-branch.h> 48 49#include "calling.h" 50 51 .section .entry.text, "ax" 52 53/* 54 * We use macros for low-level operations which need to be overridden 55 * for paravirtualization. The following will never clobber any registers: 56 * INTERRUPT_RETURN (aka. "iret") 57 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 58 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 59 * 60 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 61 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 62 * Allowing a register to be clobbered can shrink the paravirt replacement 63 * enough to patch inline, increasing performance. 64 */ 65 66#ifdef CONFIG_PREEMPTION 67# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 68#else 69# define preempt_stop(clobbers) 70#endif 71 72.macro TRACE_IRQS_IRET 73#ifdef CONFIG_TRACE_IRQFLAGS 74 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off? 75 jz 1f 76 TRACE_IRQS_ON 771: 78#endif 79.endm 80 81#define PTI_SWITCH_MASK (1 << PAGE_SHIFT) 82 83/* 84 * User gs save/restore 85 * 86 * %gs is used for userland TLS and kernel only uses it for stack 87 * canary which is required to be at %gs:20 by gcc. Read the comment 88 * at the top of stackprotector.h for more info. 89 * 90 * Local labels 98 and 99 are used. 91 */ 92#ifdef CONFIG_X86_32_LAZY_GS 93 94 /* unfortunately push/pop can't be no-op */ 95.macro PUSH_GS 96 pushl $0 97.endm 98.macro POP_GS pop=0 99 addl $(4 + \pop), %esp 100.endm 101.macro POP_GS_EX 102.endm 103 104 /* all the rest are no-op */ 105.macro PTGS_TO_GS 106.endm 107.macro PTGS_TO_GS_EX 108.endm 109.macro GS_TO_REG reg 110.endm 111.macro REG_TO_PTGS reg 112.endm 113.macro SET_KERNEL_GS reg 114.endm 115 116#else /* CONFIG_X86_32_LAZY_GS */ 117 118.macro PUSH_GS 119 pushl %gs 120.endm 121 122.macro POP_GS pop=0 12398: popl %gs 124 .if \pop <> 0 125 add $\pop, %esp 126 .endif 127.endm 128.macro POP_GS_EX 129.pushsection .fixup, "ax" 13099: movl $0, (%esp) 131 jmp 98b 132.popsection 133 _ASM_EXTABLE(98b, 99b) 134.endm 135 136.macro PTGS_TO_GS 13798: mov PT_GS(%esp), %gs 138.endm 139.macro PTGS_TO_GS_EX 140.pushsection .fixup, "ax" 14199: movl $0, PT_GS(%esp) 142 jmp 98b 143.popsection 144 _ASM_EXTABLE(98b, 99b) 145.endm 146 147.macro GS_TO_REG reg 148 movl %gs, \reg 149.endm 150.macro REG_TO_PTGS reg 151 movl \reg, PT_GS(%esp) 152.endm 153.macro SET_KERNEL_GS reg 154 movl $(__KERNEL_STACK_CANARY), \reg 155 movl \reg, %gs 156.endm 157 158#endif /* CONFIG_X86_32_LAZY_GS */ 159 160/* Unconditionally switch to user cr3 */ 161.macro SWITCH_TO_USER_CR3 scratch_reg:req 162 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI 163 164 movl %cr3, \scratch_reg 165 orl $PTI_SWITCH_MASK, \scratch_reg 166 movl \scratch_reg, %cr3 167.Lend_\@: 168.endm 169 170.macro BUG_IF_WRONG_CR3 no_user_check=0 171#ifdef CONFIG_DEBUG_ENTRY 172 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI 173 .if \no_user_check == 0 174 /* coming from usermode? */ 175 testl $USER_SEGMENT_RPL_MASK, PT_CS(%esp) 176 jz .Lend_\@ 177 .endif 178 /* On user-cr3? */ 179 movl %cr3, %eax 180 testl $PTI_SWITCH_MASK, %eax 181 jnz .Lend_\@ 182 /* From userspace with kernel cr3 - BUG */ 183 ud2 184.Lend_\@: 185#endif 186.endm 187 188/* 189 * Switch to kernel cr3 if not already loaded and return current cr3 in 190 * \scratch_reg 191 */ 192.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req 193 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI 194 movl %cr3, \scratch_reg 195 /* Test if we are already on kernel CR3 */ 196 testl $PTI_SWITCH_MASK, \scratch_reg 197 jz .Lend_\@ 198 andl $(~PTI_SWITCH_MASK), \scratch_reg 199 movl \scratch_reg, %cr3 200 /* Return original CR3 in \scratch_reg */ 201 orl $PTI_SWITCH_MASK, \scratch_reg 202.Lend_\@: 203.endm 204 205#define CS_FROM_ENTRY_STACK (1 << 31) 206#define CS_FROM_USER_CR3 (1 << 30) 207#define CS_FROM_KERNEL (1 << 29) 208#define CS_FROM_ESPFIX (1 << 28) 209 210.macro FIXUP_FRAME 211 /* 212 * The high bits of the CS dword (__csh) are used for CS_FROM_*. 213 * Clear them in case hardware didn't do this for us. 214 */ 215 andl $0x0000ffff, 4*4(%esp) 216 217#ifdef CONFIG_VM86 218 testl $X86_EFLAGS_VM, 5*4(%esp) 219 jnz .Lfrom_usermode_no_fixup_\@ 220#endif 221 testl $USER_SEGMENT_RPL_MASK, 4*4(%esp) 222 jnz .Lfrom_usermode_no_fixup_\@ 223 224 orl $CS_FROM_KERNEL, 4*4(%esp) 225 226 /* 227 * When we're here from kernel mode; the (exception) stack looks like: 228 * 229 * 6*4(%esp) - <previous context> 230 * 5*4(%esp) - flags 231 * 4*4(%esp) - cs 232 * 3*4(%esp) - ip 233 * 2*4(%esp) - orig_eax 234 * 1*4(%esp) - gs / function 235 * 0*4(%esp) - fs 236 * 237 * Lets build a 5 entry IRET frame after that, such that struct pt_regs 238 * is complete and in particular regs->sp is correct. This gives us 239 * the original 6 enties as gap: 240 * 241 * 14*4(%esp) - <previous context> 242 * 13*4(%esp) - gap / flags 243 * 12*4(%esp) - gap / cs 244 * 11*4(%esp) - gap / ip 245 * 10*4(%esp) - gap / orig_eax 246 * 9*4(%esp) - gap / gs / function 247 * 8*4(%esp) - gap / fs 248 * 7*4(%esp) - ss 249 * 6*4(%esp) - sp 250 * 5*4(%esp) - flags 251 * 4*4(%esp) - cs 252 * 3*4(%esp) - ip 253 * 2*4(%esp) - orig_eax 254 * 1*4(%esp) - gs / function 255 * 0*4(%esp) - fs 256 */ 257 258 pushl %ss # ss 259 pushl %esp # sp (points at ss) 260 addl $7*4, (%esp) # point sp back at the previous context 261 pushl 7*4(%esp) # flags 262 pushl 7*4(%esp) # cs 263 pushl 7*4(%esp) # ip 264 pushl 7*4(%esp) # orig_eax 265 pushl 7*4(%esp) # gs / function 266 pushl 7*4(%esp) # fs 267.Lfrom_usermode_no_fixup_\@: 268.endm 269 270.macro IRET_FRAME 271 /* 272 * We're called with %ds, %es, %fs, and %gs from the interrupted 273 * frame, so we shouldn't use them. Also, we may be in ESPFIX 274 * mode and therefore have a nonzero SS base and an offset ESP, 275 * so any attempt to access the stack needs to use SS. (except for 276 * accesses through %esp, which automatically use SS.) 277 */ 278 testl $CS_FROM_KERNEL, 1*4(%esp) 279 jz .Lfinished_frame_\@ 280 281 /* 282 * Reconstruct the 3 entry IRET frame right after the (modified) 283 * regs->sp without lowering %esp in between, such that an NMI in the 284 * middle doesn't scribble our stack. 285 */ 286 pushl %eax 287 pushl %ecx 288 movl 5*4(%esp), %eax # (modified) regs->sp 289 290 movl 4*4(%esp), %ecx # flags 291 movl %ecx, %ss:-1*4(%eax) 292 293 movl 3*4(%esp), %ecx # cs 294 andl $0x0000ffff, %ecx 295 movl %ecx, %ss:-2*4(%eax) 296 297 movl 2*4(%esp), %ecx # ip 298 movl %ecx, %ss:-3*4(%eax) 299 300 movl 1*4(%esp), %ecx # eax 301 movl %ecx, %ss:-4*4(%eax) 302 303 popl %ecx 304 lea -4*4(%eax), %esp 305 popl %eax 306.Lfinished_frame_\@: 307.endm 308 309.macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 skip_gs=0 unwind_espfix=0 310 cld 311.if \skip_gs == 0 312 PUSH_GS 313.endif 314 pushl %fs 315 316 pushl %eax 317 movl $(__KERNEL_PERCPU), %eax 318 movl %eax, %fs 319.if \unwind_espfix > 0 320 UNWIND_ESPFIX_STACK 321.endif 322 popl %eax 323 324 FIXUP_FRAME 325 pushl %es 326 pushl %ds 327 pushl \pt_regs_ax 328 pushl %ebp 329 pushl %edi 330 pushl %esi 331 pushl %edx 332 pushl %ecx 333 pushl %ebx 334 movl $(__USER_DS), %edx 335 movl %edx, %ds 336 movl %edx, %es 337.if \skip_gs == 0 338 SET_KERNEL_GS %edx 339.endif 340 /* Switch to kernel stack if necessary */ 341.if \switch_stacks > 0 342 SWITCH_TO_KERNEL_STACK 343.endif 344.endm 345 346.macro SAVE_ALL_NMI cr3_reg:req unwind_espfix=0 347 SAVE_ALL unwind_espfix=\unwind_espfix 348 349 BUG_IF_WRONG_CR3 350 351 /* 352 * Now switch the CR3 when PTI is enabled. 353 * 354 * We can enter with either user or kernel cr3, the code will 355 * store the old cr3 in \cr3_reg and switches to the kernel cr3 356 * if necessary. 357 */ 358 SWITCH_TO_KERNEL_CR3 scratch_reg=\cr3_reg 359 360.Lend_\@: 361.endm 362 363.macro RESTORE_INT_REGS 364 popl %ebx 365 popl %ecx 366 popl %edx 367 popl %esi 368 popl %edi 369 popl %ebp 370 popl %eax 371.endm 372 373.macro RESTORE_REGS pop=0 374 RESTORE_INT_REGS 3751: popl %ds 3762: popl %es 3773: popl %fs 378 POP_GS \pop 379 IRET_FRAME 380.pushsection .fixup, "ax" 3814: movl $0, (%esp) 382 jmp 1b 3835: movl $0, (%esp) 384 jmp 2b 3856: movl $0, (%esp) 386 jmp 3b 387.popsection 388 _ASM_EXTABLE(1b, 4b) 389 _ASM_EXTABLE(2b, 5b) 390 _ASM_EXTABLE(3b, 6b) 391 POP_GS_EX 392.endm 393 394.macro RESTORE_ALL_NMI cr3_reg:req pop=0 395 /* 396 * Now switch the CR3 when PTI is enabled. 397 * 398 * We enter with kernel cr3 and switch the cr3 to the value 399 * stored on \cr3_reg, which is either a user or a kernel cr3. 400 */ 401 ALTERNATIVE "jmp .Lswitched_\@", "", X86_FEATURE_PTI 402 403 testl $PTI_SWITCH_MASK, \cr3_reg 404 jz .Lswitched_\@ 405 406 /* User cr3 in \cr3_reg - write it to hardware cr3 */ 407 movl \cr3_reg, %cr3 408 409.Lswitched_\@: 410 411 BUG_IF_WRONG_CR3 412 413 RESTORE_REGS pop=\pop 414.endm 415 416.macro CHECK_AND_APPLY_ESPFIX 417#ifdef CONFIG_X86_ESPFIX32 418#define GDT_ESPFIX_OFFSET (GDT_ENTRY_ESPFIX_SS * 8) 419#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + GDT_ESPFIX_OFFSET 420 421 ALTERNATIVE "jmp .Lend_\@", "", X86_BUG_ESPFIX 422 423 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 424 /* 425 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we 426 * are returning to the kernel. 427 * See comments in process.c:copy_thread() for details. 428 */ 429 movb PT_OLDSS(%esp), %ah 430 movb PT_CS(%esp), %al 431 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 432 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 433 jne .Lend_\@ # returning to user-space with LDT SS 434 435 /* 436 * Setup and switch to ESPFIX stack 437 * 438 * We're returning to userspace with a 16 bit stack. The CPU will not 439 * restore the high word of ESP for us on executing iret... This is an 440 * "official" bug of all the x86-compatible CPUs, which we can work 441 * around to make dosemu and wine happy. We do this by preloading the 442 * high word of ESP with the high word of the userspace ESP while 443 * compensating for the offset by changing to the ESPFIX segment with 444 * a base address that matches for the difference. 445 */ 446 mov %esp, %edx /* load kernel esp */ 447 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 448 mov %dx, %ax /* eax: new kernel esp */ 449 sub %eax, %edx /* offset (low word is 0) */ 450 shr $16, %edx 451 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 452 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 453 pushl $__ESPFIX_SS 454 pushl %eax /* new kernel esp */ 455 /* 456 * Disable interrupts, but do not irqtrace this section: we 457 * will soon execute iret and the tracer was already set to 458 * the irqstate after the IRET: 459 */ 460 DISABLE_INTERRUPTS(CLBR_ANY) 461 lss (%esp), %esp /* switch to espfix segment */ 462.Lend_\@: 463#endif /* CONFIG_X86_ESPFIX32 */ 464.endm 465 466/* 467 * Called with pt_regs fully populated and kernel segments loaded, 468 * so we can access PER_CPU and use the integer registers. 469 * 470 * We need to be very careful here with the %esp switch, because an NMI 471 * can happen everywhere. If the NMI handler finds itself on the 472 * entry-stack, it will overwrite the task-stack and everything we 473 * copied there. So allocate the stack-frame on the task-stack and 474 * switch to it before we do any copying. 475 */ 476 477.macro SWITCH_TO_KERNEL_STACK 478 479 ALTERNATIVE "", "jmp .Lend_\@", X86_FEATURE_XENPV 480 481 BUG_IF_WRONG_CR3 482 483 SWITCH_TO_KERNEL_CR3 scratch_reg=%eax 484 485 /* 486 * %eax now contains the entry cr3 and we carry it forward in 487 * that register for the time this macro runs 488 */ 489 490 /* Are we on the entry stack? Bail out if not! */ 491 movl PER_CPU_VAR(cpu_entry_area), %ecx 492 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx 493 subl %esp, %ecx /* ecx = (end of entry_stack) - esp */ 494 cmpl $SIZEOF_entry_stack, %ecx 495 jae .Lend_\@ 496 497 /* Load stack pointer into %esi and %edi */ 498 movl %esp, %esi 499 movl %esi, %edi 500 501 /* Move %edi to the top of the entry stack */ 502 andl $(MASK_entry_stack), %edi 503 addl $(SIZEOF_entry_stack), %edi 504 505 /* Load top of task-stack into %edi */ 506 movl TSS_entry2task_stack(%edi), %edi 507 508 /* Special case - entry from kernel mode via entry stack */ 509#ifdef CONFIG_VM86 510 movl PT_EFLAGS(%esp), %ecx # mix EFLAGS and CS 511 movb PT_CS(%esp), %cl 512 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %ecx 513#else 514 movl PT_CS(%esp), %ecx 515 andl $SEGMENT_RPL_MASK, %ecx 516#endif 517 cmpl $USER_RPL, %ecx 518 jb .Lentry_from_kernel_\@ 519 520 /* Bytes to copy */ 521 movl $PTREGS_SIZE, %ecx 522 523#ifdef CONFIG_VM86 524 testl $X86_EFLAGS_VM, PT_EFLAGS(%esi) 525 jz .Lcopy_pt_regs_\@ 526 527 /* 528 * Stack-frame contains 4 additional segment registers when 529 * coming from VM86 mode 530 */ 531 addl $(4 * 4), %ecx 532 533#endif 534.Lcopy_pt_regs_\@: 535 536 /* Allocate frame on task-stack */ 537 subl %ecx, %edi 538 539 /* Switch to task-stack */ 540 movl %edi, %esp 541 542 /* 543 * We are now on the task-stack and can safely copy over the 544 * stack-frame 545 */ 546 shrl $2, %ecx 547 cld 548 rep movsl 549 550 jmp .Lend_\@ 551 552.Lentry_from_kernel_\@: 553 554 /* 555 * This handles the case when we enter the kernel from 556 * kernel-mode and %esp points to the entry-stack. When this 557 * happens we need to switch to the task-stack to run C code, 558 * but switch back to the entry-stack again when we approach 559 * iret and return to the interrupted code-path. This usually 560 * happens when we hit an exception while restoring user-space 561 * segment registers on the way back to user-space or when the 562 * sysenter handler runs with eflags.tf set. 563 * 564 * When we switch to the task-stack here, we can't trust the 565 * contents of the entry-stack anymore, as the exception handler 566 * might be scheduled out or moved to another CPU. Therefore we 567 * copy the complete entry-stack to the task-stack and set a 568 * marker in the iret-frame (bit 31 of the CS dword) to detect 569 * what we've done on the iret path. 570 * 571 * On the iret path we copy everything back and switch to the 572 * entry-stack, so that the interrupted kernel code-path 573 * continues on the same stack it was interrupted with. 574 * 575 * Be aware that an NMI can happen anytime in this code. 576 * 577 * %esi: Entry-Stack pointer (same as %esp) 578 * %edi: Top of the task stack 579 * %eax: CR3 on kernel entry 580 */ 581 582 /* Calculate number of bytes on the entry stack in %ecx */ 583 movl %esi, %ecx 584 585 /* %ecx to the top of entry-stack */ 586 andl $(MASK_entry_stack), %ecx 587 addl $(SIZEOF_entry_stack), %ecx 588 589 /* Number of bytes on the entry stack to %ecx */ 590 sub %esi, %ecx 591 592 /* Mark stackframe as coming from entry stack */ 593 orl $CS_FROM_ENTRY_STACK, PT_CS(%esp) 594 595 /* 596 * Test the cr3 used to enter the kernel and add a marker 597 * so that we can switch back to it before iret. 598 */ 599 testl $PTI_SWITCH_MASK, %eax 600 jz .Lcopy_pt_regs_\@ 601 orl $CS_FROM_USER_CR3, PT_CS(%esp) 602 603 /* 604 * %esi and %edi are unchanged, %ecx contains the number of 605 * bytes to copy. The code at .Lcopy_pt_regs_\@ will allocate 606 * the stack-frame on task-stack and copy everything over 607 */ 608 jmp .Lcopy_pt_regs_\@ 609 610.Lend_\@: 611.endm 612 613/* 614 * Switch back from the kernel stack to the entry stack. 615 * 616 * The %esp register must point to pt_regs on the task stack. It will 617 * first calculate the size of the stack-frame to copy, depending on 618 * whether we return to VM86 mode or not. With that it uses 'rep movsl' 619 * to copy the contents of the stack over to the entry stack. 620 * 621 * We must be very careful here, as we can't trust the contents of the 622 * task-stack once we switched to the entry-stack. When an NMI happens 623 * while on the entry-stack, the NMI handler will switch back to the top 624 * of the task stack, overwriting our stack-frame we are about to copy. 625 * Therefore we switch the stack only after everything is copied over. 626 */ 627.macro SWITCH_TO_ENTRY_STACK 628 629 ALTERNATIVE "", "jmp .Lend_\@", X86_FEATURE_XENPV 630 631 /* Bytes to copy */ 632 movl $PTREGS_SIZE, %ecx 633 634#ifdef CONFIG_VM86 635 testl $(X86_EFLAGS_VM), PT_EFLAGS(%esp) 636 jz .Lcopy_pt_regs_\@ 637 638 /* Additional 4 registers to copy when returning to VM86 mode */ 639 addl $(4 * 4), %ecx 640 641.Lcopy_pt_regs_\@: 642#endif 643 644 /* Initialize source and destination for movsl */ 645 movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi 646 subl %ecx, %edi 647 movl %esp, %esi 648 649 /* Save future stack pointer in %ebx */ 650 movl %edi, %ebx 651 652 /* Copy over the stack-frame */ 653 shrl $2, %ecx 654 cld 655 rep movsl 656 657 /* 658 * Switch to entry-stack - needs to happen after everything is 659 * copied because the NMI handler will overwrite the task-stack 660 * when on entry-stack 661 */ 662 movl %ebx, %esp 663 664.Lend_\@: 665.endm 666 667/* 668 * This macro handles the case when we return to kernel-mode on the iret 669 * path and have to switch back to the entry stack and/or user-cr3 670 * 671 * See the comments below the .Lentry_from_kernel_\@ label in the 672 * SWITCH_TO_KERNEL_STACK macro for more details. 673 */ 674.macro PARANOID_EXIT_TO_KERNEL_MODE 675 676 /* 677 * Test if we entered the kernel with the entry-stack. Most 678 * likely we did not, because this code only runs on the 679 * return-to-kernel path. 680 */ 681 testl $CS_FROM_ENTRY_STACK, PT_CS(%esp) 682 jz .Lend_\@ 683 684 /* Unlikely slow-path */ 685 686 /* Clear marker from stack-frame */ 687 andl $(~CS_FROM_ENTRY_STACK), PT_CS(%esp) 688 689 /* Copy the remaining task-stack contents to entry-stack */ 690 movl %esp, %esi 691 movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi 692 693 /* Bytes on the task-stack to ecx */ 694 movl PER_CPU_VAR(cpu_tss_rw + TSS_sp1), %ecx 695 subl %esi, %ecx 696 697 /* Allocate stack-frame on entry-stack */ 698 subl %ecx, %edi 699 700 /* 701 * Save future stack-pointer, we must not switch until the 702 * copy is done, otherwise the NMI handler could destroy the 703 * contents of the task-stack we are about to copy. 704 */ 705 movl %edi, %ebx 706 707 /* Do the copy */ 708 shrl $2, %ecx 709 cld 710 rep movsl 711 712 /* Safe to switch to entry-stack now */ 713 movl %ebx, %esp 714 715 /* 716 * We came from entry-stack and need to check if we also need to 717 * switch back to user cr3. 718 */ 719 testl $CS_FROM_USER_CR3, PT_CS(%esp) 720 jz .Lend_\@ 721 722 /* Clear marker from stack-frame */ 723 andl $(~CS_FROM_USER_CR3), PT_CS(%esp) 724 725 SWITCH_TO_USER_CR3 scratch_reg=%eax 726 727.Lend_\@: 728.endm 729/* 730 * %eax: prev task 731 * %edx: next task 732 */ 733SYM_CODE_START(__switch_to_asm) 734 /* 735 * Save callee-saved registers 736 * This must match the order in struct inactive_task_frame 737 */ 738 pushl %ebp 739 pushl %ebx 740 pushl %edi 741 pushl %esi 742 /* 743 * Flags are saved to prevent AC leakage. This could go 744 * away if objtool would have 32bit support to verify 745 * the STAC/CLAC correctness. 746 */ 747 pushfl 748 749 /* switch stack */ 750 movl %esp, TASK_threadsp(%eax) 751 movl TASK_threadsp(%edx), %esp 752 753#ifdef CONFIG_STACKPROTECTOR 754 movl TASK_stack_canary(%edx), %ebx 755 movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset 756#endif 757 758#ifdef CONFIG_RETPOLINE 759 /* 760 * When switching from a shallower to a deeper call stack 761 * the RSB may either underflow or use entries populated 762 * with userspace addresses. On CPUs where those concerns 763 * exist, overwrite the RSB with entries which capture 764 * speculative execution to prevent attack. 765 */ 766 FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW 767#endif 768 769 /* Restore flags or the incoming task to restore AC state. */ 770 popfl 771 /* restore callee-saved registers */ 772 popl %esi 773 popl %edi 774 popl %ebx 775 popl %ebp 776 777 jmp __switch_to 778SYM_CODE_END(__switch_to_asm) 779 780/* 781 * The unwinder expects the last frame on the stack to always be at the same 782 * offset from the end of the page, which allows it to validate the stack. 783 * Calling schedule_tail() directly would break that convention because its an 784 * asmlinkage function so its argument has to be pushed on the stack. This 785 * wrapper creates a proper "end of stack" frame header before the call. 786 */ 787SYM_FUNC_START(schedule_tail_wrapper) 788 FRAME_BEGIN 789 790 pushl %eax 791 call schedule_tail 792 popl %eax 793 794 FRAME_END 795 ret 796SYM_FUNC_END(schedule_tail_wrapper) 797/* 798 * A newly forked process directly context switches into this address. 799 * 800 * eax: prev task we switched from 801 * ebx: kernel thread func (NULL for user thread) 802 * edi: kernel thread arg 803 */ 804SYM_CODE_START(ret_from_fork) 805 call schedule_tail_wrapper 806 807 testl %ebx, %ebx 808 jnz 1f /* kernel threads are uncommon */ 809 8102: 811 /* When we fork, we trace the syscall return in the child, too. */ 812 movl %esp, %eax 813 call syscall_return_slowpath 814 STACKLEAK_ERASE 815 jmp restore_all 816 817 /* kernel thread */ 8181: movl %edi, %eax 819 CALL_NOSPEC %ebx 820 /* 821 * A kernel thread is allowed to return here after successfully 822 * calling do_execve(). Exit to userspace to complete the execve() 823 * syscall. 824 */ 825 movl $0, PT_EAX(%esp) 826 jmp 2b 827SYM_CODE_END(ret_from_fork) 828 829/* 830 * Return to user mode is not as complex as all this looks, 831 * but we want the default path for a system call return to 832 * go as quickly as possible which is why some of this is 833 * less clear than it otherwise should be. 834 */ 835 836 # userspace resumption stub bypassing syscall exit tracing 837SYM_CODE_START_LOCAL(ret_from_exception) 838 preempt_stop(CLBR_ANY) 839ret_from_intr: 840#ifdef CONFIG_VM86 841 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 842 movb PT_CS(%esp), %al 843 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 844#else 845 /* 846 * We can be coming here from child spawned by kernel_thread(). 847 */ 848 movl PT_CS(%esp), %eax 849 andl $SEGMENT_RPL_MASK, %eax 850#endif 851 cmpl $USER_RPL, %eax 852 jb restore_all_kernel # not returning to v8086 or userspace 853 854 DISABLE_INTERRUPTS(CLBR_ANY) 855 TRACE_IRQS_OFF 856 movl %esp, %eax 857 call prepare_exit_to_usermode 858 jmp restore_all 859SYM_CODE_END(ret_from_exception) 860 861SYM_ENTRY(__begin_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE) 862/* 863 * All code from here through __end_SYSENTER_singlestep_region is subject 864 * to being single-stepped if a user program sets TF and executes SYSENTER. 865 * There is absolutely nothing that we can do to prevent this from happening 866 * (thanks Intel!). To keep our handling of this situation as simple as 867 * possible, we handle TF just like AC and NT, except that our #DB handler 868 * will ignore all of the single-step traps generated in this range. 869 */ 870 871#ifdef CONFIG_XEN_PV 872/* 873 * Xen doesn't set %esp to be precisely what the normal SYSENTER 874 * entry point expects, so fix it up before using the normal path. 875 */ 876SYM_CODE_START(xen_sysenter_target) 877 addl $5*4, %esp /* remove xen-provided frame */ 878 jmp .Lsysenter_past_esp 879SYM_CODE_END(xen_sysenter_target) 880#endif 881 882/* 883 * 32-bit SYSENTER entry. 884 * 885 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here 886 * if X86_FEATURE_SEP is available. This is the preferred system call 887 * entry on 32-bit systems. 888 * 889 * The SYSENTER instruction, in principle, should *only* occur in the 890 * vDSO. In practice, a small number of Android devices were shipped 891 * with a copy of Bionic that inlined a SYSENTER instruction. This 892 * never happened in any of Google's Bionic versions -- it only happened 893 * in a narrow range of Intel-provided versions. 894 * 895 * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs. 896 * IF and VM in RFLAGS are cleared (IOW: interrupts are off). 897 * SYSENTER does not save anything on the stack, 898 * and does not save old EIP (!!!), ESP, or EFLAGS. 899 * 900 * To avoid losing track of EFLAGS.VM (and thus potentially corrupting 901 * user and/or vm86 state), we explicitly disable the SYSENTER 902 * instruction in vm86 mode by reprogramming the MSRs. 903 * 904 * Arguments: 905 * eax system call number 906 * ebx arg1 907 * ecx arg2 908 * edx arg3 909 * esi arg4 910 * edi arg5 911 * ebp user stack 912 * 0(%ebp) arg6 913 */ 914SYM_FUNC_START(entry_SYSENTER_32) 915 /* 916 * On entry-stack with all userspace-regs live - save and 917 * restore eflags and %eax to use it as scratch-reg for the cr3 918 * switch. 919 */ 920 pushfl 921 pushl %eax 922 BUG_IF_WRONG_CR3 no_user_check=1 923 SWITCH_TO_KERNEL_CR3 scratch_reg=%eax 924 popl %eax 925 popfl 926 927 /* Stack empty again, switch to task stack */ 928 movl TSS_entry2task_stack(%esp), %esp 929 930.Lsysenter_past_esp: 931 pushl $__USER_DS /* pt_regs->ss */ 932 pushl %ebp /* pt_regs->sp (stashed in bp) */ 933 pushfl /* pt_regs->flags (except IF = 0) */ 934 orl $X86_EFLAGS_IF, (%esp) /* Fix IF */ 935 pushl $__USER_CS /* pt_regs->cs */ 936 pushl $0 /* pt_regs->ip = 0 (placeholder) */ 937 pushl %eax /* pt_regs->orig_ax */ 938 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest, stack already switched */ 939 940 /* 941 * SYSENTER doesn't filter flags, so we need to clear NT, AC 942 * and TF ourselves. To save a few cycles, we can check whether 943 * either was set instead of doing an unconditional popfq. 944 * This needs to happen before enabling interrupts so that 945 * we don't get preempted with NT set. 946 * 947 * If TF is set, we will single-step all the way to here -- do_debug 948 * will ignore all the traps. (Yes, this is slow, but so is 949 * single-stepping in general. This allows us to avoid having 950 * a more complicated code to handle the case where a user program 951 * forces us to single-step through the SYSENTER entry code.) 952 * 953 * NB.: .Lsysenter_fix_flags is a label with the code under it moved 954 * out-of-line as an optimization: NT is unlikely to be set in the 955 * majority of the cases and instead of polluting the I$ unnecessarily, 956 * we're keeping that code behind a branch which will predict as 957 * not-taken and therefore its instructions won't be fetched. 958 */ 959 testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp) 960 jnz .Lsysenter_fix_flags 961.Lsysenter_flags_fixed: 962 963 /* 964 * User mode is traced as though IRQs are on, and SYSENTER 965 * turned them off. 966 */ 967 TRACE_IRQS_OFF 968 969 movl %esp, %eax 970 call do_fast_syscall_32 971 /* XEN PV guests always use IRET path */ 972 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \ 973 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV 974 975 STACKLEAK_ERASE 976 977/* Opportunistic SYSEXIT */ 978 TRACE_IRQS_ON /* User mode traces as IRQs on. */ 979 980 /* 981 * Setup entry stack - we keep the pointer in %eax and do the 982 * switch after almost all user-state is restored. 983 */ 984 985 /* Load entry stack pointer and allocate frame for eflags/eax */ 986 movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %eax 987 subl $(2*4), %eax 988 989 /* Copy eflags and eax to entry stack */ 990 movl PT_EFLAGS(%esp), %edi 991 movl PT_EAX(%esp), %esi 992 movl %edi, (%eax) 993 movl %esi, 4(%eax) 994 995 /* Restore user registers and segments */ 996 movl PT_EIP(%esp), %edx /* pt_regs->ip */ 997 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */ 9981: mov PT_FS(%esp), %fs 999 PTGS_TO_GS 1000 1001 popl %ebx /* pt_regs->bx */ 1002 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */ 1003 popl %esi /* pt_regs->si */ 1004 popl %edi /* pt_regs->di */ 1005 popl %ebp /* pt_regs->bp */ 1006 1007 /* Switch to entry stack */ 1008 movl %eax, %esp 1009 1010 /* Now ready to switch the cr3 */ 1011 SWITCH_TO_USER_CR3 scratch_reg=%eax 1012 1013 /* 1014 * Restore all flags except IF. (We restore IF separately because 1015 * STI gives a one-instruction window in which we won't be interrupted, 1016 * whereas POPF does not.) 1017 */ 1018 btrl $X86_EFLAGS_IF_BIT, (%esp) 1019 BUG_IF_WRONG_CR3 no_user_check=1 1020 popfl 1021 popl %eax 1022 1023 /* 1024 * Return back to the vDSO, which will pop ecx and edx. 1025 * Don't bother with DS and ES (they already contain __USER_DS). 1026 */ 1027 sti 1028 sysexit 1029 1030.pushsection .fixup, "ax" 10312: movl $0, PT_FS(%esp) 1032 jmp 1b 1033.popsection 1034 _ASM_EXTABLE(1b, 2b) 1035 PTGS_TO_GS_EX 1036 1037.Lsysenter_fix_flags: 1038 pushl $X86_EFLAGS_FIXED 1039 popfl 1040 jmp .Lsysenter_flags_fixed 1041SYM_ENTRY(__end_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE) 1042SYM_FUNC_END(entry_SYSENTER_32) 1043 1044/* 1045 * 32-bit legacy system call entry. 1046 * 1047 * 32-bit x86 Linux system calls traditionally used the INT $0x80 1048 * instruction. INT $0x80 lands here. 1049 * 1050 * This entry point can be used by any 32-bit perform system calls. 1051 * Instances of INT $0x80 can be found inline in various programs and 1052 * libraries. It is also used by the vDSO's __kernel_vsyscall 1053 * fallback for hardware that doesn't support a faster entry method. 1054 * Restarted 32-bit system calls also fall back to INT $0x80 1055 * regardless of what instruction was originally used to do the system 1056 * call. (64-bit programs can use INT $0x80 as well, but they can 1057 * only run on 64-bit kernels and therefore land in 1058 * entry_INT80_compat.) 1059 * 1060 * This is considered a slow path. It is not used by most libc 1061 * implementations on modern hardware except during process startup. 1062 * 1063 * Arguments: 1064 * eax system call number 1065 * ebx arg1 1066 * ecx arg2 1067 * edx arg3 1068 * esi arg4 1069 * edi arg5 1070 * ebp arg6 1071 */ 1072SYM_FUNC_START(entry_INT80_32) 1073 ASM_CLAC 1074 pushl %eax /* pt_regs->orig_ax */ 1075 1076 SAVE_ALL pt_regs_ax=$-ENOSYS switch_stacks=1 /* save rest */ 1077 1078 /* 1079 * User mode is traced as though IRQs are on, and the interrupt gate 1080 * turned them off. 1081 */ 1082 TRACE_IRQS_OFF 1083 1084 movl %esp, %eax 1085 call do_int80_syscall_32 1086.Lsyscall_32_done: 1087 1088 STACKLEAK_ERASE 1089 1090restore_all: 1091 TRACE_IRQS_IRET 1092 SWITCH_TO_ENTRY_STACK 1093.Lrestore_all_notrace: 1094 CHECK_AND_APPLY_ESPFIX 1095.Lrestore_nocheck: 1096 /* Switch back to user CR3 */ 1097 SWITCH_TO_USER_CR3 scratch_reg=%eax 1098 1099 BUG_IF_WRONG_CR3 1100 1101 /* Restore user state */ 1102 RESTORE_REGS pop=4 # skip orig_eax/error_code 1103.Lirq_return: 1104 /* 1105 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization 1106 * when returning from IPI handler and when returning from 1107 * scheduler to user-space. 1108 */ 1109 INTERRUPT_RETURN 1110 1111restore_all_kernel: 1112#ifdef CONFIG_PREEMPTION 1113 DISABLE_INTERRUPTS(CLBR_ANY) 1114 cmpl $0, PER_CPU_VAR(__preempt_count) 1115 jnz .Lno_preempt 1116 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? 1117 jz .Lno_preempt 1118 call preempt_schedule_irq 1119.Lno_preempt: 1120#endif 1121 TRACE_IRQS_IRET 1122 PARANOID_EXIT_TO_KERNEL_MODE 1123 BUG_IF_WRONG_CR3 1124 RESTORE_REGS 4 1125 jmp .Lirq_return 1126 1127.section .fixup, "ax" 1128SYM_CODE_START(iret_exc) 1129 pushl $0 # no error code 1130 pushl $do_iret_error 1131 1132#ifdef CONFIG_DEBUG_ENTRY 1133 /* 1134 * The stack-frame here is the one that iret faulted on, so its a 1135 * return-to-user frame. We are on kernel-cr3 because we come here from 1136 * the fixup code. This confuses the CR3 checker, so switch to user-cr3 1137 * as the checker expects it. 1138 */ 1139 pushl %eax 1140 SWITCH_TO_USER_CR3 scratch_reg=%eax 1141 popl %eax 1142#endif 1143 1144 jmp common_exception 1145SYM_CODE_END(iret_exc) 1146.previous 1147 _ASM_EXTABLE(.Lirq_return, iret_exc) 1148SYM_FUNC_END(entry_INT80_32) 1149 1150.macro FIXUP_ESPFIX_STACK 1151/* 1152 * Switch back for ESPFIX stack to the normal zerobased stack 1153 * 1154 * We can't call C functions using the ESPFIX stack. This code reads 1155 * the high word of the segment base from the GDT and swiches to the 1156 * normal stack and adjusts ESP with the matching offset. 1157 * 1158 * We might be on user CR3 here, so percpu data is not mapped and we can't 1159 * access the GDT through the percpu segment. Instead, use SGDT to find 1160 * the cpu_entry_area alias of the GDT. 1161 */ 1162#ifdef CONFIG_X86_ESPFIX32 1163 /* fixup the stack */ 1164 pushl %ecx 1165 subl $2*4, %esp 1166 sgdt (%esp) 1167 movl 2(%esp), %ecx /* GDT address */ 1168 /* 1169 * Careful: ECX is a linear pointer, so we need to force base 1170 * zero. %cs is the only known-linear segment we have right now. 1171 */ 1172 mov %cs:GDT_ESPFIX_OFFSET + 4(%ecx), %al /* bits 16..23 */ 1173 mov %cs:GDT_ESPFIX_OFFSET + 7(%ecx), %ah /* bits 24..31 */ 1174 shl $16, %eax 1175 addl $2*4, %esp 1176 popl %ecx 1177 addl %esp, %eax /* the adjusted stack pointer */ 1178 pushl $__KERNEL_DS 1179 pushl %eax 1180 lss (%esp), %esp /* switch to the normal stack segment */ 1181#endif 1182.endm 1183 1184.macro UNWIND_ESPFIX_STACK 1185 /* It's safe to clobber %eax, all other regs need to be preserved */ 1186#ifdef CONFIG_X86_ESPFIX32 1187 movl %ss, %eax 1188 /* see if on espfix stack */ 1189 cmpw $__ESPFIX_SS, %ax 1190 jne .Lno_fixup_\@ 1191 /* switch to normal stack */ 1192 FIXUP_ESPFIX_STACK 1193.Lno_fixup_\@: 1194#endif 1195.endm 1196 1197/* 1198 * Build the entry stubs with some assembler magic. 1199 * We pack 1 stub into every 8-byte block. 1200 */ 1201 .align 8 1202SYM_CODE_START(irq_entries_start) 1203 vector=FIRST_EXTERNAL_VECTOR 1204 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 1205 pushl $(~vector+0x80) /* Note: always in signed byte range */ 1206 vector=vector+1 1207 jmp common_interrupt 1208 .align 8 1209 .endr 1210SYM_CODE_END(irq_entries_start) 1211 1212#ifdef CONFIG_X86_LOCAL_APIC 1213 .align 8 1214SYM_CODE_START(spurious_entries_start) 1215 vector=FIRST_SYSTEM_VECTOR 1216 .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR) 1217 pushl $(~vector+0x80) /* Note: always in signed byte range */ 1218 vector=vector+1 1219 jmp common_spurious 1220 .align 8 1221 .endr 1222SYM_CODE_END(spurious_entries_start) 1223 1224SYM_CODE_START_LOCAL(common_spurious) 1225 ASM_CLAC 1226 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ 1227 SAVE_ALL switch_stacks=1 1228 ENCODE_FRAME_POINTER 1229 TRACE_IRQS_OFF 1230 movl %esp, %eax 1231 call smp_spurious_interrupt 1232 jmp ret_from_intr 1233SYM_CODE_END(common_spurious) 1234#endif 1235 1236/* 1237 * the CPU automatically disables interrupts when executing an IRQ vector, 1238 * so IRQ-flags tracing has to follow that: 1239 */ 1240 .p2align CONFIG_X86_L1_CACHE_SHIFT 1241SYM_CODE_START_LOCAL(common_interrupt) 1242 ASM_CLAC 1243 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ 1244 1245 SAVE_ALL switch_stacks=1 1246 ENCODE_FRAME_POINTER 1247 TRACE_IRQS_OFF 1248 movl %esp, %eax 1249 call do_IRQ 1250 jmp ret_from_intr 1251SYM_CODE_END(common_interrupt) 1252 1253#define BUILD_INTERRUPT3(name, nr, fn) \ 1254SYM_FUNC_START(name) \ 1255 ASM_CLAC; \ 1256 pushl $~(nr); \ 1257 SAVE_ALL switch_stacks=1; \ 1258 ENCODE_FRAME_POINTER; \ 1259 TRACE_IRQS_OFF \ 1260 movl %esp, %eax; \ 1261 call fn; \ 1262 jmp ret_from_intr; \ 1263SYM_FUNC_END(name) 1264 1265#define BUILD_INTERRUPT(name, nr) \ 1266 BUILD_INTERRUPT3(name, nr, smp_##name); \ 1267 1268/* The include is where all of the SMP etc. interrupts come from */ 1269#include <asm/entry_arch.h> 1270 1271SYM_CODE_START(coprocessor_error) 1272 ASM_CLAC 1273 pushl $0 1274 pushl $do_coprocessor_error 1275 jmp common_exception 1276SYM_CODE_END(coprocessor_error) 1277 1278SYM_CODE_START(simd_coprocessor_error) 1279 ASM_CLAC 1280 pushl $0 1281#ifdef CONFIG_X86_INVD_BUG 1282 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 1283 ALTERNATIVE "pushl $do_general_protection", \ 1284 "pushl $do_simd_coprocessor_error", \ 1285 X86_FEATURE_XMM 1286#else 1287 pushl $do_simd_coprocessor_error 1288#endif 1289 jmp common_exception 1290SYM_CODE_END(simd_coprocessor_error) 1291 1292SYM_CODE_START(device_not_available) 1293 ASM_CLAC 1294 pushl $-1 # mark this as an int 1295 pushl $do_device_not_available 1296 jmp common_exception 1297SYM_CODE_END(device_not_available) 1298 1299#ifdef CONFIG_PARAVIRT 1300SYM_CODE_START(native_iret) 1301 iret 1302 _ASM_EXTABLE(native_iret, iret_exc) 1303SYM_CODE_END(native_iret) 1304#endif 1305 1306SYM_CODE_START(overflow) 1307 ASM_CLAC 1308 pushl $0 1309 pushl $do_overflow 1310 jmp common_exception 1311SYM_CODE_END(overflow) 1312 1313SYM_CODE_START(bounds) 1314 ASM_CLAC 1315 pushl $0 1316 pushl $do_bounds 1317 jmp common_exception 1318SYM_CODE_END(bounds) 1319 1320SYM_CODE_START(invalid_op) 1321 ASM_CLAC 1322 pushl $0 1323 pushl $do_invalid_op 1324 jmp common_exception 1325SYM_CODE_END(invalid_op) 1326 1327SYM_CODE_START(coprocessor_segment_overrun) 1328 ASM_CLAC 1329 pushl $0 1330 pushl $do_coprocessor_segment_overrun 1331 jmp common_exception 1332SYM_CODE_END(coprocessor_segment_overrun) 1333 1334SYM_CODE_START(invalid_TSS) 1335 ASM_CLAC 1336 pushl $do_invalid_TSS 1337 jmp common_exception 1338SYM_CODE_END(invalid_TSS) 1339 1340SYM_CODE_START(segment_not_present) 1341 ASM_CLAC 1342 pushl $do_segment_not_present 1343 jmp common_exception 1344SYM_CODE_END(segment_not_present) 1345 1346SYM_CODE_START(stack_segment) 1347 ASM_CLAC 1348 pushl $do_stack_segment 1349 jmp common_exception 1350SYM_CODE_END(stack_segment) 1351 1352SYM_CODE_START(alignment_check) 1353 ASM_CLAC 1354 pushl $do_alignment_check 1355 jmp common_exception 1356SYM_CODE_END(alignment_check) 1357 1358SYM_CODE_START(divide_error) 1359 ASM_CLAC 1360 pushl $0 # no error code 1361 pushl $do_divide_error 1362 jmp common_exception 1363SYM_CODE_END(divide_error) 1364 1365#ifdef CONFIG_X86_MCE 1366SYM_CODE_START(machine_check) 1367 ASM_CLAC 1368 pushl $0 1369 pushl machine_check_vector 1370 jmp common_exception 1371SYM_CODE_END(machine_check) 1372#endif 1373 1374SYM_CODE_START(spurious_interrupt_bug) 1375 ASM_CLAC 1376 pushl $0 1377 pushl $do_spurious_interrupt_bug 1378 jmp common_exception 1379SYM_CODE_END(spurious_interrupt_bug) 1380 1381#ifdef CONFIG_XEN_PV 1382SYM_FUNC_START(xen_hypervisor_callback) 1383 /* 1384 * Check to see if we got the event in the critical 1385 * region in xen_iret_direct, after we've reenabled 1386 * events and checked for pending events. This simulates 1387 * iret instruction's behaviour where it delivers a 1388 * pending interrupt when enabling interrupts: 1389 */ 1390 cmpl $xen_iret_start_crit, (%esp) 1391 jb 1f 1392 cmpl $xen_iret_end_crit, (%esp) 1393 jae 1f 1394 call xen_iret_crit_fixup 13951: 1396 pushl $-1 /* orig_ax = -1 => not a system call */ 1397 SAVE_ALL 1398 ENCODE_FRAME_POINTER 1399 TRACE_IRQS_OFF 1400 mov %esp, %eax 1401 call xen_evtchn_do_upcall 1402#ifndef CONFIG_PREEMPTION 1403 call xen_maybe_preempt_hcall 1404#endif 1405 jmp ret_from_intr 1406SYM_FUNC_END(xen_hypervisor_callback) 1407 1408/* 1409 * Hypervisor uses this for application faults while it executes. 1410 * We get here for two reasons: 1411 * 1. Fault while reloading DS, ES, FS or GS 1412 * 2. Fault while executing IRET 1413 * Category 1 we fix up by reattempting the load, and zeroing the segment 1414 * register if the load fails. 1415 * Category 2 we fix up by jumping to do_iret_error. We cannot use the 1416 * normal Linux return path in this case because if we use the IRET hypercall 1417 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1418 * We distinguish between categories by maintaining a status value in EAX. 1419 */ 1420SYM_FUNC_START(xen_failsafe_callback) 1421 pushl %eax 1422 movl $1, %eax 14231: mov 4(%esp), %ds 14242: mov 8(%esp), %es 14253: mov 12(%esp), %fs 14264: mov 16(%esp), %gs 1427 /* EAX == 0 => Category 1 (Bad segment) 1428 EAX != 0 => Category 2 (Bad IRET) */ 1429 testl %eax, %eax 1430 popl %eax 1431 lea 16(%esp), %esp 1432 jz 5f 1433 jmp iret_exc 14345: pushl $-1 /* orig_ax = -1 => not a system call */ 1435 SAVE_ALL 1436 ENCODE_FRAME_POINTER 1437 jmp ret_from_exception 1438 1439.section .fixup, "ax" 14406: xorl %eax, %eax 1441 movl %eax, 4(%esp) 1442 jmp 1b 14437: xorl %eax, %eax 1444 movl %eax, 8(%esp) 1445 jmp 2b 14468: xorl %eax, %eax 1447 movl %eax, 12(%esp) 1448 jmp 3b 14499: xorl %eax, %eax 1450 movl %eax, 16(%esp) 1451 jmp 4b 1452.previous 1453 _ASM_EXTABLE(1b, 6b) 1454 _ASM_EXTABLE(2b, 7b) 1455 _ASM_EXTABLE(3b, 8b) 1456 _ASM_EXTABLE(4b, 9b) 1457SYM_FUNC_END(xen_failsafe_callback) 1458#endif /* CONFIG_XEN_PV */ 1459 1460#ifdef CONFIG_XEN_PVHVM 1461BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 1462 xen_evtchn_do_upcall) 1463#endif 1464 1465 1466#if IS_ENABLED(CONFIG_HYPERV) 1467 1468BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 1469 hyperv_vector_handler) 1470 1471BUILD_INTERRUPT3(hyperv_reenlightenment_vector, HYPERV_REENLIGHTENMENT_VECTOR, 1472 hyperv_reenlightenment_intr) 1473 1474BUILD_INTERRUPT3(hv_stimer0_callback_vector, HYPERV_STIMER0_VECTOR, 1475 hv_stimer0_vector_handler) 1476 1477#endif /* CONFIG_HYPERV */ 1478 1479SYM_CODE_START(page_fault) 1480 ASM_CLAC 1481 pushl $do_page_fault 1482 jmp common_exception_read_cr2 1483SYM_CODE_END(page_fault) 1484 1485SYM_CODE_START_LOCAL_NOALIGN(common_exception_read_cr2) 1486 /* the function address is in %gs's slot on the stack */ 1487 SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1 1488 1489 ENCODE_FRAME_POINTER 1490 1491 /* fixup %gs */ 1492 GS_TO_REG %ecx 1493 movl PT_GS(%esp), %edi 1494 REG_TO_PTGS %ecx 1495 SET_KERNEL_GS %ecx 1496 1497 GET_CR2_INTO(%ecx) # might clobber %eax 1498 1499 /* fixup orig %eax */ 1500 movl PT_ORIG_EAX(%esp), %edx # get the error code 1501 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 1502 1503 TRACE_IRQS_OFF 1504 movl %esp, %eax # pt_regs pointer 1505 CALL_NOSPEC %edi 1506 jmp ret_from_exception 1507SYM_CODE_END(common_exception_read_cr2) 1508 1509SYM_CODE_START_LOCAL_NOALIGN(common_exception) 1510 /* the function address is in %gs's slot on the stack */ 1511 SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1 1512 ENCODE_FRAME_POINTER 1513 1514 /* fixup %gs */ 1515 GS_TO_REG %ecx 1516 movl PT_GS(%esp), %edi # get the function address 1517 REG_TO_PTGS %ecx 1518 SET_KERNEL_GS %ecx 1519 1520 /* fixup orig %eax */ 1521 movl PT_ORIG_EAX(%esp), %edx # get the error code 1522 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 1523 1524 TRACE_IRQS_OFF 1525 movl %esp, %eax # pt_regs pointer 1526 CALL_NOSPEC %edi 1527 jmp ret_from_exception 1528SYM_CODE_END(common_exception) 1529 1530SYM_CODE_START(debug) 1531 /* 1532 * Entry from sysenter is now handled in common_exception 1533 */ 1534 ASM_CLAC 1535 pushl $-1 # mark this as an int 1536 pushl $do_debug 1537 jmp common_exception 1538SYM_CODE_END(debug) 1539 1540/* 1541 * NMI is doubly nasty. It can happen on the first instruction of 1542 * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning 1543 * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32 1544 * switched stacks. We handle both conditions by simply checking whether we 1545 * interrupted kernel code running on the SYSENTER stack. 1546 */ 1547SYM_CODE_START(nmi) 1548 ASM_CLAC 1549 1550#ifdef CONFIG_X86_ESPFIX32 1551 /* 1552 * ESPFIX_SS is only ever set on the return to user path 1553 * after we've switched to the entry stack. 1554 */ 1555 pushl %eax 1556 movl %ss, %eax 1557 cmpw $__ESPFIX_SS, %ax 1558 popl %eax 1559 je .Lnmi_espfix_stack 1560#endif 1561 1562 pushl %eax # pt_regs->orig_ax 1563 SAVE_ALL_NMI cr3_reg=%edi 1564 ENCODE_FRAME_POINTER 1565 xorl %edx, %edx # zero error code 1566 movl %esp, %eax # pt_regs pointer 1567 1568 /* Are we currently on the SYSENTER stack? */ 1569 movl PER_CPU_VAR(cpu_entry_area), %ecx 1570 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx 1571 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */ 1572 cmpl $SIZEOF_entry_stack, %ecx 1573 jb .Lnmi_from_sysenter_stack 1574 1575 /* Not on SYSENTER stack. */ 1576 call do_nmi 1577 jmp .Lnmi_return 1578 1579.Lnmi_from_sysenter_stack: 1580 /* 1581 * We're on the SYSENTER stack. Switch off. No one (not even debug) 1582 * is using the thread stack right now, so it's safe for us to use it. 1583 */ 1584 movl %esp, %ebx 1585 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp 1586 call do_nmi 1587 movl %ebx, %esp 1588 1589.Lnmi_return: 1590#ifdef CONFIG_X86_ESPFIX32 1591 testl $CS_FROM_ESPFIX, PT_CS(%esp) 1592 jnz .Lnmi_from_espfix 1593#endif 1594 1595 CHECK_AND_APPLY_ESPFIX 1596 RESTORE_ALL_NMI cr3_reg=%edi pop=4 1597 jmp .Lirq_return 1598 1599#ifdef CONFIG_X86_ESPFIX32 1600.Lnmi_espfix_stack: 1601 /* 1602 * Create the pointer to LSS back 1603 */ 1604 pushl %ss 1605 pushl %esp 1606 addl $4, (%esp) 1607 1608 /* Copy the (short) IRET frame */ 1609 pushl 4*4(%esp) # flags 1610 pushl 4*4(%esp) # cs 1611 pushl 4*4(%esp) # ip 1612 1613 pushl %eax # orig_ax 1614 1615 SAVE_ALL_NMI cr3_reg=%edi unwind_espfix=1 1616 ENCODE_FRAME_POINTER 1617 1618 /* clear CS_FROM_KERNEL, set CS_FROM_ESPFIX */ 1619 xorl $(CS_FROM_ESPFIX | CS_FROM_KERNEL), PT_CS(%esp) 1620 1621 xorl %edx, %edx # zero error code 1622 movl %esp, %eax # pt_regs pointer 1623 jmp .Lnmi_from_sysenter_stack 1624 1625.Lnmi_from_espfix: 1626 RESTORE_ALL_NMI cr3_reg=%edi 1627 /* 1628 * Because we cleared CS_FROM_KERNEL, IRET_FRAME 'forgot' to 1629 * fix up the gap and long frame: 1630 * 1631 * 3 - original frame (exception) 1632 * 2 - ESPFIX block (above) 1633 * 6 - gap (FIXUP_FRAME) 1634 * 5 - long frame (FIXUP_FRAME) 1635 * 1 - orig_ax 1636 */ 1637 lss (1+5+6)*4(%esp), %esp # back to espfix stack 1638 jmp .Lirq_return 1639#endif 1640SYM_CODE_END(nmi) 1641 1642SYM_CODE_START(int3) 1643 ASM_CLAC 1644 pushl $-1 # mark this as an int 1645 1646 SAVE_ALL switch_stacks=1 1647 ENCODE_FRAME_POINTER 1648 TRACE_IRQS_OFF 1649 xorl %edx, %edx # zero error code 1650 movl %esp, %eax # pt_regs pointer 1651 call do_int3 1652 jmp ret_from_exception 1653SYM_CODE_END(int3) 1654 1655SYM_CODE_START(general_protection) 1656 pushl $do_general_protection 1657 jmp common_exception 1658SYM_CODE_END(general_protection) 1659 1660#ifdef CONFIG_KVM_GUEST 1661SYM_CODE_START(async_page_fault) 1662 ASM_CLAC 1663 pushl $do_async_page_fault 1664 jmp common_exception_read_cr2 1665SYM_CODE_END(async_page_fault) 1666#endif 1667 1668SYM_CODE_START(rewind_stack_do_exit) 1669 /* Prevent any naive code from trying to unwind to our caller. */ 1670 xorl %ebp, %ebp 1671 1672 movl PER_CPU_VAR(cpu_current_top_of_stack), %esi 1673 leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp 1674 1675 call do_exit 16761: jmp 1b 1677SYM_CODE_END(rewind_stack_do_exit) 1678