1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 1991,1992 Linus Torvalds 4 * 5 * entry_32.S contains the system-call and low-level fault and trap handling routines. 6 * 7 * Stack layout while running C code: 8 * ptrace needs to have all registers on the stack. 9 * If the order here is changed, it needs to be 10 * updated in fork.c:copy_process(), signal.c:do_signal(), 11 * ptrace.c and ptrace.h 12 * 13 * 0(%esp) - %ebx 14 * 4(%esp) - %ecx 15 * 8(%esp) - %edx 16 * C(%esp) - %esi 17 * 10(%esp) - %edi 18 * 14(%esp) - %ebp 19 * 18(%esp) - %eax 20 * 1C(%esp) - %ds 21 * 20(%esp) - %es 22 * 24(%esp) - %fs 23 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 24 * 2C(%esp) - orig_eax 25 * 30(%esp) - %eip 26 * 34(%esp) - %cs 27 * 38(%esp) - %eflags 28 * 3C(%esp) - %oldesp 29 * 40(%esp) - %oldss 30 */ 31 32#include <linux/linkage.h> 33#include <linux/err.h> 34#include <asm/thread_info.h> 35#include <asm/irqflags.h> 36#include <asm/errno.h> 37#include <asm/segment.h> 38#include <asm/smp.h> 39#include <asm/percpu.h> 40#include <asm/processor-flags.h> 41#include <asm/irq_vectors.h> 42#include <asm/cpufeatures.h> 43#include <asm/alternative-asm.h> 44#include <asm/asm.h> 45#include <asm/smap.h> 46#include <asm/frame.h> 47#include <asm/nospec-branch.h> 48 49 .section .entry.text, "ax" 50 51/* 52 * We use macros for low-level operations which need to be overridden 53 * for paravirtualization. The following will never clobber any registers: 54 * INTERRUPT_RETURN (aka. "iret") 55 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 56 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 57 * 58 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 59 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 60 * Allowing a register to be clobbered can shrink the paravirt replacement 61 * enough to patch inline, increasing performance. 62 */ 63 64#ifdef CONFIG_PREEMPT 65# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 66#else 67# define preempt_stop(clobbers) 68# define resume_kernel restore_all 69#endif 70 71.macro TRACE_IRQS_IRET 72#ifdef CONFIG_TRACE_IRQFLAGS 73 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off? 74 jz 1f 75 TRACE_IRQS_ON 761: 77#endif 78.endm 79 80/* 81 * User gs save/restore 82 * 83 * %gs is used for userland TLS and kernel only uses it for stack 84 * canary which is required to be at %gs:20 by gcc. Read the comment 85 * at the top of stackprotector.h for more info. 86 * 87 * Local labels 98 and 99 are used. 88 */ 89#ifdef CONFIG_X86_32_LAZY_GS 90 91 /* unfortunately push/pop can't be no-op */ 92.macro PUSH_GS 93 pushl $0 94.endm 95.macro POP_GS pop=0 96 addl $(4 + \pop), %esp 97.endm 98.macro POP_GS_EX 99.endm 100 101 /* all the rest are no-op */ 102.macro PTGS_TO_GS 103.endm 104.macro PTGS_TO_GS_EX 105.endm 106.macro GS_TO_REG reg 107.endm 108.macro REG_TO_PTGS reg 109.endm 110.macro SET_KERNEL_GS reg 111.endm 112 113#else /* CONFIG_X86_32_LAZY_GS */ 114 115.macro PUSH_GS 116 pushl %gs 117.endm 118 119.macro POP_GS pop=0 12098: popl %gs 121 .if \pop <> 0 122 add $\pop, %esp 123 .endif 124.endm 125.macro POP_GS_EX 126.pushsection .fixup, "ax" 12799: movl $0, (%esp) 128 jmp 98b 129.popsection 130 _ASM_EXTABLE(98b, 99b) 131.endm 132 133.macro PTGS_TO_GS 13498: mov PT_GS(%esp), %gs 135.endm 136.macro PTGS_TO_GS_EX 137.pushsection .fixup, "ax" 13899: movl $0, PT_GS(%esp) 139 jmp 98b 140.popsection 141 _ASM_EXTABLE(98b, 99b) 142.endm 143 144.macro GS_TO_REG reg 145 movl %gs, \reg 146.endm 147.macro REG_TO_PTGS reg 148 movl \reg, PT_GS(%esp) 149.endm 150.macro SET_KERNEL_GS reg 151 movl $(__KERNEL_STACK_CANARY), \reg 152 movl \reg, %gs 153.endm 154 155#endif /* CONFIG_X86_32_LAZY_GS */ 156 157.macro SAVE_ALL pt_regs_ax=%eax 158 cld 159 PUSH_GS 160 pushl %fs 161 pushl %es 162 pushl %ds 163 pushl \pt_regs_ax 164 pushl %ebp 165 pushl %edi 166 pushl %esi 167 pushl %edx 168 pushl %ecx 169 pushl %ebx 170 movl $(__USER_DS), %edx 171 movl %edx, %ds 172 movl %edx, %es 173 movl $(__KERNEL_PERCPU), %edx 174 movl %edx, %fs 175 SET_KERNEL_GS %edx 176.endm 177 178/* 179 * This is a sneaky trick to help the unwinder find pt_regs on the stack. The 180 * frame pointer is replaced with an encoded pointer to pt_regs. The encoding 181 * is just clearing the MSB, which makes it an invalid stack address and is also 182 * a signal to the unwinder that it's a pt_regs pointer in disguise. 183 * 184 * NOTE: This macro must be used *after* SAVE_ALL because it corrupts the 185 * original rbp. 186 */ 187.macro ENCODE_FRAME_POINTER 188#ifdef CONFIG_FRAME_POINTER 189 mov %esp, %ebp 190 andl $0x7fffffff, %ebp 191#endif 192.endm 193 194.macro RESTORE_INT_REGS 195 popl %ebx 196 popl %ecx 197 popl %edx 198 popl %esi 199 popl %edi 200 popl %ebp 201 popl %eax 202.endm 203 204.macro RESTORE_REGS pop=0 205 RESTORE_INT_REGS 2061: popl %ds 2072: popl %es 2083: popl %fs 209 POP_GS \pop 210.pushsection .fixup, "ax" 2114: movl $0, (%esp) 212 jmp 1b 2135: movl $0, (%esp) 214 jmp 2b 2156: movl $0, (%esp) 216 jmp 3b 217.popsection 218 _ASM_EXTABLE(1b, 4b) 219 _ASM_EXTABLE(2b, 5b) 220 _ASM_EXTABLE(3b, 6b) 221 POP_GS_EX 222.endm 223 224/* 225 * %eax: prev task 226 * %edx: next task 227 */ 228ENTRY(__switch_to_asm) 229 /* 230 * Save callee-saved registers 231 * This must match the order in struct inactive_task_frame 232 */ 233 pushl %ebp 234 pushl %ebx 235 pushl %edi 236 pushl %esi 237 238 /* switch stack */ 239 movl %esp, TASK_threadsp(%eax) 240 movl TASK_threadsp(%edx), %esp 241 242#ifdef CONFIG_CC_STACKPROTECTOR 243 movl TASK_stack_canary(%edx), %ebx 244 movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset 245#endif 246 247#ifdef CONFIG_RETPOLINE 248 /* 249 * When switching from a shallower to a deeper call stack 250 * the RSB may either underflow or use entries populated 251 * with userspace addresses. On CPUs where those concerns 252 * exist, overwrite the RSB with entries which capture 253 * speculative execution to prevent attack. 254 */ 255 /* Clobbers %ebx */ 256 FILL_RETURN_BUFFER RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW 257#endif 258 259 /* restore callee-saved registers */ 260 popl %esi 261 popl %edi 262 popl %ebx 263 popl %ebp 264 265 jmp __switch_to 266END(__switch_to_asm) 267 268/* 269 * The unwinder expects the last frame on the stack to always be at the same 270 * offset from the end of the page, which allows it to validate the stack. 271 * Calling schedule_tail() directly would break that convention because its an 272 * asmlinkage function so its argument has to be pushed on the stack. This 273 * wrapper creates a proper "end of stack" frame header before the call. 274 */ 275ENTRY(schedule_tail_wrapper) 276 FRAME_BEGIN 277 278 pushl %eax 279 call schedule_tail 280 popl %eax 281 282 FRAME_END 283 ret 284ENDPROC(schedule_tail_wrapper) 285/* 286 * A newly forked process directly context switches into this address. 287 * 288 * eax: prev task we switched from 289 * ebx: kernel thread func (NULL for user thread) 290 * edi: kernel thread arg 291 */ 292ENTRY(ret_from_fork) 293 call schedule_tail_wrapper 294 295 testl %ebx, %ebx 296 jnz 1f /* kernel threads are uncommon */ 297 2982: 299 /* When we fork, we trace the syscall return in the child, too. */ 300 movl %esp, %eax 301 call syscall_return_slowpath 302 jmp restore_all 303 304 /* kernel thread */ 3051: movl %edi, %eax 306 CALL_NOSPEC %ebx 307 /* 308 * A kernel thread is allowed to return here after successfully 309 * calling do_execve(). Exit to userspace to complete the execve() 310 * syscall. 311 */ 312 movl $0, PT_EAX(%esp) 313 jmp 2b 314END(ret_from_fork) 315 316/* 317 * Return to user mode is not as complex as all this looks, 318 * but we want the default path for a system call return to 319 * go as quickly as possible which is why some of this is 320 * less clear than it otherwise should be. 321 */ 322 323 # userspace resumption stub bypassing syscall exit tracing 324 ALIGN 325ret_from_exception: 326 preempt_stop(CLBR_ANY) 327ret_from_intr: 328#ifdef CONFIG_VM86 329 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 330 movb PT_CS(%esp), %al 331 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 332#else 333 /* 334 * We can be coming here from child spawned by kernel_thread(). 335 */ 336 movl PT_CS(%esp), %eax 337 andl $SEGMENT_RPL_MASK, %eax 338#endif 339 cmpl $USER_RPL, %eax 340 jb resume_kernel # not returning to v8086 or userspace 341 342ENTRY(resume_userspace) 343 DISABLE_INTERRUPTS(CLBR_ANY) 344 TRACE_IRQS_OFF 345 movl %esp, %eax 346 call prepare_exit_to_usermode 347 jmp restore_all 348END(ret_from_exception) 349 350#ifdef CONFIG_PREEMPT 351ENTRY(resume_kernel) 352 DISABLE_INTERRUPTS(CLBR_ANY) 353.Lneed_resched: 354 cmpl $0, PER_CPU_VAR(__preempt_count) 355 jnz restore_all 356 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? 357 jz restore_all 358 call preempt_schedule_irq 359 jmp .Lneed_resched 360END(resume_kernel) 361#endif 362 363GLOBAL(__begin_SYSENTER_singlestep_region) 364/* 365 * All code from here through __end_SYSENTER_singlestep_region is subject 366 * to being single-stepped if a user program sets TF and executes SYSENTER. 367 * There is absolutely nothing that we can do to prevent this from happening 368 * (thanks Intel!). To keep our handling of this situation as simple as 369 * possible, we handle TF just like AC and NT, except that our #DB handler 370 * will ignore all of the single-step traps generated in this range. 371 */ 372 373#ifdef CONFIG_XEN 374/* 375 * Xen doesn't set %esp to be precisely what the normal SYSENTER 376 * entry point expects, so fix it up before using the normal path. 377 */ 378ENTRY(xen_sysenter_target) 379 addl $5*4, %esp /* remove xen-provided frame */ 380 jmp .Lsysenter_past_esp 381#endif 382 383/* 384 * 32-bit SYSENTER entry. 385 * 386 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here 387 * if X86_FEATURE_SEP is available. This is the preferred system call 388 * entry on 32-bit systems. 389 * 390 * The SYSENTER instruction, in principle, should *only* occur in the 391 * vDSO. In practice, a small number of Android devices were shipped 392 * with a copy of Bionic that inlined a SYSENTER instruction. This 393 * never happened in any of Google's Bionic versions -- it only happened 394 * in a narrow range of Intel-provided versions. 395 * 396 * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs. 397 * IF and VM in RFLAGS are cleared (IOW: interrupts are off). 398 * SYSENTER does not save anything on the stack, 399 * and does not save old EIP (!!!), ESP, or EFLAGS. 400 * 401 * To avoid losing track of EFLAGS.VM (and thus potentially corrupting 402 * user and/or vm86 state), we explicitly disable the SYSENTER 403 * instruction in vm86 mode by reprogramming the MSRs. 404 * 405 * Arguments: 406 * eax system call number 407 * ebx arg1 408 * ecx arg2 409 * edx arg3 410 * esi arg4 411 * edi arg5 412 * ebp user stack 413 * 0(%ebp) arg6 414 */ 415ENTRY(entry_SYSENTER_32) 416 movl TSS_sysenter_sp0(%esp), %esp 417.Lsysenter_past_esp: 418 pushl $__USER_DS /* pt_regs->ss */ 419 pushl %ebp /* pt_regs->sp (stashed in bp) */ 420 pushfl /* pt_regs->flags (except IF = 0) */ 421 orl $X86_EFLAGS_IF, (%esp) /* Fix IF */ 422 pushl $__USER_CS /* pt_regs->cs */ 423 pushl $0 /* pt_regs->ip = 0 (placeholder) */ 424 pushl %eax /* pt_regs->orig_ax */ 425 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 426 427 /* 428 * SYSENTER doesn't filter flags, so we need to clear NT, AC 429 * and TF ourselves. To save a few cycles, we can check whether 430 * either was set instead of doing an unconditional popfq. 431 * This needs to happen before enabling interrupts so that 432 * we don't get preempted with NT set. 433 * 434 * If TF is set, we will single-step all the way to here -- do_debug 435 * will ignore all the traps. (Yes, this is slow, but so is 436 * single-stepping in general. This allows us to avoid having 437 * a more complicated code to handle the case where a user program 438 * forces us to single-step through the SYSENTER entry code.) 439 * 440 * NB.: .Lsysenter_fix_flags is a label with the code under it moved 441 * out-of-line as an optimization: NT is unlikely to be set in the 442 * majority of the cases and instead of polluting the I$ unnecessarily, 443 * we're keeping that code behind a branch which will predict as 444 * not-taken and therefore its instructions won't be fetched. 445 */ 446 testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp) 447 jnz .Lsysenter_fix_flags 448.Lsysenter_flags_fixed: 449 450 /* 451 * User mode is traced as though IRQs are on, and SYSENTER 452 * turned them off. 453 */ 454 TRACE_IRQS_OFF 455 456 movl %esp, %eax 457 call do_fast_syscall_32 458 /* XEN PV guests always use IRET path */ 459 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \ 460 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV 461 462/* Opportunistic SYSEXIT */ 463 TRACE_IRQS_ON /* User mode traces as IRQs on. */ 464 movl PT_EIP(%esp), %edx /* pt_regs->ip */ 465 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */ 4661: mov PT_FS(%esp), %fs 467 PTGS_TO_GS 468 popl %ebx /* pt_regs->bx */ 469 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */ 470 popl %esi /* pt_regs->si */ 471 popl %edi /* pt_regs->di */ 472 popl %ebp /* pt_regs->bp */ 473 popl %eax /* pt_regs->ax */ 474 475 /* 476 * Restore all flags except IF. (We restore IF separately because 477 * STI gives a one-instruction window in which we won't be interrupted, 478 * whereas POPF does not.) 479 */ 480 addl $PT_EFLAGS-PT_DS, %esp /* point esp at pt_regs->flags */ 481 btr $X86_EFLAGS_IF_BIT, (%esp) 482 popfl 483 484 /* 485 * Return back to the vDSO, which will pop ecx and edx. 486 * Don't bother with DS and ES (they already contain __USER_DS). 487 */ 488 sti 489 sysexit 490 491.pushsection .fixup, "ax" 4922: movl $0, PT_FS(%esp) 493 jmp 1b 494.popsection 495 _ASM_EXTABLE(1b, 2b) 496 PTGS_TO_GS_EX 497 498.Lsysenter_fix_flags: 499 pushl $X86_EFLAGS_FIXED 500 popfl 501 jmp .Lsysenter_flags_fixed 502GLOBAL(__end_SYSENTER_singlestep_region) 503ENDPROC(entry_SYSENTER_32) 504 505/* 506 * 32-bit legacy system call entry. 507 * 508 * 32-bit x86 Linux system calls traditionally used the INT $0x80 509 * instruction. INT $0x80 lands here. 510 * 511 * This entry point can be used by any 32-bit perform system calls. 512 * Instances of INT $0x80 can be found inline in various programs and 513 * libraries. It is also used by the vDSO's __kernel_vsyscall 514 * fallback for hardware that doesn't support a faster entry method. 515 * Restarted 32-bit system calls also fall back to INT $0x80 516 * regardless of what instruction was originally used to do the system 517 * call. (64-bit programs can use INT $0x80 as well, but they can 518 * only run on 64-bit kernels and therefore land in 519 * entry_INT80_compat.) 520 * 521 * This is considered a slow path. It is not used by most libc 522 * implementations on modern hardware except during process startup. 523 * 524 * Arguments: 525 * eax system call number 526 * ebx arg1 527 * ecx arg2 528 * edx arg3 529 * esi arg4 530 * edi arg5 531 * ebp arg6 532 */ 533ENTRY(entry_INT80_32) 534 ASM_CLAC 535 pushl %eax /* pt_regs->orig_ax */ 536 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 537 538 /* 539 * User mode is traced as though IRQs are on, and the interrupt gate 540 * turned them off. 541 */ 542 TRACE_IRQS_OFF 543 544 movl %esp, %eax 545 call do_int80_syscall_32 546.Lsyscall_32_done: 547 548restore_all: 549 TRACE_IRQS_IRET 550.Lrestore_all_notrace: 551#ifdef CONFIG_X86_ESPFIX32 552 ALTERNATIVE "jmp .Lrestore_nocheck", "", X86_BUG_ESPFIX 553 554 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 555 /* 556 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we 557 * are returning to the kernel. 558 * See comments in process.c:copy_thread() for details. 559 */ 560 movb PT_OLDSS(%esp), %ah 561 movb PT_CS(%esp), %al 562 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 563 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 564 je .Lldt_ss # returning to user-space with LDT SS 565#endif 566.Lrestore_nocheck: 567 RESTORE_REGS 4 # skip orig_eax/error_code 568.Lirq_return: 569 INTERRUPT_RETURN 570 571.section .fixup, "ax" 572ENTRY(iret_exc ) 573 pushl $0 # no error code 574 pushl $do_iret_error 575 jmp common_exception 576.previous 577 _ASM_EXTABLE(.Lirq_return, iret_exc) 578 579#ifdef CONFIG_X86_ESPFIX32 580.Lldt_ss: 581/* 582 * Setup and switch to ESPFIX stack 583 * 584 * We're returning to userspace with a 16 bit stack. The CPU will not 585 * restore the high word of ESP for us on executing iret... This is an 586 * "official" bug of all the x86-compatible CPUs, which we can work 587 * around to make dosemu and wine happy. We do this by preloading the 588 * high word of ESP with the high word of the userspace ESP while 589 * compensating for the offset by changing to the ESPFIX segment with 590 * a base address that matches for the difference. 591 */ 592#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 593 mov %esp, %edx /* load kernel esp */ 594 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 595 mov %dx, %ax /* eax: new kernel esp */ 596 sub %eax, %edx /* offset (low word is 0) */ 597 shr $16, %edx 598 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 599 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 600 pushl $__ESPFIX_SS 601 pushl %eax /* new kernel esp */ 602 /* 603 * Disable interrupts, but do not irqtrace this section: we 604 * will soon execute iret and the tracer was already set to 605 * the irqstate after the IRET: 606 */ 607 DISABLE_INTERRUPTS(CLBR_ANY) 608 lss (%esp), %esp /* switch to espfix segment */ 609 jmp .Lrestore_nocheck 610#endif 611ENDPROC(entry_INT80_32) 612 613.macro FIXUP_ESPFIX_STACK 614/* 615 * Switch back for ESPFIX stack to the normal zerobased stack 616 * 617 * We can't call C functions using the ESPFIX stack. This code reads 618 * the high word of the segment base from the GDT and swiches to the 619 * normal stack and adjusts ESP with the matching offset. 620 */ 621#ifdef CONFIG_X86_ESPFIX32 622 /* fixup the stack */ 623 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 624 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 625 shl $16, %eax 626 addl %esp, %eax /* the adjusted stack pointer */ 627 pushl $__KERNEL_DS 628 pushl %eax 629 lss (%esp), %esp /* switch to the normal stack segment */ 630#endif 631.endm 632.macro UNWIND_ESPFIX_STACK 633#ifdef CONFIG_X86_ESPFIX32 634 movl %ss, %eax 635 /* see if on espfix stack */ 636 cmpw $__ESPFIX_SS, %ax 637 jne 27f 638 movl $__KERNEL_DS, %eax 639 movl %eax, %ds 640 movl %eax, %es 641 /* switch to normal stack */ 642 FIXUP_ESPFIX_STACK 64327: 644#endif 645.endm 646 647/* 648 * Build the entry stubs with some assembler magic. 649 * We pack 1 stub into every 8-byte block. 650 */ 651 .align 8 652ENTRY(irq_entries_start) 653 vector=FIRST_EXTERNAL_VECTOR 654 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 655 pushl $(~vector+0x80) /* Note: always in signed byte range */ 656 vector=vector+1 657 jmp common_interrupt 658 .align 8 659 .endr 660END(irq_entries_start) 661 662/* 663 * the CPU automatically disables interrupts when executing an IRQ vector, 664 * so IRQ-flags tracing has to follow that: 665 */ 666 .p2align CONFIG_X86_L1_CACHE_SHIFT 667common_interrupt: 668 ASM_CLAC 669 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ 670 SAVE_ALL 671 ENCODE_FRAME_POINTER 672 TRACE_IRQS_OFF 673 movl %esp, %eax 674 call do_IRQ 675 jmp ret_from_intr 676ENDPROC(common_interrupt) 677 678#define BUILD_INTERRUPT3(name, nr, fn) \ 679ENTRY(name) \ 680 ASM_CLAC; \ 681 pushl $~(nr); \ 682 SAVE_ALL; \ 683 ENCODE_FRAME_POINTER; \ 684 TRACE_IRQS_OFF \ 685 movl %esp, %eax; \ 686 call fn; \ 687 jmp ret_from_intr; \ 688ENDPROC(name) 689 690#define BUILD_INTERRUPT(name, nr) \ 691 BUILD_INTERRUPT3(name, nr, smp_##name); \ 692 693/* The include is where all of the SMP etc. interrupts come from */ 694#include <asm/entry_arch.h> 695 696ENTRY(coprocessor_error) 697 ASM_CLAC 698 pushl $0 699 pushl $do_coprocessor_error 700 jmp common_exception 701END(coprocessor_error) 702 703ENTRY(simd_coprocessor_error) 704 ASM_CLAC 705 pushl $0 706#ifdef CONFIG_X86_INVD_BUG 707 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 708 ALTERNATIVE "pushl $do_general_protection", \ 709 "pushl $do_simd_coprocessor_error", \ 710 X86_FEATURE_XMM 711#else 712 pushl $do_simd_coprocessor_error 713#endif 714 jmp common_exception 715END(simd_coprocessor_error) 716 717ENTRY(device_not_available) 718 ASM_CLAC 719 pushl $-1 # mark this as an int 720 pushl $do_device_not_available 721 jmp common_exception 722END(device_not_available) 723 724#ifdef CONFIG_PARAVIRT 725ENTRY(native_iret) 726 iret 727 _ASM_EXTABLE(native_iret, iret_exc) 728END(native_iret) 729#endif 730 731ENTRY(overflow) 732 ASM_CLAC 733 pushl $0 734 pushl $do_overflow 735 jmp common_exception 736END(overflow) 737 738ENTRY(bounds) 739 ASM_CLAC 740 pushl $0 741 pushl $do_bounds 742 jmp common_exception 743END(bounds) 744 745ENTRY(invalid_op) 746 ASM_CLAC 747 pushl $0 748 pushl $do_invalid_op 749 jmp common_exception 750END(invalid_op) 751 752ENTRY(coprocessor_segment_overrun) 753 ASM_CLAC 754 pushl $0 755 pushl $do_coprocessor_segment_overrun 756 jmp common_exception 757END(coprocessor_segment_overrun) 758 759ENTRY(invalid_TSS) 760 ASM_CLAC 761 pushl $do_invalid_TSS 762 jmp common_exception 763END(invalid_TSS) 764 765ENTRY(segment_not_present) 766 ASM_CLAC 767 pushl $do_segment_not_present 768 jmp common_exception 769END(segment_not_present) 770 771ENTRY(stack_segment) 772 ASM_CLAC 773 pushl $do_stack_segment 774 jmp common_exception 775END(stack_segment) 776 777ENTRY(alignment_check) 778 ASM_CLAC 779 pushl $do_alignment_check 780 jmp common_exception 781END(alignment_check) 782 783ENTRY(divide_error) 784 ASM_CLAC 785 pushl $0 # no error code 786 pushl $do_divide_error 787 jmp common_exception 788END(divide_error) 789 790#ifdef CONFIG_X86_MCE 791ENTRY(machine_check) 792 ASM_CLAC 793 pushl $0 794 pushl machine_check_vector 795 jmp common_exception 796END(machine_check) 797#endif 798 799ENTRY(spurious_interrupt_bug) 800 ASM_CLAC 801 pushl $0 802 pushl $do_spurious_interrupt_bug 803 jmp common_exception 804END(spurious_interrupt_bug) 805 806#ifdef CONFIG_XEN 807ENTRY(xen_hypervisor_callback) 808 pushl $-1 /* orig_ax = -1 => not a system call */ 809 SAVE_ALL 810 ENCODE_FRAME_POINTER 811 TRACE_IRQS_OFF 812 813 /* 814 * Check to see if we got the event in the critical 815 * region in xen_iret_direct, after we've reenabled 816 * events and checked for pending events. This simulates 817 * iret instruction's behaviour where it delivers a 818 * pending interrupt when enabling interrupts: 819 */ 820 movl PT_EIP(%esp), %eax 821 cmpl $xen_iret_start_crit, %eax 822 jb 1f 823 cmpl $xen_iret_end_crit, %eax 824 jae 1f 825 826 jmp xen_iret_crit_fixup 827 828ENTRY(xen_do_upcall) 8291: mov %esp, %eax 830 call xen_evtchn_do_upcall 831#ifndef CONFIG_PREEMPT 832 call xen_maybe_preempt_hcall 833#endif 834 jmp ret_from_intr 835ENDPROC(xen_hypervisor_callback) 836 837/* 838 * Hypervisor uses this for application faults while it executes. 839 * We get here for two reasons: 840 * 1. Fault while reloading DS, ES, FS or GS 841 * 2. Fault while executing IRET 842 * Category 1 we fix up by reattempting the load, and zeroing the segment 843 * register if the load fails. 844 * Category 2 we fix up by jumping to do_iret_error. We cannot use the 845 * normal Linux return path in this case because if we use the IRET hypercall 846 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 847 * We distinguish between categories by maintaining a status value in EAX. 848 */ 849ENTRY(xen_failsafe_callback) 850 pushl %eax 851 movl $1, %eax 8521: mov 4(%esp), %ds 8532: mov 8(%esp), %es 8543: mov 12(%esp), %fs 8554: mov 16(%esp), %gs 856 /* EAX == 0 => Category 1 (Bad segment) 857 EAX != 0 => Category 2 (Bad IRET) */ 858 testl %eax, %eax 859 popl %eax 860 lea 16(%esp), %esp 861 jz 5f 862 jmp iret_exc 8635: pushl $-1 /* orig_ax = -1 => not a system call */ 864 SAVE_ALL 865 ENCODE_FRAME_POINTER 866 jmp ret_from_exception 867 868.section .fixup, "ax" 8696: xorl %eax, %eax 870 movl %eax, 4(%esp) 871 jmp 1b 8727: xorl %eax, %eax 873 movl %eax, 8(%esp) 874 jmp 2b 8758: xorl %eax, %eax 876 movl %eax, 12(%esp) 877 jmp 3b 8789: xorl %eax, %eax 879 movl %eax, 16(%esp) 880 jmp 4b 881.previous 882 _ASM_EXTABLE(1b, 6b) 883 _ASM_EXTABLE(2b, 7b) 884 _ASM_EXTABLE(3b, 8b) 885 _ASM_EXTABLE(4b, 9b) 886ENDPROC(xen_failsafe_callback) 887 888BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 889 xen_evtchn_do_upcall) 890 891#endif /* CONFIG_XEN */ 892 893#if IS_ENABLED(CONFIG_HYPERV) 894 895BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 896 hyperv_vector_handler) 897 898#endif /* CONFIG_HYPERV */ 899 900ENTRY(page_fault) 901 ASM_CLAC 902 pushl $do_page_fault 903 ALIGN 904 jmp common_exception 905END(page_fault) 906 907common_exception: 908 /* the function address is in %gs's slot on the stack */ 909 pushl %fs 910 pushl %es 911 pushl %ds 912 pushl %eax 913 pushl %ebp 914 pushl %edi 915 pushl %esi 916 pushl %edx 917 pushl %ecx 918 pushl %ebx 919 ENCODE_FRAME_POINTER 920 cld 921 movl $(__KERNEL_PERCPU), %ecx 922 movl %ecx, %fs 923 UNWIND_ESPFIX_STACK 924 GS_TO_REG %ecx 925 movl PT_GS(%esp), %edi # get the function address 926 movl PT_ORIG_EAX(%esp), %edx # get the error code 927 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 928 REG_TO_PTGS %ecx 929 SET_KERNEL_GS %ecx 930 movl $(__USER_DS), %ecx 931 movl %ecx, %ds 932 movl %ecx, %es 933 TRACE_IRQS_OFF 934 movl %esp, %eax # pt_regs pointer 935 CALL_NOSPEC %edi 936 jmp ret_from_exception 937END(common_exception) 938 939ENTRY(debug) 940 /* 941 * #DB can happen at the first instruction of 942 * entry_SYSENTER_32 or in Xen's SYSENTER prologue. If this 943 * happens, then we will be running on a very small stack. We 944 * need to detect this condition and switch to the thread 945 * stack before calling any C code at all. 946 * 947 * If you edit this code, keep in mind that NMIs can happen in here. 948 */ 949 ASM_CLAC 950 pushl $-1 # mark this as an int 951 SAVE_ALL 952 ENCODE_FRAME_POINTER 953 xorl %edx, %edx # error code 0 954 movl %esp, %eax # pt_regs pointer 955 956 /* Are we currently on the SYSENTER stack? */ 957 movl PER_CPU_VAR(cpu_entry_area), %ecx 958 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx 959 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */ 960 cmpl $SIZEOF_entry_stack, %ecx 961 jb .Ldebug_from_sysenter_stack 962 963 TRACE_IRQS_OFF 964 call do_debug 965 jmp ret_from_exception 966 967.Ldebug_from_sysenter_stack: 968 /* We're on the SYSENTER stack. Switch off. */ 969 movl %esp, %ebx 970 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp 971 TRACE_IRQS_OFF 972 call do_debug 973 movl %ebx, %esp 974 jmp ret_from_exception 975END(debug) 976 977/* 978 * NMI is doubly nasty. It can happen on the first instruction of 979 * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning 980 * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32 981 * switched stacks. We handle both conditions by simply checking whether we 982 * interrupted kernel code running on the SYSENTER stack. 983 */ 984ENTRY(nmi) 985 ASM_CLAC 986#ifdef CONFIG_X86_ESPFIX32 987 pushl %eax 988 movl %ss, %eax 989 cmpw $__ESPFIX_SS, %ax 990 popl %eax 991 je .Lnmi_espfix_stack 992#endif 993 994 pushl %eax # pt_regs->orig_ax 995 SAVE_ALL 996 ENCODE_FRAME_POINTER 997 xorl %edx, %edx # zero error code 998 movl %esp, %eax # pt_regs pointer 999 1000 /* Are we currently on the SYSENTER stack? */ 1001 movl PER_CPU_VAR(cpu_entry_area), %ecx 1002 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx 1003 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */ 1004 cmpl $SIZEOF_entry_stack, %ecx 1005 jb .Lnmi_from_sysenter_stack 1006 1007 /* Not on SYSENTER stack. */ 1008 call do_nmi 1009 jmp .Lrestore_all_notrace 1010 1011.Lnmi_from_sysenter_stack: 1012 /* 1013 * We're on the SYSENTER stack. Switch off. No one (not even debug) 1014 * is using the thread stack right now, so it's safe for us to use it. 1015 */ 1016 movl %esp, %ebx 1017 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp 1018 call do_nmi 1019 movl %ebx, %esp 1020 jmp .Lrestore_all_notrace 1021 1022#ifdef CONFIG_X86_ESPFIX32 1023.Lnmi_espfix_stack: 1024 /* 1025 * create the pointer to lss back 1026 */ 1027 pushl %ss 1028 pushl %esp 1029 addl $4, (%esp) 1030 /* copy the iret frame of 12 bytes */ 1031 .rept 3 1032 pushl 16(%esp) 1033 .endr 1034 pushl %eax 1035 SAVE_ALL 1036 ENCODE_FRAME_POINTER 1037 FIXUP_ESPFIX_STACK # %eax == %esp 1038 xorl %edx, %edx # zero error code 1039 call do_nmi 1040 RESTORE_REGS 1041 lss 12+4(%esp), %esp # back to espfix stack 1042 jmp .Lirq_return 1043#endif 1044END(nmi) 1045 1046ENTRY(int3) 1047 ASM_CLAC 1048 pushl $-1 # mark this as an int 1049 SAVE_ALL 1050 ENCODE_FRAME_POINTER 1051 TRACE_IRQS_OFF 1052 xorl %edx, %edx # zero error code 1053 movl %esp, %eax # pt_regs pointer 1054 call do_int3 1055 jmp ret_from_exception 1056END(int3) 1057 1058ENTRY(general_protection) 1059 pushl $do_general_protection 1060 jmp common_exception 1061END(general_protection) 1062 1063#ifdef CONFIG_KVM_GUEST 1064ENTRY(async_page_fault) 1065 ASM_CLAC 1066 pushl $do_async_page_fault 1067 jmp common_exception 1068END(async_page_fault) 1069#endif 1070 1071ENTRY(rewind_stack_do_exit) 1072 /* Prevent any naive code from trying to unwind to our caller. */ 1073 xorl %ebp, %ebp 1074 1075 movl PER_CPU_VAR(cpu_current_top_of_stack), %esi 1076 leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp 1077 1078 call do_exit 10791: jmp 1b 1080END(rewind_stack_do_exit) 1081