1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 1991,1992 Linus Torvalds 4 * 5 * entry_32.S contains the system-call and low-level fault and trap handling routines. 6 * 7 * Stack layout while running C code: 8 * ptrace needs to have all registers on the stack. 9 * If the order here is changed, it needs to be 10 * updated in fork.c:copy_process(), signal.c:do_signal(), 11 * ptrace.c and ptrace.h 12 * 13 * 0(%esp) - %ebx 14 * 4(%esp) - %ecx 15 * 8(%esp) - %edx 16 * C(%esp) - %esi 17 * 10(%esp) - %edi 18 * 14(%esp) - %ebp 19 * 18(%esp) - %eax 20 * 1C(%esp) - %ds 21 * 20(%esp) - %es 22 * 24(%esp) - %fs 23 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 24 * 2C(%esp) - orig_eax 25 * 30(%esp) - %eip 26 * 34(%esp) - %cs 27 * 38(%esp) - %eflags 28 * 3C(%esp) - %oldesp 29 * 40(%esp) - %oldss 30 */ 31 32#include <linux/linkage.h> 33#include <linux/err.h> 34#include <asm/thread_info.h> 35#include <asm/irqflags.h> 36#include <asm/errno.h> 37#include <asm/segment.h> 38#include <asm/smp.h> 39#include <asm/percpu.h> 40#include <asm/processor-flags.h> 41#include <asm/irq_vectors.h> 42#include <asm/cpufeatures.h> 43#include <asm/alternative-asm.h> 44#include <asm/asm.h> 45#include <asm/smap.h> 46#include <asm/frame.h> 47#include <asm/nospec-branch.h> 48 49 .section .entry.text, "ax" 50 51/* 52 * We use macros for low-level operations which need to be overridden 53 * for paravirtualization. The following will never clobber any registers: 54 * INTERRUPT_RETURN (aka. "iret") 55 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 56 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 57 * 58 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 59 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 60 * Allowing a register to be clobbered can shrink the paravirt replacement 61 * enough to patch inline, increasing performance. 62 */ 63 64#ifdef CONFIG_PREEMPT 65# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 66#else 67# define preempt_stop(clobbers) 68# define resume_kernel restore_all 69#endif 70 71.macro TRACE_IRQS_IRET 72#ifdef CONFIG_TRACE_IRQFLAGS 73 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off? 74 jz 1f 75 TRACE_IRQS_ON 761: 77#endif 78.endm 79 80/* 81 * User gs save/restore 82 * 83 * %gs is used for userland TLS and kernel only uses it for stack 84 * canary which is required to be at %gs:20 by gcc. Read the comment 85 * at the top of stackprotector.h for more info. 86 * 87 * Local labels 98 and 99 are used. 88 */ 89#ifdef CONFIG_X86_32_LAZY_GS 90 91 /* unfortunately push/pop can't be no-op */ 92.macro PUSH_GS 93 pushl $0 94.endm 95.macro POP_GS pop=0 96 addl $(4 + \pop), %esp 97.endm 98.macro POP_GS_EX 99.endm 100 101 /* all the rest are no-op */ 102.macro PTGS_TO_GS 103.endm 104.macro PTGS_TO_GS_EX 105.endm 106.macro GS_TO_REG reg 107.endm 108.macro REG_TO_PTGS reg 109.endm 110.macro SET_KERNEL_GS reg 111.endm 112 113#else /* CONFIG_X86_32_LAZY_GS */ 114 115.macro PUSH_GS 116 pushl %gs 117.endm 118 119.macro POP_GS pop=0 12098: popl %gs 121 .if \pop <> 0 122 add $\pop, %esp 123 .endif 124.endm 125.macro POP_GS_EX 126.pushsection .fixup, "ax" 12799: movl $0, (%esp) 128 jmp 98b 129.popsection 130 _ASM_EXTABLE(98b, 99b) 131.endm 132 133.macro PTGS_TO_GS 13498: mov PT_GS(%esp), %gs 135.endm 136.macro PTGS_TO_GS_EX 137.pushsection .fixup, "ax" 13899: movl $0, PT_GS(%esp) 139 jmp 98b 140.popsection 141 _ASM_EXTABLE(98b, 99b) 142.endm 143 144.macro GS_TO_REG reg 145 movl %gs, \reg 146.endm 147.macro REG_TO_PTGS reg 148 movl \reg, PT_GS(%esp) 149.endm 150.macro SET_KERNEL_GS reg 151 movl $(__KERNEL_STACK_CANARY), \reg 152 movl \reg, %gs 153.endm 154 155#endif /* CONFIG_X86_32_LAZY_GS */ 156 157.macro SAVE_ALL pt_regs_ax=%eax 158 cld 159 PUSH_GS 160 pushl %fs 161 pushl %es 162 pushl %ds 163 pushl \pt_regs_ax 164 pushl %ebp 165 pushl %edi 166 pushl %esi 167 pushl %edx 168 pushl %ecx 169 pushl %ebx 170 movl $(__USER_DS), %edx 171 movl %edx, %ds 172 movl %edx, %es 173 movl $(__KERNEL_PERCPU), %edx 174 movl %edx, %fs 175 SET_KERNEL_GS %edx 176.endm 177 178/* 179 * This is a sneaky trick to help the unwinder find pt_regs on the stack. The 180 * frame pointer is replaced with an encoded pointer to pt_regs. The encoding 181 * is just clearing the MSB, which makes it an invalid stack address and is also 182 * a signal to the unwinder that it's a pt_regs pointer in disguise. 183 * 184 * NOTE: This macro must be used *after* SAVE_ALL because it corrupts the 185 * original rbp. 186 */ 187.macro ENCODE_FRAME_POINTER 188#ifdef CONFIG_FRAME_POINTER 189 mov %esp, %ebp 190 andl $0x7fffffff, %ebp 191#endif 192.endm 193 194.macro RESTORE_INT_REGS 195 popl %ebx 196 popl %ecx 197 popl %edx 198 popl %esi 199 popl %edi 200 popl %ebp 201 popl %eax 202.endm 203 204.macro RESTORE_REGS pop=0 205 RESTORE_INT_REGS 2061: popl %ds 2072: popl %es 2083: popl %fs 209 POP_GS \pop 210.pushsection .fixup, "ax" 2114: movl $0, (%esp) 212 jmp 1b 2135: movl $0, (%esp) 214 jmp 2b 2156: movl $0, (%esp) 216 jmp 3b 217.popsection 218 _ASM_EXTABLE(1b, 4b) 219 _ASM_EXTABLE(2b, 5b) 220 _ASM_EXTABLE(3b, 6b) 221 POP_GS_EX 222.endm 223 224/* 225 * %eax: prev task 226 * %edx: next task 227 */ 228ENTRY(__switch_to_asm) 229 /* 230 * Save callee-saved registers 231 * This must match the order in struct inactive_task_frame 232 */ 233 pushl %ebp 234 pushl %ebx 235 pushl %edi 236 pushl %esi 237 238 /* switch stack */ 239 movl %esp, TASK_threadsp(%eax) 240 movl TASK_threadsp(%edx), %esp 241 242#ifdef CONFIG_CC_STACKPROTECTOR 243 movl TASK_stack_canary(%edx), %ebx 244 movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset 245#endif 246 247#ifdef CONFIG_RETPOLINE 248 /* 249 * When switching from a shallower to a deeper call stack 250 * the RSB may either underflow or use entries populated 251 * with userspace addresses. On CPUs where those concerns 252 * exist, overwrite the RSB with entries which capture 253 * speculative execution to prevent attack. 254 */ 255 FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW 256#endif 257 258 /* restore callee-saved registers */ 259 popl %esi 260 popl %edi 261 popl %ebx 262 popl %ebp 263 264 jmp __switch_to 265END(__switch_to_asm) 266 267/* 268 * The unwinder expects the last frame on the stack to always be at the same 269 * offset from the end of the page, which allows it to validate the stack. 270 * Calling schedule_tail() directly would break that convention because its an 271 * asmlinkage function so its argument has to be pushed on the stack. This 272 * wrapper creates a proper "end of stack" frame header before the call. 273 */ 274ENTRY(schedule_tail_wrapper) 275 FRAME_BEGIN 276 277 pushl %eax 278 call schedule_tail 279 popl %eax 280 281 FRAME_END 282 ret 283ENDPROC(schedule_tail_wrapper) 284/* 285 * A newly forked process directly context switches into this address. 286 * 287 * eax: prev task we switched from 288 * ebx: kernel thread func (NULL for user thread) 289 * edi: kernel thread arg 290 */ 291ENTRY(ret_from_fork) 292 call schedule_tail_wrapper 293 294 testl %ebx, %ebx 295 jnz 1f /* kernel threads are uncommon */ 296 2972: 298 /* When we fork, we trace the syscall return in the child, too. */ 299 movl %esp, %eax 300 call syscall_return_slowpath 301 jmp restore_all 302 303 /* kernel thread */ 3041: movl %edi, %eax 305 CALL_NOSPEC %ebx 306 /* 307 * A kernel thread is allowed to return here after successfully 308 * calling do_execve(). Exit to userspace to complete the execve() 309 * syscall. 310 */ 311 movl $0, PT_EAX(%esp) 312 jmp 2b 313END(ret_from_fork) 314 315/* 316 * Return to user mode is not as complex as all this looks, 317 * but we want the default path for a system call return to 318 * go as quickly as possible which is why some of this is 319 * less clear than it otherwise should be. 320 */ 321 322 # userspace resumption stub bypassing syscall exit tracing 323 ALIGN 324ret_from_exception: 325 preempt_stop(CLBR_ANY) 326ret_from_intr: 327#ifdef CONFIG_VM86 328 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 329 movb PT_CS(%esp), %al 330 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 331#else 332 /* 333 * We can be coming here from child spawned by kernel_thread(). 334 */ 335 movl PT_CS(%esp), %eax 336 andl $SEGMENT_RPL_MASK, %eax 337#endif 338 cmpl $USER_RPL, %eax 339 jb resume_kernel # not returning to v8086 or userspace 340 341ENTRY(resume_userspace) 342 DISABLE_INTERRUPTS(CLBR_ANY) 343 TRACE_IRQS_OFF 344 movl %esp, %eax 345 call prepare_exit_to_usermode 346 jmp restore_all 347END(ret_from_exception) 348 349#ifdef CONFIG_PREEMPT 350ENTRY(resume_kernel) 351 DISABLE_INTERRUPTS(CLBR_ANY) 352.Lneed_resched: 353 cmpl $0, PER_CPU_VAR(__preempt_count) 354 jnz restore_all 355 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? 356 jz restore_all 357 call preempt_schedule_irq 358 jmp .Lneed_resched 359END(resume_kernel) 360#endif 361 362GLOBAL(__begin_SYSENTER_singlestep_region) 363/* 364 * All code from here through __end_SYSENTER_singlestep_region is subject 365 * to being single-stepped if a user program sets TF and executes SYSENTER. 366 * There is absolutely nothing that we can do to prevent this from happening 367 * (thanks Intel!). To keep our handling of this situation as simple as 368 * possible, we handle TF just like AC and NT, except that our #DB handler 369 * will ignore all of the single-step traps generated in this range. 370 */ 371 372#ifdef CONFIG_XEN 373/* 374 * Xen doesn't set %esp to be precisely what the normal SYSENTER 375 * entry point expects, so fix it up before using the normal path. 376 */ 377ENTRY(xen_sysenter_target) 378 addl $5*4, %esp /* remove xen-provided frame */ 379 jmp .Lsysenter_past_esp 380#endif 381 382/* 383 * 32-bit SYSENTER entry. 384 * 385 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here 386 * if X86_FEATURE_SEP is available. This is the preferred system call 387 * entry on 32-bit systems. 388 * 389 * The SYSENTER instruction, in principle, should *only* occur in the 390 * vDSO. In practice, a small number of Android devices were shipped 391 * with a copy of Bionic that inlined a SYSENTER instruction. This 392 * never happened in any of Google's Bionic versions -- it only happened 393 * in a narrow range of Intel-provided versions. 394 * 395 * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs. 396 * IF and VM in RFLAGS are cleared (IOW: interrupts are off). 397 * SYSENTER does not save anything on the stack, 398 * and does not save old EIP (!!!), ESP, or EFLAGS. 399 * 400 * To avoid losing track of EFLAGS.VM (and thus potentially corrupting 401 * user and/or vm86 state), we explicitly disable the SYSENTER 402 * instruction in vm86 mode by reprogramming the MSRs. 403 * 404 * Arguments: 405 * eax system call number 406 * ebx arg1 407 * ecx arg2 408 * edx arg3 409 * esi arg4 410 * edi arg5 411 * ebp user stack 412 * 0(%ebp) arg6 413 */ 414ENTRY(entry_SYSENTER_32) 415 movl TSS_sysenter_sp0(%esp), %esp 416.Lsysenter_past_esp: 417 pushl $__USER_DS /* pt_regs->ss */ 418 pushl %ebp /* pt_regs->sp (stashed in bp) */ 419 pushfl /* pt_regs->flags (except IF = 0) */ 420 orl $X86_EFLAGS_IF, (%esp) /* Fix IF */ 421 pushl $__USER_CS /* pt_regs->cs */ 422 pushl $0 /* pt_regs->ip = 0 (placeholder) */ 423 pushl %eax /* pt_regs->orig_ax */ 424 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 425 426 /* 427 * SYSENTER doesn't filter flags, so we need to clear NT, AC 428 * and TF ourselves. To save a few cycles, we can check whether 429 * either was set instead of doing an unconditional popfq. 430 * This needs to happen before enabling interrupts so that 431 * we don't get preempted with NT set. 432 * 433 * If TF is set, we will single-step all the way to here -- do_debug 434 * will ignore all the traps. (Yes, this is slow, but so is 435 * single-stepping in general. This allows us to avoid having 436 * a more complicated code to handle the case where a user program 437 * forces us to single-step through the SYSENTER entry code.) 438 * 439 * NB.: .Lsysenter_fix_flags is a label with the code under it moved 440 * out-of-line as an optimization: NT is unlikely to be set in the 441 * majority of the cases and instead of polluting the I$ unnecessarily, 442 * we're keeping that code behind a branch which will predict as 443 * not-taken and therefore its instructions won't be fetched. 444 */ 445 testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp) 446 jnz .Lsysenter_fix_flags 447.Lsysenter_flags_fixed: 448 449 /* 450 * User mode is traced as though IRQs are on, and SYSENTER 451 * turned them off. 452 */ 453 TRACE_IRQS_OFF 454 455 movl %esp, %eax 456 call do_fast_syscall_32 457 /* XEN PV guests always use IRET path */ 458 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \ 459 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV 460 461/* Opportunistic SYSEXIT */ 462 TRACE_IRQS_ON /* User mode traces as IRQs on. */ 463 movl PT_EIP(%esp), %edx /* pt_regs->ip */ 464 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */ 4651: mov PT_FS(%esp), %fs 466 PTGS_TO_GS 467 popl %ebx /* pt_regs->bx */ 468 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */ 469 popl %esi /* pt_regs->si */ 470 popl %edi /* pt_regs->di */ 471 popl %ebp /* pt_regs->bp */ 472 popl %eax /* pt_regs->ax */ 473 474 /* 475 * Restore all flags except IF. (We restore IF separately because 476 * STI gives a one-instruction window in which we won't be interrupted, 477 * whereas POPF does not.) 478 */ 479 addl $PT_EFLAGS-PT_DS, %esp /* point esp at pt_regs->flags */ 480 btr $X86_EFLAGS_IF_BIT, (%esp) 481 popfl 482 483 /* 484 * Return back to the vDSO, which will pop ecx and edx. 485 * Don't bother with DS and ES (they already contain __USER_DS). 486 */ 487 sti 488 sysexit 489 490.pushsection .fixup, "ax" 4912: movl $0, PT_FS(%esp) 492 jmp 1b 493.popsection 494 _ASM_EXTABLE(1b, 2b) 495 PTGS_TO_GS_EX 496 497.Lsysenter_fix_flags: 498 pushl $X86_EFLAGS_FIXED 499 popfl 500 jmp .Lsysenter_flags_fixed 501GLOBAL(__end_SYSENTER_singlestep_region) 502ENDPROC(entry_SYSENTER_32) 503 504/* 505 * 32-bit legacy system call entry. 506 * 507 * 32-bit x86 Linux system calls traditionally used the INT $0x80 508 * instruction. INT $0x80 lands here. 509 * 510 * This entry point can be used by any 32-bit perform system calls. 511 * Instances of INT $0x80 can be found inline in various programs and 512 * libraries. It is also used by the vDSO's __kernel_vsyscall 513 * fallback for hardware that doesn't support a faster entry method. 514 * Restarted 32-bit system calls also fall back to INT $0x80 515 * regardless of what instruction was originally used to do the system 516 * call. (64-bit programs can use INT $0x80 as well, but they can 517 * only run on 64-bit kernels and therefore land in 518 * entry_INT80_compat.) 519 * 520 * This is considered a slow path. It is not used by most libc 521 * implementations on modern hardware except during process startup. 522 * 523 * Arguments: 524 * eax system call number 525 * ebx arg1 526 * ecx arg2 527 * edx arg3 528 * esi arg4 529 * edi arg5 530 * ebp arg6 531 */ 532ENTRY(entry_INT80_32) 533 ASM_CLAC 534 pushl %eax /* pt_regs->orig_ax */ 535 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 536 537 /* 538 * User mode is traced as though IRQs are on, and the interrupt gate 539 * turned them off. 540 */ 541 TRACE_IRQS_OFF 542 543 movl %esp, %eax 544 call do_int80_syscall_32 545.Lsyscall_32_done: 546 547restore_all: 548 TRACE_IRQS_IRET 549.Lrestore_all_notrace: 550#ifdef CONFIG_X86_ESPFIX32 551 ALTERNATIVE "jmp .Lrestore_nocheck", "", X86_BUG_ESPFIX 552 553 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 554 /* 555 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we 556 * are returning to the kernel. 557 * See comments in process.c:copy_thread() for details. 558 */ 559 movb PT_OLDSS(%esp), %ah 560 movb PT_CS(%esp), %al 561 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 562 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 563 je .Lldt_ss # returning to user-space with LDT SS 564#endif 565.Lrestore_nocheck: 566 RESTORE_REGS 4 # skip orig_eax/error_code 567.Lirq_return: 568 INTERRUPT_RETURN 569 570.section .fixup, "ax" 571ENTRY(iret_exc ) 572 pushl $0 # no error code 573 pushl $do_iret_error 574 jmp common_exception 575.previous 576 _ASM_EXTABLE(.Lirq_return, iret_exc) 577 578#ifdef CONFIG_X86_ESPFIX32 579.Lldt_ss: 580/* 581 * Setup and switch to ESPFIX stack 582 * 583 * We're returning to userspace with a 16 bit stack. The CPU will not 584 * restore the high word of ESP for us on executing iret... This is an 585 * "official" bug of all the x86-compatible CPUs, which we can work 586 * around to make dosemu and wine happy. We do this by preloading the 587 * high word of ESP with the high word of the userspace ESP while 588 * compensating for the offset by changing to the ESPFIX segment with 589 * a base address that matches for the difference. 590 */ 591#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 592 mov %esp, %edx /* load kernel esp */ 593 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 594 mov %dx, %ax /* eax: new kernel esp */ 595 sub %eax, %edx /* offset (low word is 0) */ 596 shr $16, %edx 597 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 598 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 599 pushl $__ESPFIX_SS 600 pushl %eax /* new kernel esp */ 601 /* 602 * Disable interrupts, but do not irqtrace this section: we 603 * will soon execute iret and the tracer was already set to 604 * the irqstate after the IRET: 605 */ 606 DISABLE_INTERRUPTS(CLBR_ANY) 607 lss (%esp), %esp /* switch to espfix segment */ 608 jmp .Lrestore_nocheck 609#endif 610ENDPROC(entry_INT80_32) 611 612.macro FIXUP_ESPFIX_STACK 613/* 614 * Switch back for ESPFIX stack to the normal zerobased stack 615 * 616 * We can't call C functions using the ESPFIX stack. This code reads 617 * the high word of the segment base from the GDT and swiches to the 618 * normal stack and adjusts ESP with the matching offset. 619 */ 620#ifdef CONFIG_X86_ESPFIX32 621 /* fixup the stack */ 622 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 623 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 624 shl $16, %eax 625 addl %esp, %eax /* the adjusted stack pointer */ 626 pushl $__KERNEL_DS 627 pushl %eax 628 lss (%esp), %esp /* switch to the normal stack segment */ 629#endif 630.endm 631.macro UNWIND_ESPFIX_STACK 632#ifdef CONFIG_X86_ESPFIX32 633 movl %ss, %eax 634 /* see if on espfix stack */ 635 cmpw $__ESPFIX_SS, %ax 636 jne 27f 637 movl $__KERNEL_DS, %eax 638 movl %eax, %ds 639 movl %eax, %es 640 /* switch to normal stack */ 641 FIXUP_ESPFIX_STACK 64227: 643#endif 644.endm 645 646/* 647 * Build the entry stubs with some assembler magic. 648 * We pack 1 stub into every 8-byte block. 649 */ 650 .align 8 651ENTRY(irq_entries_start) 652 vector=FIRST_EXTERNAL_VECTOR 653 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 654 pushl $(~vector+0x80) /* Note: always in signed byte range */ 655 vector=vector+1 656 jmp common_interrupt 657 .align 8 658 .endr 659END(irq_entries_start) 660 661/* 662 * the CPU automatically disables interrupts when executing an IRQ vector, 663 * so IRQ-flags tracing has to follow that: 664 */ 665 .p2align CONFIG_X86_L1_CACHE_SHIFT 666common_interrupt: 667 ASM_CLAC 668 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ 669 SAVE_ALL 670 ENCODE_FRAME_POINTER 671 TRACE_IRQS_OFF 672 movl %esp, %eax 673 call do_IRQ 674 jmp ret_from_intr 675ENDPROC(common_interrupt) 676 677#define BUILD_INTERRUPT3(name, nr, fn) \ 678ENTRY(name) \ 679 ASM_CLAC; \ 680 pushl $~(nr); \ 681 SAVE_ALL; \ 682 ENCODE_FRAME_POINTER; \ 683 TRACE_IRQS_OFF \ 684 movl %esp, %eax; \ 685 call fn; \ 686 jmp ret_from_intr; \ 687ENDPROC(name) 688 689#define BUILD_INTERRUPT(name, nr) \ 690 BUILD_INTERRUPT3(name, nr, smp_##name); \ 691 692/* The include is where all of the SMP etc. interrupts come from */ 693#include <asm/entry_arch.h> 694 695ENTRY(coprocessor_error) 696 ASM_CLAC 697 pushl $0 698 pushl $do_coprocessor_error 699 jmp common_exception 700END(coprocessor_error) 701 702ENTRY(simd_coprocessor_error) 703 ASM_CLAC 704 pushl $0 705#ifdef CONFIG_X86_INVD_BUG 706 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 707 ALTERNATIVE "pushl $do_general_protection", \ 708 "pushl $do_simd_coprocessor_error", \ 709 X86_FEATURE_XMM 710#else 711 pushl $do_simd_coprocessor_error 712#endif 713 jmp common_exception 714END(simd_coprocessor_error) 715 716ENTRY(device_not_available) 717 ASM_CLAC 718 pushl $-1 # mark this as an int 719 pushl $do_device_not_available 720 jmp common_exception 721END(device_not_available) 722 723#ifdef CONFIG_PARAVIRT 724ENTRY(native_iret) 725 iret 726 _ASM_EXTABLE(native_iret, iret_exc) 727END(native_iret) 728#endif 729 730ENTRY(overflow) 731 ASM_CLAC 732 pushl $0 733 pushl $do_overflow 734 jmp common_exception 735END(overflow) 736 737ENTRY(bounds) 738 ASM_CLAC 739 pushl $0 740 pushl $do_bounds 741 jmp common_exception 742END(bounds) 743 744ENTRY(invalid_op) 745 ASM_CLAC 746 pushl $0 747 pushl $do_invalid_op 748 jmp common_exception 749END(invalid_op) 750 751ENTRY(coprocessor_segment_overrun) 752 ASM_CLAC 753 pushl $0 754 pushl $do_coprocessor_segment_overrun 755 jmp common_exception 756END(coprocessor_segment_overrun) 757 758ENTRY(invalid_TSS) 759 ASM_CLAC 760 pushl $do_invalid_TSS 761 jmp common_exception 762END(invalid_TSS) 763 764ENTRY(segment_not_present) 765 ASM_CLAC 766 pushl $do_segment_not_present 767 jmp common_exception 768END(segment_not_present) 769 770ENTRY(stack_segment) 771 ASM_CLAC 772 pushl $do_stack_segment 773 jmp common_exception 774END(stack_segment) 775 776ENTRY(alignment_check) 777 ASM_CLAC 778 pushl $do_alignment_check 779 jmp common_exception 780END(alignment_check) 781 782ENTRY(divide_error) 783 ASM_CLAC 784 pushl $0 # no error code 785 pushl $do_divide_error 786 jmp common_exception 787END(divide_error) 788 789#ifdef CONFIG_X86_MCE 790ENTRY(machine_check) 791 ASM_CLAC 792 pushl $0 793 pushl machine_check_vector 794 jmp common_exception 795END(machine_check) 796#endif 797 798ENTRY(spurious_interrupt_bug) 799 ASM_CLAC 800 pushl $0 801 pushl $do_spurious_interrupt_bug 802 jmp common_exception 803END(spurious_interrupt_bug) 804 805#ifdef CONFIG_XEN 806ENTRY(xen_hypervisor_callback) 807 pushl $-1 /* orig_ax = -1 => not a system call */ 808 SAVE_ALL 809 ENCODE_FRAME_POINTER 810 TRACE_IRQS_OFF 811 812 /* 813 * Check to see if we got the event in the critical 814 * region in xen_iret_direct, after we've reenabled 815 * events and checked for pending events. This simulates 816 * iret instruction's behaviour where it delivers a 817 * pending interrupt when enabling interrupts: 818 */ 819 movl PT_EIP(%esp), %eax 820 cmpl $xen_iret_start_crit, %eax 821 jb 1f 822 cmpl $xen_iret_end_crit, %eax 823 jae 1f 824 825 jmp xen_iret_crit_fixup 826 827ENTRY(xen_do_upcall) 8281: mov %esp, %eax 829 call xen_evtchn_do_upcall 830#ifndef CONFIG_PREEMPT 831 call xen_maybe_preempt_hcall 832#endif 833 jmp ret_from_intr 834ENDPROC(xen_hypervisor_callback) 835 836/* 837 * Hypervisor uses this for application faults while it executes. 838 * We get here for two reasons: 839 * 1. Fault while reloading DS, ES, FS or GS 840 * 2. Fault while executing IRET 841 * Category 1 we fix up by reattempting the load, and zeroing the segment 842 * register if the load fails. 843 * Category 2 we fix up by jumping to do_iret_error. We cannot use the 844 * normal Linux return path in this case because if we use the IRET hypercall 845 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 846 * We distinguish between categories by maintaining a status value in EAX. 847 */ 848ENTRY(xen_failsafe_callback) 849 pushl %eax 850 movl $1, %eax 8511: mov 4(%esp), %ds 8522: mov 8(%esp), %es 8533: mov 12(%esp), %fs 8544: mov 16(%esp), %gs 855 /* EAX == 0 => Category 1 (Bad segment) 856 EAX != 0 => Category 2 (Bad IRET) */ 857 testl %eax, %eax 858 popl %eax 859 lea 16(%esp), %esp 860 jz 5f 861 jmp iret_exc 8625: pushl $-1 /* orig_ax = -1 => not a system call */ 863 SAVE_ALL 864 ENCODE_FRAME_POINTER 865 jmp ret_from_exception 866 867.section .fixup, "ax" 8686: xorl %eax, %eax 869 movl %eax, 4(%esp) 870 jmp 1b 8717: xorl %eax, %eax 872 movl %eax, 8(%esp) 873 jmp 2b 8748: xorl %eax, %eax 875 movl %eax, 12(%esp) 876 jmp 3b 8779: xorl %eax, %eax 878 movl %eax, 16(%esp) 879 jmp 4b 880.previous 881 _ASM_EXTABLE(1b, 6b) 882 _ASM_EXTABLE(2b, 7b) 883 _ASM_EXTABLE(3b, 8b) 884 _ASM_EXTABLE(4b, 9b) 885ENDPROC(xen_failsafe_callback) 886 887BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 888 xen_evtchn_do_upcall) 889 890#endif /* CONFIG_XEN */ 891 892#if IS_ENABLED(CONFIG_HYPERV) 893 894BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 895 hyperv_vector_handler) 896 897#endif /* CONFIG_HYPERV */ 898 899ENTRY(page_fault) 900 ASM_CLAC 901 pushl $do_page_fault 902 ALIGN 903 jmp common_exception 904END(page_fault) 905 906common_exception: 907 /* the function address is in %gs's slot on the stack */ 908 pushl %fs 909 pushl %es 910 pushl %ds 911 pushl %eax 912 pushl %ebp 913 pushl %edi 914 pushl %esi 915 pushl %edx 916 pushl %ecx 917 pushl %ebx 918 ENCODE_FRAME_POINTER 919 cld 920 movl $(__KERNEL_PERCPU), %ecx 921 movl %ecx, %fs 922 UNWIND_ESPFIX_STACK 923 GS_TO_REG %ecx 924 movl PT_GS(%esp), %edi # get the function address 925 movl PT_ORIG_EAX(%esp), %edx # get the error code 926 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 927 REG_TO_PTGS %ecx 928 SET_KERNEL_GS %ecx 929 movl $(__USER_DS), %ecx 930 movl %ecx, %ds 931 movl %ecx, %es 932 TRACE_IRQS_OFF 933 movl %esp, %eax # pt_regs pointer 934 CALL_NOSPEC %edi 935 jmp ret_from_exception 936END(common_exception) 937 938ENTRY(debug) 939 /* 940 * #DB can happen at the first instruction of 941 * entry_SYSENTER_32 or in Xen's SYSENTER prologue. If this 942 * happens, then we will be running on a very small stack. We 943 * need to detect this condition and switch to the thread 944 * stack before calling any C code at all. 945 * 946 * If you edit this code, keep in mind that NMIs can happen in here. 947 */ 948 ASM_CLAC 949 pushl $-1 # mark this as an int 950 SAVE_ALL 951 ENCODE_FRAME_POINTER 952 xorl %edx, %edx # error code 0 953 movl %esp, %eax # pt_regs pointer 954 955 /* Are we currently on the SYSENTER stack? */ 956 movl PER_CPU_VAR(cpu_entry_area), %ecx 957 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx 958 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */ 959 cmpl $SIZEOF_entry_stack, %ecx 960 jb .Ldebug_from_sysenter_stack 961 962 TRACE_IRQS_OFF 963 call do_debug 964 jmp ret_from_exception 965 966.Ldebug_from_sysenter_stack: 967 /* We're on the SYSENTER stack. Switch off. */ 968 movl %esp, %ebx 969 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp 970 TRACE_IRQS_OFF 971 call do_debug 972 movl %ebx, %esp 973 jmp ret_from_exception 974END(debug) 975 976/* 977 * NMI is doubly nasty. It can happen on the first instruction of 978 * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning 979 * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32 980 * switched stacks. We handle both conditions by simply checking whether we 981 * interrupted kernel code running on the SYSENTER stack. 982 */ 983ENTRY(nmi) 984 ASM_CLAC 985#ifdef CONFIG_X86_ESPFIX32 986 pushl %eax 987 movl %ss, %eax 988 cmpw $__ESPFIX_SS, %ax 989 popl %eax 990 je .Lnmi_espfix_stack 991#endif 992 993 pushl %eax # pt_regs->orig_ax 994 SAVE_ALL 995 ENCODE_FRAME_POINTER 996 xorl %edx, %edx # zero error code 997 movl %esp, %eax # pt_regs pointer 998 999 /* Are we currently on the SYSENTER stack? */ 1000 movl PER_CPU_VAR(cpu_entry_area), %ecx 1001 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx 1002 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */ 1003 cmpl $SIZEOF_entry_stack, %ecx 1004 jb .Lnmi_from_sysenter_stack 1005 1006 /* Not on SYSENTER stack. */ 1007 call do_nmi 1008 jmp .Lrestore_all_notrace 1009 1010.Lnmi_from_sysenter_stack: 1011 /* 1012 * We're on the SYSENTER stack. Switch off. No one (not even debug) 1013 * is using the thread stack right now, so it's safe for us to use it. 1014 */ 1015 movl %esp, %ebx 1016 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp 1017 call do_nmi 1018 movl %ebx, %esp 1019 jmp .Lrestore_all_notrace 1020 1021#ifdef CONFIG_X86_ESPFIX32 1022.Lnmi_espfix_stack: 1023 /* 1024 * create the pointer to lss back 1025 */ 1026 pushl %ss 1027 pushl %esp 1028 addl $4, (%esp) 1029 /* copy the iret frame of 12 bytes */ 1030 .rept 3 1031 pushl 16(%esp) 1032 .endr 1033 pushl %eax 1034 SAVE_ALL 1035 ENCODE_FRAME_POINTER 1036 FIXUP_ESPFIX_STACK # %eax == %esp 1037 xorl %edx, %edx # zero error code 1038 call do_nmi 1039 RESTORE_REGS 1040 lss 12+4(%esp), %esp # back to espfix stack 1041 jmp .Lirq_return 1042#endif 1043END(nmi) 1044 1045ENTRY(int3) 1046 ASM_CLAC 1047 pushl $-1 # mark this as an int 1048 SAVE_ALL 1049 ENCODE_FRAME_POINTER 1050 TRACE_IRQS_OFF 1051 xorl %edx, %edx # zero error code 1052 movl %esp, %eax # pt_regs pointer 1053 call do_int3 1054 jmp ret_from_exception 1055END(int3) 1056 1057ENTRY(general_protection) 1058 pushl $do_general_protection 1059 jmp common_exception 1060END(general_protection) 1061 1062#ifdef CONFIG_KVM_GUEST 1063ENTRY(async_page_fault) 1064 ASM_CLAC 1065 pushl $do_async_page_fault 1066 jmp common_exception 1067END(async_page_fault) 1068#endif 1069 1070ENTRY(rewind_stack_do_exit) 1071 /* Prevent any naive code from trying to unwind to our caller. */ 1072 xorl %ebp, %ebp 1073 1074 movl PER_CPU_VAR(cpu_current_top_of_stack), %esi 1075 leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp 1076 1077 call do_exit 10781: jmp 1b 1079END(rewind_stack_do_exit) 1080