1/* 2 * Copyright (C) 1991,1992 Linus Torvalds 3 * 4 * entry_32.S contains the system-call and low-level fault and trap handling routines. 5 * 6 * Stack layout in 'syscall_exit': 7 * ptrace needs to have all registers on the stack. 8 * If the order here is changed, it needs to be 9 * updated in fork.c:copy_process(), signal.c:do_signal(), 10 * ptrace.c and ptrace.h 11 * 12 * 0(%esp) - %ebx 13 * 4(%esp) - %ecx 14 * 8(%esp) - %edx 15 * C(%esp) - %esi 16 * 10(%esp) - %edi 17 * 14(%esp) - %ebp 18 * 18(%esp) - %eax 19 * 1C(%esp) - %ds 20 * 20(%esp) - %es 21 * 24(%esp) - %fs 22 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 23 * 2C(%esp) - orig_eax 24 * 30(%esp) - %eip 25 * 34(%esp) - %cs 26 * 38(%esp) - %eflags 27 * 3C(%esp) - %oldesp 28 * 40(%esp) - %oldss 29 */ 30 31#include <linux/linkage.h> 32#include <linux/err.h> 33#include <asm/thread_info.h> 34#include <asm/irqflags.h> 35#include <asm/errno.h> 36#include <asm/segment.h> 37#include <asm/smp.h> 38#include <asm/page_types.h> 39#include <asm/percpu.h> 40#include <asm/processor-flags.h> 41#include <asm/ftrace.h> 42#include <asm/irq_vectors.h> 43#include <asm/cpufeature.h> 44#include <asm/alternative-asm.h> 45#include <asm/asm.h> 46#include <asm/smap.h> 47 48 .section .entry.text, "ax" 49 50/* 51 * We use macros for low-level operations which need to be overridden 52 * for paravirtualization. The following will never clobber any registers: 53 * INTERRUPT_RETURN (aka. "iret") 54 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 55 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 56 * 57 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 58 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 59 * Allowing a register to be clobbered can shrink the paravirt replacement 60 * enough to patch inline, increasing performance. 61 */ 62 63#ifdef CONFIG_PREEMPT 64# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 65#else 66# define preempt_stop(clobbers) 67# define resume_kernel restore_all 68#endif 69 70.macro TRACE_IRQS_IRET 71#ifdef CONFIG_TRACE_IRQFLAGS 72 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off? 73 jz 1f 74 TRACE_IRQS_ON 751: 76#endif 77.endm 78 79/* 80 * User gs save/restore 81 * 82 * %gs is used for userland TLS and kernel only uses it for stack 83 * canary which is required to be at %gs:20 by gcc. Read the comment 84 * at the top of stackprotector.h for more info. 85 * 86 * Local labels 98 and 99 are used. 87 */ 88#ifdef CONFIG_X86_32_LAZY_GS 89 90 /* unfortunately push/pop can't be no-op */ 91.macro PUSH_GS 92 pushl $0 93.endm 94.macro POP_GS pop=0 95 addl $(4 + \pop), %esp 96.endm 97.macro POP_GS_EX 98.endm 99 100 /* all the rest are no-op */ 101.macro PTGS_TO_GS 102.endm 103.macro PTGS_TO_GS_EX 104.endm 105.macro GS_TO_REG reg 106.endm 107.macro REG_TO_PTGS reg 108.endm 109.macro SET_KERNEL_GS reg 110.endm 111 112#else /* CONFIG_X86_32_LAZY_GS */ 113 114.macro PUSH_GS 115 pushl %gs 116.endm 117 118.macro POP_GS pop=0 11998: popl %gs 120 .if \pop <> 0 121 add $\pop, %esp 122 .endif 123.endm 124.macro POP_GS_EX 125.pushsection .fixup, "ax" 12699: movl $0, (%esp) 127 jmp 98b 128.popsection 129 _ASM_EXTABLE(98b, 99b) 130.endm 131 132.macro PTGS_TO_GS 13398: mov PT_GS(%esp), %gs 134.endm 135.macro PTGS_TO_GS_EX 136.pushsection .fixup, "ax" 13799: movl $0, PT_GS(%esp) 138 jmp 98b 139.popsection 140 _ASM_EXTABLE(98b, 99b) 141.endm 142 143.macro GS_TO_REG reg 144 movl %gs, \reg 145.endm 146.macro REG_TO_PTGS reg 147 movl \reg, PT_GS(%esp) 148.endm 149.macro SET_KERNEL_GS reg 150 movl $(__KERNEL_STACK_CANARY), \reg 151 movl \reg, %gs 152.endm 153 154#endif /* CONFIG_X86_32_LAZY_GS */ 155 156.macro SAVE_ALL 157 cld 158 PUSH_GS 159 pushl %fs 160 pushl %es 161 pushl %ds 162 pushl %eax 163 pushl %ebp 164 pushl %edi 165 pushl %esi 166 pushl %edx 167 pushl %ecx 168 pushl %ebx 169 movl $(__USER_DS), %edx 170 movl %edx, %ds 171 movl %edx, %es 172 movl $(__KERNEL_PERCPU), %edx 173 movl %edx, %fs 174 SET_KERNEL_GS %edx 175.endm 176 177.macro RESTORE_INT_REGS 178 popl %ebx 179 popl %ecx 180 popl %edx 181 popl %esi 182 popl %edi 183 popl %ebp 184 popl %eax 185.endm 186 187.macro RESTORE_REGS pop=0 188 RESTORE_INT_REGS 1891: popl %ds 1902: popl %es 1913: popl %fs 192 POP_GS \pop 193.pushsection .fixup, "ax" 1944: movl $0, (%esp) 195 jmp 1b 1965: movl $0, (%esp) 197 jmp 2b 1986: movl $0, (%esp) 199 jmp 3b 200.popsection 201 _ASM_EXTABLE(1b, 4b) 202 _ASM_EXTABLE(2b, 5b) 203 _ASM_EXTABLE(3b, 6b) 204 POP_GS_EX 205.endm 206 207ENTRY(ret_from_fork) 208 pushl %eax 209 call schedule_tail 210 GET_THREAD_INFO(%ebp) 211 popl %eax 212 pushl $0x0202 # Reset kernel eflags 213 popfl 214 jmp syscall_exit 215END(ret_from_fork) 216 217ENTRY(ret_from_kernel_thread) 218 pushl %eax 219 call schedule_tail 220 GET_THREAD_INFO(%ebp) 221 popl %eax 222 pushl $0x0202 # Reset kernel eflags 223 popfl 224 movl PT_EBP(%esp), %eax 225 call *PT_EBX(%esp) 226 movl $0, PT_EAX(%esp) 227 jmp syscall_exit 228ENDPROC(ret_from_kernel_thread) 229 230/* 231 * Return to user mode is not as complex as all this looks, 232 * but we want the default path for a system call return to 233 * go as quickly as possible which is why some of this is 234 * less clear than it otherwise should be. 235 */ 236 237 # userspace resumption stub bypassing syscall exit tracing 238 ALIGN 239ret_from_exception: 240 preempt_stop(CLBR_ANY) 241ret_from_intr: 242 GET_THREAD_INFO(%ebp) 243#ifdef CONFIG_VM86 244 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 245 movb PT_CS(%esp), %al 246 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 247#else 248 /* 249 * We can be coming here from child spawned by kernel_thread(). 250 */ 251 movl PT_CS(%esp), %eax 252 andl $SEGMENT_RPL_MASK, %eax 253#endif 254 cmpl $USER_RPL, %eax 255 jb resume_kernel # not returning to v8086 or userspace 256 257ENTRY(resume_userspace) 258 LOCKDEP_SYS_EXIT 259 DISABLE_INTERRUPTS(CLBR_ANY) 260 TRACE_IRQS_OFF 261 movl %esp, %eax 262 call prepare_exit_to_usermode 263 jmp restore_all 264END(ret_from_exception) 265 266#ifdef CONFIG_PREEMPT 267ENTRY(resume_kernel) 268 DISABLE_INTERRUPTS(CLBR_ANY) 269need_resched: 270 cmpl $0, PER_CPU_VAR(__preempt_count) 271 jnz restore_all 272 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? 273 jz restore_all 274 call preempt_schedule_irq 275 jmp need_resched 276END(resume_kernel) 277#endif 278 279/* 280 * SYSENTER_RETURN points to after the SYSENTER instruction 281 * in the vsyscall page. See vsyscall-sysentry.S, which defines 282 * the symbol. 283 */ 284 285 # SYSENTER call handler stub 286ENTRY(entry_SYSENTER_32) 287 movl TSS_sysenter_sp0(%esp), %esp 288sysenter_past_esp: 289 /* 290 * Interrupts are disabled here, but we can't trace it until 291 * enough kernel state to call TRACE_IRQS_OFF can be called - but 292 * we immediately enable interrupts at that point anyway. 293 */ 294 pushl $__USER_DS 295 pushl %ebp 296 pushfl 297 orl $X86_EFLAGS_IF, (%esp) 298 pushl $__USER_CS 299 /* 300 * Push current_thread_info()->sysenter_return to the stack. 301 * A tiny bit of offset fixup is necessary: TI_sysenter_return 302 * is relative to thread_info, which is at the bottom of the 303 * kernel stack page. 4*4 means the 4 words pushed above; 304 * TOP_OF_KERNEL_STACK_PADDING takes us to the top of the stack; 305 * and THREAD_SIZE takes us to the bottom. 306 */ 307 pushl ((TI_sysenter_return) - THREAD_SIZE + TOP_OF_KERNEL_STACK_PADDING + 4*4)(%esp) 308 309 pushl %eax 310 SAVE_ALL 311 ENABLE_INTERRUPTS(CLBR_NONE) 312 313/* 314 * Load the potential sixth argument from user stack. 315 * Careful about security. 316 */ 317 cmpl $__PAGE_OFFSET-3, %ebp 318 jae syscall_fault 319 ASM_STAC 3201: movl (%ebp), %ebp 321 ASM_CLAC 322 movl %ebp, PT_EBP(%esp) 323 _ASM_EXTABLE(1b, syscall_fault) 324 325 GET_THREAD_INFO(%ebp) 326 327 testl $_TIF_WORK_SYSCALL_ENTRY, TI_flags(%ebp) 328 jnz syscall_trace_entry 329sysenter_do_call: 330 cmpl $(NR_syscalls), %eax 331 jae sysenter_badsys 332 call *sys_call_table(, %eax, 4) 333sysenter_after_call: 334 movl %eax, PT_EAX(%esp) 335 LOCKDEP_SYS_EXIT 336 DISABLE_INTERRUPTS(CLBR_ANY) 337 TRACE_IRQS_OFF 338 movl TI_flags(%ebp), %ecx 339 testl $_TIF_ALLWORK_MASK, %ecx 340 jnz syscall_exit_work_irqs_off 341sysenter_exit: 342/* if something modifies registers it must also disable sysexit */ 343 movl PT_EIP(%esp), %edx 344 movl PT_OLDESP(%esp), %ecx 345 xorl %ebp, %ebp 346 TRACE_IRQS_ON 3471: mov PT_FS(%esp), %fs 348 PTGS_TO_GS 349 ENABLE_INTERRUPTS_SYSEXIT 350 351.pushsection .fixup, "ax" 3522: movl $0, PT_FS(%esp) 353 jmp 1b 354.popsection 355 _ASM_EXTABLE(1b, 2b) 356 PTGS_TO_GS_EX 357ENDPROC(entry_SYSENTER_32) 358 359 # system call handler stub 360ENTRY(entry_INT80_32) 361 ASM_CLAC 362 pushl %eax # save orig_eax 363 SAVE_ALL 364 GET_THREAD_INFO(%ebp) 365 # system call tracing in operation / emulation 366 testl $_TIF_WORK_SYSCALL_ENTRY, TI_flags(%ebp) 367 jnz syscall_trace_entry 368 cmpl $(NR_syscalls), %eax 369 jae syscall_badsys 370syscall_call: 371 call *sys_call_table(, %eax, 4) 372syscall_after_call: 373 movl %eax, PT_EAX(%esp) # store the return value 374syscall_exit: 375 LOCKDEP_SYS_EXIT 376 jmp syscall_exit_work 377 378restore_all: 379 TRACE_IRQS_IRET 380restore_all_notrace: 381#ifdef CONFIG_X86_ESPFIX32 382 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 383 /* 384 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we 385 * are returning to the kernel. 386 * See comments in process.c:copy_thread() for details. 387 */ 388 movb PT_OLDSS(%esp), %ah 389 movb PT_CS(%esp), %al 390 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 391 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 392 je ldt_ss # returning to user-space with LDT SS 393#endif 394restore_nocheck: 395 RESTORE_REGS 4 # skip orig_eax/error_code 396irq_return: 397 INTERRUPT_RETURN 398.section .fixup, "ax" 399ENTRY(iret_exc ) 400 pushl $0 # no error code 401 pushl $do_iret_error 402 jmp error_code 403.previous 404 _ASM_EXTABLE(irq_return, iret_exc) 405 406#ifdef CONFIG_X86_ESPFIX32 407ldt_ss: 408#ifdef CONFIG_PARAVIRT 409 /* 410 * The kernel can't run on a non-flat stack if paravirt mode 411 * is active. Rather than try to fixup the high bits of 412 * ESP, bypass this code entirely. This may break DOSemu 413 * and/or Wine support in a paravirt VM, although the option 414 * is still available to implement the setting of the high 415 * 16-bits in the INTERRUPT_RETURN paravirt-op. 416 */ 417 cmpl $0, pv_info+PARAVIRT_enabled 418 jne restore_nocheck 419#endif 420 421/* 422 * Setup and switch to ESPFIX stack 423 * 424 * We're returning to userspace with a 16 bit stack. The CPU will not 425 * restore the high word of ESP for us on executing iret... This is an 426 * "official" bug of all the x86-compatible CPUs, which we can work 427 * around to make dosemu and wine happy. We do this by preloading the 428 * high word of ESP with the high word of the userspace ESP while 429 * compensating for the offset by changing to the ESPFIX segment with 430 * a base address that matches for the difference. 431 */ 432#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 433 mov %esp, %edx /* load kernel esp */ 434 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 435 mov %dx, %ax /* eax: new kernel esp */ 436 sub %eax, %edx /* offset (low word is 0) */ 437 shr $16, %edx 438 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 439 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 440 pushl $__ESPFIX_SS 441 pushl %eax /* new kernel esp */ 442 /* 443 * Disable interrupts, but do not irqtrace this section: we 444 * will soon execute iret and the tracer was already set to 445 * the irqstate after the IRET: 446 */ 447 DISABLE_INTERRUPTS(CLBR_EAX) 448 lss (%esp), %esp /* switch to espfix segment */ 449 jmp restore_nocheck 450#endif 451ENDPROC(entry_INT80_32) 452 453 # perform syscall exit tracing 454 ALIGN 455syscall_trace_entry: 456 movl $-ENOSYS, PT_EAX(%esp) 457 movl %esp, %eax 458 call syscall_trace_enter 459 /* What it returned is what we'll actually use. */ 460 cmpl $(NR_syscalls), %eax 461 jnae syscall_call 462 jmp syscall_exit 463END(syscall_trace_entry) 464 465 # perform syscall exit tracing 466 ALIGN 467syscall_exit_work_irqs_off: 468 TRACE_IRQS_ON 469 ENABLE_INTERRUPTS(CLBR_ANY) 470 471syscall_exit_work: 472 movl %esp, %eax 473 call syscall_return_slowpath 474 jmp restore_all 475END(syscall_exit_work) 476 477syscall_fault: 478 ASM_CLAC 479 GET_THREAD_INFO(%ebp) 480 movl $-EFAULT, PT_EAX(%esp) 481 jmp resume_userspace 482END(syscall_fault) 483 484syscall_badsys: 485 movl $-ENOSYS, %eax 486 jmp syscall_after_call 487END(syscall_badsys) 488 489sysenter_badsys: 490 movl $-ENOSYS, %eax 491 jmp sysenter_after_call 492END(sysenter_badsys) 493 494.macro FIXUP_ESPFIX_STACK 495/* 496 * Switch back for ESPFIX stack to the normal zerobased stack 497 * 498 * We can't call C functions using the ESPFIX stack. This code reads 499 * the high word of the segment base from the GDT and swiches to the 500 * normal stack and adjusts ESP with the matching offset. 501 */ 502#ifdef CONFIG_X86_ESPFIX32 503 /* fixup the stack */ 504 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 505 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 506 shl $16, %eax 507 addl %esp, %eax /* the adjusted stack pointer */ 508 pushl $__KERNEL_DS 509 pushl %eax 510 lss (%esp), %esp /* switch to the normal stack segment */ 511#endif 512.endm 513.macro UNWIND_ESPFIX_STACK 514#ifdef CONFIG_X86_ESPFIX32 515 movl %ss, %eax 516 /* see if on espfix stack */ 517 cmpw $__ESPFIX_SS, %ax 518 jne 27f 519 movl $__KERNEL_DS, %eax 520 movl %eax, %ds 521 movl %eax, %es 522 /* switch to normal stack */ 523 FIXUP_ESPFIX_STACK 52427: 525#endif 526.endm 527 528/* 529 * Build the entry stubs with some assembler magic. 530 * We pack 1 stub into every 8-byte block. 531 */ 532 .align 8 533ENTRY(irq_entries_start) 534 vector=FIRST_EXTERNAL_VECTOR 535 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 536 pushl $(~vector+0x80) /* Note: always in signed byte range */ 537 vector=vector+1 538 jmp common_interrupt 539 .align 8 540 .endr 541END(irq_entries_start) 542 543/* 544 * the CPU automatically disables interrupts when executing an IRQ vector, 545 * so IRQ-flags tracing has to follow that: 546 */ 547 .p2align CONFIG_X86_L1_CACHE_SHIFT 548common_interrupt: 549 ASM_CLAC 550 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ 551 SAVE_ALL 552 TRACE_IRQS_OFF 553 movl %esp, %eax 554 call do_IRQ 555 jmp ret_from_intr 556ENDPROC(common_interrupt) 557 558#define BUILD_INTERRUPT3(name, nr, fn) \ 559ENTRY(name) \ 560 ASM_CLAC; \ 561 pushl $~(nr); \ 562 SAVE_ALL; \ 563 TRACE_IRQS_OFF \ 564 movl %esp, %eax; \ 565 call fn; \ 566 jmp ret_from_intr; \ 567ENDPROC(name) 568 569 570#ifdef CONFIG_TRACING 571# define TRACE_BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name) 572#else 573# define TRACE_BUILD_INTERRUPT(name, nr) 574#endif 575 576#define BUILD_INTERRUPT(name, nr) \ 577 BUILD_INTERRUPT3(name, nr, smp_##name); \ 578 TRACE_BUILD_INTERRUPT(name, nr) 579 580/* The include is where all of the SMP etc. interrupts come from */ 581#include <asm/entry_arch.h> 582 583ENTRY(coprocessor_error) 584 ASM_CLAC 585 pushl $0 586 pushl $do_coprocessor_error 587 jmp error_code 588END(coprocessor_error) 589 590ENTRY(simd_coprocessor_error) 591 ASM_CLAC 592 pushl $0 593#ifdef CONFIG_X86_INVD_BUG 594 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 595 ALTERNATIVE "pushl $do_general_protection", \ 596 "pushl $do_simd_coprocessor_error", \ 597 X86_FEATURE_XMM 598#else 599 pushl $do_simd_coprocessor_error 600#endif 601 jmp error_code 602END(simd_coprocessor_error) 603 604ENTRY(device_not_available) 605 ASM_CLAC 606 pushl $-1 # mark this as an int 607 pushl $do_device_not_available 608 jmp error_code 609END(device_not_available) 610 611#ifdef CONFIG_PARAVIRT 612ENTRY(native_iret) 613 iret 614 _ASM_EXTABLE(native_iret, iret_exc) 615END(native_iret) 616 617ENTRY(native_irq_enable_sysexit) 618 sti 619 sysexit 620END(native_irq_enable_sysexit) 621#endif 622 623ENTRY(overflow) 624 ASM_CLAC 625 pushl $0 626 pushl $do_overflow 627 jmp error_code 628END(overflow) 629 630ENTRY(bounds) 631 ASM_CLAC 632 pushl $0 633 pushl $do_bounds 634 jmp error_code 635END(bounds) 636 637ENTRY(invalid_op) 638 ASM_CLAC 639 pushl $0 640 pushl $do_invalid_op 641 jmp error_code 642END(invalid_op) 643 644ENTRY(coprocessor_segment_overrun) 645 ASM_CLAC 646 pushl $0 647 pushl $do_coprocessor_segment_overrun 648 jmp error_code 649END(coprocessor_segment_overrun) 650 651ENTRY(invalid_TSS) 652 ASM_CLAC 653 pushl $do_invalid_TSS 654 jmp error_code 655END(invalid_TSS) 656 657ENTRY(segment_not_present) 658 ASM_CLAC 659 pushl $do_segment_not_present 660 jmp error_code 661END(segment_not_present) 662 663ENTRY(stack_segment) 664 ASM_CLAC 665 pushl $do_stack_segment 666 jmp error_code 667END(stack_segment) 668 669ENTRY(alignment_check) 670 ASM_CLAC 671 pushl $do_alignment_check 672 jmp error_code 673END(alignment_check) 674 675ENTRY(divide_error) 676 ASM_CLAC 677 pushl $0 # no error code 678 pushl $do_divide_error 679 jmp error_code 680END(divide_error) 681 682#ifdef CONFIG_X86_MCE 683ENTRY(machine_check) 684 ASM_CLAC 685 pushl $0 686 pushl machine_check_vector 687 jmp error_code 688END(machine_check) 689#endif 690 691ENTRY(spurious_interrupt_bug) 692 ASM_CLAC 693 pushl $0 694 pushl $do_spurious_interrupt_bug 695 jmp error_code 696END(spurious_interrupt_bug) 697 698#ifdef CONFIG_XEN 699/* 700 * Xen doesn't set %esp to be precisely what the normal SYSENTER 701 * entry point expects, so fix it up before using the normal path. 702 */ 703ENTRY(xen_sysenter_target) 704 addl $5*4, %esp /* remove xen-provided frame */ 705 jmp sysenter_past_esp 706 707ENTRY(xen_hypervisor_callback) 708 pushl $-1 /* orig_ax = -1 => not a system call */ 709 SAVE_ALL 710 TRACE_IRQS_OFF 711 712 /* 713 * Check to see if we got the event in the critical 714 * region in xen_iret_direct, after we've reenabled 715 * events and checked for pending events. This simulates 716 * iret instruction's behaviour where it delivers a 717 * pending interrupt when enabling interrupts: 718 */ 719 movl PT_EIP(%esp), %eax 720 cmpl $xen_iret_start_crit, %eax 721 jb 1f 722 cmpl $xen_iret_end_crit, %eax 723 jae 1f 724 725 jmp xen_iret_crit_fixup 726 727ENTRY(xen_do_upcall) 7281: mov %esp, %eax 729 call xen_evtchn_do_upcall 730#ifndef CONFIG_PREEMPT 731 call xen_maybe_preempt_hcall 732#endif 733 jmp ret_from_intr 734ENDPROC(xen_hypervisor_callback) 735 736/* 737 * Hypervisor uses this for application faults while it executes. 738 * We get here for two reasons: 739 * 1. Fault while reloading DS, ES, FS or GS 740 * 2. Fault while executing IRET 741 * Category 1 we fix up by reattempting the load, and zeroing the segment 742 * register if the load fails. 743 * Category 2 we fix up by jumping to do_iret_error. We cannot use the 744 * normal Linux return path in this case because if we use the IRET hypercall 745 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 746 * We distinguish between categories by maintaining a status value in EAX. 747 */ 748ENTRY(xen_failsafe_callback) 749 pushl %eax 750 movl $1, %eax 7511: mov 4(%esp), %ds 7522: mov 8(%esp), %es 7533: mov 12(%esp), %fs 7544: mov 16(%esp), %gs 755 /* EAX == 0 => Category 1 (Bad segment) 756 EAX != 0 => Category 2 (Bad IRET) */ 757 testl %eax, %eax 758 popl %eax 759 lea 16(%esp), %esp 760 jz 5f 761 jmp iret_exc 7625: pushl $-1 /* orig_ax = -1 => not a system call */ 763 SAVE_ALL 764 jmp ret_from_exception 765 766.section .fixup, "ax" 7676: xorl %eax, %eax 768 movl %eax, 4(%esp) 769 jmp 1b 7707: xorl %eax, %eax 771 movl %eax, 8(%esp) 772 jmp 2b 7738: xorl %eax, %eax 774 movl %eax, 12(%esp) 775 jmp 3b 7769: xorl %eax, %eax 777 movl %eax, 16(%esp) 778 jmp 4b 779.previous 780 _ASM_EXTABLE(1b, 6b) 781 _ASM_EXTABLE(2b, 7b) 782 _ASM_EXTABLE(3b, 8b) 783 _ASM_EXTABLE(4b, 9b) 784ENDPROC(xen_failsafe_callback) 785 786BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 787 xen_evtchn_do_upcall) 788 789#endif /* CONFIG_XEN */ 790 791#if IS_ENABLED(CONFIG_HYPERV) 792 793BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 794 hyperv_vector_handler) 795 796#endif /* CONFIG_HYPERV */ 797 798#ifdef CONFIG_FUNCTION_TRACER 799#ifdef CONFIG_DYNAMIC_FTRACE 800 801ENTRY(mcount) 802 ret 803END(mcount) 804 805ENTRY(ftrace_caller) 806 pushl %eax 807 pushl %ecx 808 pushl %edx 809 pushl $0 /* Pass NULL as regs pointer */ 810 movl 4*4(%esp), %eax 811 movl 0x4(%ebp), %edx 812 movl function_trace_op, %ecx 813 subl $MCOUNT_INSN_SIZE, %eax 814 815.globl ftrace_call 816ftrace_call: 817 call ftrace_stub 818 819 addl $4, %esp /* skip NULL pointer */ 820 popl %edx 821 popl %ecx 822 popl %eax 823ftrace_ret: 824#ifdef CONFIG_FUNCTION_GRAPH_TRACER 825.globl ftrace_graph_call 826ftrace_graph_call: 827 jmp ftrace_stub 828#endif 829 830.globl ftrace_stub 831ftrace_stub: 832 ret 833END(ftrace_caller) 834 835ENTRY(ftrace_regs_caller) 836 pushf /* push flags before compare (in cs location) */ 837 838 /* 839 * i386 does not save SS and ESP when coming from kernel. 840 * Instead, to get sp, ®s->sp is used (see ptrace.h). 841 * Unfortunately, that means eflags must be at the same location 842 * as the current return ip is. We move the return ip into the 843 * ip location, and move flags into the return ip location. 844 */ 845 pushl 4(%esp) /* save return ip into ip slot */ 846 847 pushl $0 /* Load 0 into orig_ax */ 848 pushl %gs 849 pushl %fs 850 pushl %es 851 pushl %ds 852 pushl %eax 853 pushl %ebp 854 pushl %edi 855 pushl %esi 856 pushl %edx 857 pushl %ecx 858 pushl %ebx 859 860 movl 13*4(%esp), %eax /* Get the saved flags */ 861 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ 862 /* clobbering return ip */ 863 movl $__KERNEL_CS, 13*4(%esp) 864 865 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 866 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 867 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 868 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 869 pushl %esp /* Save pt_regs as 4th parameter */ 870 871GLOBAL(ftrace_regs_call) 872 call ftrace_stub 873 874 addl $4, %esp /* Skip pt_regs */ 875 movl 14*4(%esp), %eax /* Move flags back into cs */ 876 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ 877 movl 12*4(%esp), %eax /* Get return ip from regs->ip */ 878 movl %eax, 14*4(%esp) /* Put return ip back for ret */ 879 880 popl %ebx 881 popl %ecx 882 popl %edx 883 popl %esi 884 popl %edi 885 popl %ebp 886 popl %eax 887 popl %ds 888 popl %es 889 popl %fs 890 popl %gs 891 addl $8, %esp /* Skip orig_ax and ip */ 892 popf /* Pop flags at end (no addl to corrupt flags) */ 893 jmp ftrace_ret 894 895 popf 896 jmp ftrace_stub 897#else /* ! CONFIG_DYNAMIC_FTRACE */ 898 899ENTRY(mcount) 900 cmpl $__PAGE_OFFSET, %esp 901 jb ftrace_stub /* Paging not enabled yet? */ 902 903 cmpl $ftrace_stub, ftrace_trace_function 904 jnz trace 905#ifdef CONFIG_FUNCTION_GRAPH_TRACER 906 cmpl $ftrace_stub, ftrace_graph_return 907 jnz ftrace_graph_caller 908 909 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 910 jnz ftrace_graph_caller 911#endif 912.globl ftrace_stub 913ftrace_stub: 914 ret 915 916 /* taken from glibc */ 917trace: 918 pushl %eax 919 pushl %ecx 920 pushl %edx 921 movl 0xc(%esp), %eax 922 movl 0x4(%ebp), %edx 923 subl $MCOUNT_INSN_SIZE, %eax 924 925 call *ftrace_trace_function 926 927 popl %edx 928 popl %ecx 929 popl %eax 930 jmp ftrace_stub 931END(mcount) 932#endif /* CONFIG_DYNAMIC_FTRACE */ 933#endif /* CONFIG_FUNCTION_TRACER */ 934 935#ifdef CONFIG_FUNCTION_GRAPH_TRACER 936ENTRY(ftrace_graph_caller) 937 pushl %eax 938 pushl %ecx 939 pushl %edx 940 movl 0xc(%esp), %eax 941 lea 0x4(%ebp), %edx 942 movl (%ebp), %ecx 943 subl $MCOUNT_INSN_SIZE, %eax 944 call prepare_ftrace_return 945 popl %edx 946 popl %ecx 947 popl %eax 948 ret 949END(ftrace_graph_caller) 950 951.globl return_to_handler 952return_to_handler: 953 pushl %eax 954 pushl %edx 955 movl %ebp, %eax 956 call ftrace_return_to_handler 957 movl %eax, %ecx 958 popl %edx 959 popl %eax 960 jmp *%ecx 961#endif 962 963#ifdef CONFIG_TRACING 964ENTRY(trace_page_fault) 965 ASM_CLAC 966 pushl $trace_do_page_fault 967 jmp error_code 968END(trace_page_fault) 969#endif 970 971ENTRY(page_fault) 972 ASM_CLAC 973 pushl $do_page_fault 974 ALIGN 975error_code: 976 /* the function address is in %gs's slot on the stack */ 977 pushl %fs 978 pushl %es 979 pushl %ds 980 pushl %eax 981 pushl %ebp 982 pushl %edi 983 pushl %esi 984 pushl %edx 985 pushl %ecx 986 pushl %ebx 987 cld 988 movl $(__KERNEL_PERCPU), %ecx 989 movl %ecx, %fs 990 UNWIND_ESPFIX_STACK 991 GS_TO_REG %ecx 992 movl PT_GS(%esp), %edi # get the function address 993 movl PT_ORIG_EAX(%esp), %edx # get the error code 994 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 995 REG_TO_PTGS %ecx 996 SET_KERNEL_GS %ecx 997 movl $(__USER_DS), %ecx 998 movl %ecx, %ds 999 movl %ecx, %es 1000 TRACE_IRQS_OFF 1001 movl %esp, %eax # pt_regs pointer 1002 call *%edi 1003 jmp ret_from_exception 1004END(page_fault) 1005 1006/* 1007 * Debug traps and NMI can happen at the one SYSENTER instruction 1008 * that sets up the real kernel stack. Check here, since we can't 1009 * allow the wrong stack to be used. 1010 * 1011 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 1012 * already pushed 3 words if it hits on the sysenter instruction: 1013 * eflags, cs and eip. 1014 * 1015 * We just load the right stack, and push the three (known) values 1016 * by hand onto the new stack - while updating the return eip past 1017 * the instruction that would have done it for sysenter. 1018 */ 1019.macro FIX_STACK offset ok label 1020 cmpw $__KERNEL_CS, 4(%esp) 1021 jne \ok 1022\label: 1023 movl TSS_sysenter_sp0 + \offset(%esp), %esp 1024 pushfl 1025 pushl $__KERNEL_CS 1026 pushl $sysenter_past_esp 1027.endm 1028 1029ENTRY(debug) 1030 ASM_CLAC 1031 cmpl $entry_SYSENTER_32, (%esp) 1032 jne debug_stack_correct 1033 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 1034debug_stack_correct: 1035 pushl $-1 # mark this as an int 1036 SAVE_ALL 1037 TRACE_IRQS_OFF 1038 xorl %edx, %edx # error code 0 1039 movl %esp, %eax # pt_regs pointer 1040 call do_debug 1041 jmp ret_from_exception 1042END(debug) 1043 1044/* 1045 * NMI is doubly nasty. It can happen _while_ we're handling 1046 * a debug fault, and the debug fault hasn't yet been able to 1047 * clear up the stack. So we first check whether we got an 1048 * NMI on the sysenter entry path, but after that we need to 1049 * check whether we got an NMI on the debug path where the debug 1050 * fault happened on the sysenter path. 1051 */ 1052ENTRY(nmi) 1053 ASM_CLAC 1054#ifdef CONFIG_X86_ESPFIX32 1055 pushl %eax 1056 movl %ss, %eax 1057 cmpw $__ESPFIX_SS, %ax 1058 popl %eax 1059 je nmi_espfix_stack 1060#endif 1061 cmpl $entry_SYSENTER_32, (%esp) 1062 je nmi_stack_fixup 1063 pushl %eax 1064 movl %esp, %eax 1065 /* 1066 * Do not access memory above the end of our stack page, 1067 * it might not exist. 1068 */ 1069 andl $(THREAD_SIZE-1), %eax 1070 cmpl $(THREAD_SIZE-20), %eax 1071 popl %eax 1072 jae nmi_stack_correct 1073 cmpl $entry_SYSENTER_32, 12(%esp) 1074 je nmi_debug_stack_check 1075nmi_stack_correct: 1076 pushl %eax 1077 SAVE_ALL 1078 xorl %edx, %edx # zero error code 1079 movl %esp, %eax # pt_regs pointer 1080 call do_nmi 1081 jmp restore_all_notrace 1082 1083nmi_stack_fixup: 1084 FIX_STACK 12, nmi_stack_correct, 1 1085 jmp nmi_stack_correct 1086 1087nmi_debug_stack_check: 1088 cmpw $__KERNEL_CS, 16(%esp) 1089 jne nmi_stack_correct 1090 cmpl $debug, (%esp) 1091 jb nmi_stack_correct 1092 cmpl $debug_esp_fix_insn, (%esp) 1093 ja nmi_stack_correct 1094 FIX_STACK 24, nmi_stack_correct, 1 1095 jmp nmi_stack_correct 1096 1097#ifdef CONFIG_X86_ESPFIX32 1098nmi_espfix_stack: 1099 /* 1100 * create the pointer to lss back 1101 */ 1102 pushl %ss 1103 pushl %esp 1104 addl $4, (%esp) 1105 /* copy the iret frame of 12 bytes */ 1106 .rept 3 1107 pushl 16(%esp) 1108 .endr 1109 pushl %eax 1110 SAVE_ALL 1111 FIXUP_ESPFIX_STACK # %eax == %esp 1112 xorl %edx, %edx # zero error code 1113 call do_nmi 1114 RESTORE_REGS 1115 lss 12+4(%esp), %esp # back to espfix stack 1116 jmp irq_return 1117#endif 1118END(nmi) 1119 1120ENTRY(int3) 1121 ASM_CLAC 1122 pushl $-1 # mark this as an int 1123 SAVE_ALL 1124 TRACE_IRQS_OFF 1125 xorl %edx, %edx # zero error code 1126 movl %esp, %eax # pt_regs pointer 1127 call do_int3 1128 jmp ret_from_exception 1129END(int3) 1130 1131ENTRY(general_protection) 1132 pushl $do_general_protection 1133 jmp error_code 1134END(general_protection) 1135 1136#ifdef CONFIG_KVM_GUEST 1137ENTRY(async_page_fault) 1138 ASM_CLAC 1139 pushl $do_async_page_fault 1140 jmp error_code 1141END(async_page_fault) 1142#endif 1143