1/* 2 * Copyright (C) 1991,1992 Linus Torvalds 3 * 4 * entry_32.S contains the system-call and low-level fault and trap handling routines. 5 * 6 * Stack layout while running C code: 7 * ptrace needs to have all registers on the stack. 8 * If the order here is changed, it needs to be 9 * updated in fork.c:copy_process(), signal.c:do_signal(), 10 * ptrace.c and ptrace.h 11 * 12 * 0(%esp) - %ebx 13 * 4(%esp) - %ecx 14 * 8(%esp) - %edx 15 * C(%esp) - %esi 16 * 10(%esp) - %edi 17 * 14(%esp) - %ebp 18 * 18(%esp) - %eax 19 * 1C(%esp) - %ds 20 * 20(%esp) - %es 21 * 24(%esp) - %fs 22 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 23 * 2C(%esp) - orig_eax 24 * 30(%esp) - %eip 25 * 34(%esp) - %cs 26 * 38(%esp) - %eflags 27 * 3C(%esp) - %oldesp 28 * 40(%esp) - %oldss 29 */ 30 31#include <linux/linkage.h> 32#include <linux/err.h> 33#include <asm/thread_info.h> 34#include <asm/irqflags.h> 35#include <asm/errno.h> 36#include <asm/segment.h> 37#include <asm/smp.h> 38#include <asm/page_types.h> 39#include <asm/percpu.h> 40#include <asm/processor-flags.h> 41#include <asm/ftrace.h> 42#include <asm/irq_vectors.h> 43#include <asm/cpufeature.h> 44#include <asm/alternative-asm.h> 45#include <asm/asm.h> 46#include <asm/smap.h> 47 48 .section .entry.text, "ax" 49 50/* 51 * We use macros for low-level operations which need to be overridden 52 * for paravirtualization. The following will never clobber any registers: 53 * INTERRUPT_RETURN (aka. "iret") 54 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 55 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 56 * 57 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 58 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 59 * Allowing a register to be clobbered can shrink the paravirt replacement 60 * enough to patch inline, increasing performance. 61 */ 62 63#ifdef CONFIG_PREEMPT 64# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 65#else 66# define preempt_stop(clobbers) 67# define resume_kernel restore_all 68#endif 69 70.macro TRACE_IRQS_IRET 71#ifdef CONFIG_TRACE_IRQFLAGS 72 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off? 73 jz 1f 74 TRACE_IRQS_ON 751: 76#endif 77.endm 78 79/* 80 * User gs save/restore 81 * 82 * %gs is used for userland TLS and kernel only uses it for stack 83 * canary which is required to be at %gs:20 by gcc. Read the comment 84 * at the top of stackprotector.h for more info. 85 * 86 * Local labels 98 and 99 are used. 87 */ 88#ifdef CONFIG_X86_32_LAZY_GS 89 90 /* unfortunately push/pop can't be no-op */ 91.macro PUSH_GS 92 pushl $0 93.endm 94.macro POP_GS pop=0 95 addl $(4 + \pop), %esp 96.endm 97.macro POP_GS_EX 98.endm 99 100 /* all the rest are no-op */ 101.macro PTGS_TO_GS 102.endm 103.macro PTGS_TO_GS_EX 104.endm 105.macro GS_TO_REG reg 106.endm 107.macro REG_TO_PTGS reg 108.endm 109.macro SET_KERNEL_GS reg 110.endm 111 112#else /* CONFIG_X86_32_LAZY_GS */ 113 114.macro PUSH_GS 115 pushl %gs 116.endm 117 118.macro POP_GS pop=0 11998: popl %gs 120 .if \pop <> 0 121 add $\pop, %esp 122 .endif 123.endm 124.macro POP_GS_EX 125.pushsection .fixup, "ax" 12699: movl $0, (%esp) 127 jmp 98b 128.popsection 129 _ASM_EXTABLE(98b, 99b) 130.endm 131 132.macro PTGS_TO_GS 13398: mov PT_GS(%esp), %gs 134.endm 135.macro PTGS_TO_GS_EX 136.pushsection .fixup, "ax" 13799: movl $0, PT_GS(%esp) 138 jmp 98b 139.popsection 140 _ASM_EXTABLE(98b, 99b) 141.endm 142 143.macro GS_TO_REG reg 144 movl %gs, \reg 145.endm 146.macro REG_TO_PTGS reg 147 movl \reg, PT_GS(%esp) 148.endm 149.macro SET_KERNEL_GS reg 150 movl $(__KERNEL_STACK_CANARY), \reg 151 movl \reg, %gs 152.endm 153 154#endif /* CONFIG_X86_32_LAZY_GS */ 155 156.macro SAVE_ALL pt_regs_ax=%eax 157 cld 158 PUSH_GS 159 pushl %fs 160 pushl %es 161 pushl %ds 162 pushl \pt_regs_ax 163 pushl %ebp 164 pushl %edi 165 pushl %esi 166 pushl %edx 167 pushl %ecx 168 pushl %ebx 169 movl $(__USER_DS), %edx 170 movl %edx, %ds 171 movl %edx, %es 172 movl $(__KERNEL_PERCPU), %edx 173 movl %edx, %fs 174 SET_KERNEL_GS %edx 175.endm 176 177.macro RESTORE_INT_REGS 178 popl %ebx 179 popl %ecx 180 popl %edx 181 popl %esi 182 popl %edi 183 popl %ebp 184 popl %eax 185.endm 186 187.macro RESTORE_REGS pop=0 188 RESTORE_INT_REGS 1891: popl %ds 1902: popl %es 1913: popl %fs 192 POP_GS \pop 193.pushsection .fixup, "ax" 1944: movl $0, (%esp) 195 jmp 1b 1965: movl $0, (%esp) 197 jmp 2b 1986: movl $0, (%esp) 199 jmp 3b 200.popsection 201 _ASM_EXTABLE(1b, 4b) 202 _ASM_EXTABLE(2b, 5b) 203 _ASM_EXTABLE(3b, 6b) 204 POP_GS_EX 205.endm 206 207ENTRY(ret_from_fork) 208 pushl %eax 209 call schedule_tail 210 GET_THREAD_INFO(%ebp) 211 popl %eax 212 pushl $0x0202 # Reset kernel eflags 213 popfl 214 215 /* When we fork, we trace the syscall return in the child, too. */ 216 movl %esp, %eax 217 call syscall_return_slowpath 218 jmp restore_all 219END(ret_from_fork) 220 221ENTRY(ret_from_kernel_thread) 222 pushl %eax 223 call schedule_tail 224 GET_THREAD_INFO(%ebp) 225 popl %eax 226 pushl $0x0202 # Reset kernel eflags 227 popfl 228 movl PT_EBP(%esp), %eax 229 call *PT_EBX(%esp) 230 movl $0, PT_EAX(%esp) 231 232 /* 233 * Kernel threads return to userspace as if returning from a syscall. 234 * We should check whether anything actually uses this path and, if so, 235 * consider switching it over to ret_from_fork. 236 */ 237 movl %esp, %eax 238 call syscall_return_slowpath 239 jmp restore_all 240ENDPROC(ret_from_kernel_thread) 241 242/* 243 * Return to user mode is not as complex as all this looks, 244 * but we want the default path for a system call return to 245 * go as quickly as possible which is why some of this is 246 * less clear than it otherwise should be. 247 */ 248 249 # userspace resumption stub bypassing syscall exit tracing 250 ALIGN 251ret_from_exception: 252 preempt_stop(CLBR_ANY) 253ret_from_intr: 254 GET_THREAD_INFO(%ebp) 255#ifdef CONFIG_VM86 256 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 257 movb PT_CS(%esp), %al 258 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 259#else 260 /* 261 * We can be coming here from child spawned by kernel_thread(). 262 */ 263 movl PT_CS(%esp), %eax 264 andl $SEGMENT_RPL_MASK, %eax 265#endif 266 cmpl $USER_RPL, %eax 267 jb resume_kernel # not returning to v8086 or userspace 268 269ENTRY(resume_userspace) 270 DISABLE_INTERRUPTS(CLBR_ANY) 271 TRACE_IRQS_OFF 272 movl %esp, %eax 273 call prepare_exit_to_usermode 274 jmp restore_all 275END(ret_from_exception) 276 277#ifdef CONFIG_PREEMPT 278ENTRY(resume_kernel) 279 DISABLE_INTERRUPTS(CLBR_ANY) 280need_resched: 281 cmpl $0, PER_CPU_VAR(__preempt_count) 282 jnz restore_all 283 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? 284 jz restore_all 285 call preempt_schedule_irq 286 jmp need_resched 287END(resume_kernel) 288#endif 289 290 # SYSENTER call handler stub 291ENTRY(entry_SYSENTER_32) 292 movl TSS_sysenter_sp0(%esp), %esp 293sysenter_past_esp: 294 pushl $__USER_DS /* pt_regs->ss */ 295 pushl %ebp /* pt_regs->sp (stashed in bp) */ 296 pushfl /* pt_regs->flags (except IF = 0) */ 297 ASM_CLAC /* Clear AC after saving FLAGS */ 298 orl $X86_EFLAGS_IF, (%esp) /* Fix IF */ 299 pushl $__USER_CS /* pt_regs->cs */ 300 pushl $0 /* pt_regs->ip = 0 (placeholder) */ 301 pushl %eax /* pt_regs->orig_ax */ 302 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 303 304 /* 305 * User mode is traced as though IRQs are on, and SYSENTER 306 * turned them off. 307 */ 308 TRACE_IRQS_OFF 309 310 movl %esp, %eax 311 call do_fast_syscall_32 312 /* XEN PV guests always use IRET path */ 313 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \ 314 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV 315 316/* Opportunistic SYSEXIT */ 317 TRACE_IRQS_ON /* User mode traces as IRQs on. */ 318 movl PT_EIP(%esp), %edx /* pt_regs->ip */ 319 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */ 3201: mov PT_FS(%esp), %fs 321 PTGS_TO_GS 322 popl %ebx /* pt_regs->bx */ 323 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */ 324 popl %esi /* pt_regs->si */ 325 popl %edi /* pt_regs->di */ 326 popl %ebp /* pt_regs->bp */ 327 popl %eax /* pt_regs->ax */ 328 329 /* 330 * Return back to the vDSO, which will pop ecx and edx. 331 * Don't bother with DS and ES (they already contain __USER_DS). 332 */ 333 sti 334 sysexit 335 336.pushsection .fixup, "ax" 3372: movl $0, PT_FS(%esp) 338 jmp 1b 339.popsection 340 _ASM_EXTABLE(1b, 2b) 341 PTGS_TO_GS_EX 342ENDPROC(entry_SYSENTER_32) 343 344 # system call handler stub 345ENTRY(entry_INT80_32) 346 ASM_CLAC 347 pushl %eax /* pt_regs->orig_ax */ 348 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 349 350 /* 351 * User mode is traced as though IRQs are on. Unlike the 64-bit 352 * case, INT80 is a trap gate on 32-bit kernels, so interrupts 353 * are already on (unless user code is messing around with iopl). 354 */ 355 356 movl %esp, %eax 357 call do_syscall_32_irqs_on 358.Lsyscall_32_done: 359 360restore_all: 361 TRACE_IRQS_IRET 362restore_all_notrace: 363#ifdef CONFIG_X86_ESPFIX32 364 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 365 /* 366 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we 367 * are returning to the kernel. 368 * See comments in process.c:copy_thread() for details. 369 */ 370 movb PT_OLDSS(%esp), %ah 371 movb PT_CS(%esp), %al 372 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 373 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 374 je ldt_ss # returning to user-space with LDT SS 375#endif 376restore_nocheck: 377 RESTORE_REGS 4 # skip orig_eax/error_code 378irq_return: 379 INTERRUPT_RETURN 380.section .fixup, "ax" 381ENTRY(iret_exc ) 382 pushl $0 # no error code 383 pushl $do_iret_error 384 jmp error_code 385.previous 386 _ASM_EXTABLE(irq_return, iret_exc) 387 388#ifdef CONFIG_X86_ESPFIX32 389ldt_ss: 390#ifdef CONFIG_PARAVIRT 391 /* 392 * The kernel can't run on a non-flat stack if paravirt mode 393 * is active. Rather than try to fixup the high bits of 394 * ESP, bypass this code entirely. This may break DOSemu 395 * and/or Wine support in a paravirt VM, although the option 396 * is still available to implement the setting of the high 397 * 16-bits in the INTERRUPT_RETURN paravirt-op. 398 */ 399 cmpl $0, pv_info+PARAVIRT_enabled 400 jne restore_nocheck 401#endif 402 403/* 404 * Setup and switch to ESPFIX stack 405 * 406 * We're returning to userspace with a 16 bit stack. The CPU will not 407 * restore the high word of ESP for us on executing iret... This is an 408 * "official" bug of all the x86-compatible CPUs, which we can work 409 * around to make dosemu and wine happy. We do this by preloading the 410 * high word of ESP with the high word of the userspace ESP while 411 * compensating for the offset by changing to the ESPFIX segment with 412 * a base address that matches for the difference. 413 */ 414#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 415 mov %esp, %edx /* load kernel esp */ 416 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 417 mov %dx, %ax /* eax: new kernel esp */ 418 sub %eax, %edx /* offset (low word is 0) */ 419 shr $16, %edx 420 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 421 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 422 pushl $__ESPFIX_SS 423 pushl %eax /* new kernel esp */ 424 /* 425 * Disable interrupts, but do not irqtrace this section: we 426 * will soon execute iret and the tracer was already set to 427 * the irqstate after the IRET: 428 */ 429 DISABLE_INTERRUPTS(CLBR_EAX) 430 lss (%esp), %esp /* switch to espfix segment */ 431 jmp restore_nocheck 432#endif 433ENDPROC(entry_INT80_32) 434 435.macro FIXUP_ESPFIX_STACK 436/* 437 * Switch back for ESPFIX stack to the normal zerobased stack 438 * 439 * We can't call C functions using the ESPFIX stack. This code reads 440 * the high word of the segment base from the GDT and swiches to the 441 * normal stack and adjusts ESP with the matching offset. 442 */ 443#ifdef CONFIG_X86_ESPFIX32 444 /* fixup the stack */ 445 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 446 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 447 shl $16, %eax 448 addl %esp, %eax /* the adjusted stack pointer */ 449 pushl $__KERNEL_DS 450 pushl %eax 451 lss (%esp), %esp /* switch to the normal stack segment */ 452#endif 453.endm 454.macro UNWIND_ESPFIX_STACK 455#ifdef CONFIG_X86_ESPFIX32 456 movl %ss, %eax 457 /* see if on espfix stack */ 458 cmpw $__ESPFIX_SS, %ax 459 jne 27f 460 movl $__KERNEL_DS, %eax 461 movl %eax, %ds 462 movl %eax, %es 463 /* switch to normal stack */ 464 FIXUP_ESPFIX_STACK 46527: 466#endif 467.endm 468 469/* 470 * Build the entry stubs with some assembler magic. 471 * We pack 1 stub into every 8-byte block. 472 */ 473 .align 8 474ENTRY(irq_entries_start) 475 vector=FIRST_EXTERNAL_VECTOR 476 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 477 pushl $(~vector+0x80) /* Note: always in signed byte range */ 478 vector=vector+1 479 jmp common_interrupt 480 .align 8 481 .endr 482END(irq_entries_start) 483 484/* 485 * the CPU automatically disables interrupts when executing an IRQ vector, 486 * so IRQ-flags tracing has to follow that: 487 */ 488 .p2align CONFIG_X86_L1_CACHE_SHIFT 489common_interrupt: 490 ASM_CLAC 491 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ 492 SAVE_ALL 493 TRACE_IRQS_OFF 494 movl %esp, %eax 495 call do_IRQ 496 jmp ret_from_intr 497ENDPROC(common_interrupt) 498 499#define BUILD_INTERRUPT3(name, nr, fn) \ 500ENTRY(name) \ 501 ASM_CLAC; \ 502 pushl $~(nr); \ 503 SAVE_ALL; \ 504 TRACE_IRQS_OFF \ 505 movl %esp, %eax; \ 506 call fn; \ 507 jmp ret_from_intr; \ 508ENDPROC(name) 509 510 511#ifdef CONFIG_TRACING 512# define TRACE_BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name) 513#else 514# define TRACE_BUILD_INTERRUPT(name, nr) 515#endif 516 517#define BUILD_INTERRUPT(name, nr) \ 518 BUILD_INTERRUPT3(name, nr, smp_##name); \ 519 TRACE_BUILD_INTERRUPT(name, nr) 520 521/* The include is where all of the SMP etc. interrupts come from */ 522#include <asm/entry_arch.h> 523 524ENTRY(coprocessor_error) 525 ASM_CLAC 526 pushl $0 527 pushl $do_coprocessor_error 528 jmp error_code 529END(coprocessor_error) 530 531ENTRY(simd_coprocessor_error) 532 ASM_CLAC 533 pushl $0 534#ifdef CONFIG_X86_INVD_BUG 535 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 536 ALTERNATIVE "pushl $do_general_protection", \ 537 "pushl $do_simd_coprocessor_error", \ 538 X86_FEATURE_XMM 539#else 540 pushl $do_simd_coprocessor_error 541#endif 542 jmp error_code 543END(simd_coprocessor_error) 544 545ENTRY(device_not_available) 546 ASM_CLAC 547 pushl $-1 # mark this as an int 548 pushl $do_device_not_available 549 jmp error_code 550END(device_not_available) 551 552#ifdef CONFIG_PARAVIRT 553ENTRY(native_iret) 554 iret 555 _ASM_EXTABLE(native_iret, iret_exc) 556END(native_iret) 557#endif 558 559ENTRY(overflow) 560 ASM_CLAC 561 pushl $0 562 pushl $do_overflow 563 jmp error_code 564END(overflow) 565 566ENTRY(bounds) 567 ASM_CLAC 568 pushl $0 569 pushl $do_bounds 570 jmp error_code 571END(bounds) 572 573ENTRY(invalid_op) 574 ASM_CLAC 575 pushl $0 576 pushl $do_invalid_op 577 jmp error_code 578END(invalid_op) 579 580ENTRY(coprocessor_segment_overrun) 581 ASM_CLAC 582 pushl $0 583 pushl $do_coprocessor_segment_overrun 584 jmp error_code 585END(coprocessor_segment_overrun) 586 587ENTRY(invalid_TSS) 588 ASM_CLAC 589 pushl $do_invalid_TSS 590 jmp error_code 591END(invalid_TSS) 592 593ENTRY(segment_not_present) 594 ASM_CLAC 595 pushl $do_segment_not_present 596 jmp error_code 597END(segment_not_present) 598 599ENTRY(stack_segment) 600 ASM_CLAC 601 pushl $do_stack_segment 602 jmp error_code 603END(stack_segment) 604 605ENTRY(alignment_check) 606 ASM_CLAC 607 pushl $do_alignment_check 608 jmp error_code 609END(alignment_check) 610 611ENTRY(divide_error) 612 ASM_CLAC 613 pushl $0 # no error code 614 pushl $do_divide_error 615 jmp error_code 616END(divide_error) 617 618#ifdef CONFIG_X86_MCE 619ENTRY(machine_check) 620 ASM_CLAC 621 pushl $0 622 pushl machine_check_vector 623 jmp error_code 624END(machine_check) 625#endif 626 627ENTRY(spurious_interrupt_bug) 628 ASM_CLAC 629 pushl $0 630 pushl $do_spurious_interrupt_bug 631 jmp error_code 632END(spurious_interrupt_bug) 633 634#ifdef CONFIG_XEN 635/* 636 * Xen doesn't set %esp to be precisely what the normal SYSENTER 637 * entry point expects, so fix it up before using the normal path. 638 */ 639ENTRY(xen_sysenter_target) 640 addl $5*4, %esp /* remove xen-provided frame */ 641 jmp sysenter_past_esp 642 643ENTRY(xen_hypervisor_callback) 644 pushl $-1 /* orig_ax = -1 => not a system call */ 645 SAVE_ALL 646 TRACE_IRQS_OFF 647 648 /* 649 * Check to see if we got the event in the critical 650 * region in xen_iret_direct, after we've reenabled 651 * events and checked for pending events. This simulates 652 * iret instruction's behaviour where it delivers a 653 * pending interrupt when enabling interrupts: 654 */ 655 movl PT_EIP(%esp), %eax 656 cmpl $xen_iret_start_crit, %eax 657 jb 1f 658 cmpl $xen_iret_end_crit, %eax 659 jae 1f 660 661 jmp xen_iret_crit_fixup 662 663ENTRY(xen_do_upcall) 6641: mov %esp, %eax 665 call xen_evtchn_do_upcall 666#ifndef CONFIG_PREEMPT 667 call xen_maybe_preempt_hcall 668#endif 669 jmp ret_from_intr 670ENDPROC(xen_hypervisor_callback) 671 672/* 673 * Hypervisor uses this for application faults while it executes. 674 * We get here for two reasons: 675 * 1. Fault while reloading DS, ES, FS or GS 676 * 2. Fault while executing IRET 677 * Category 1 we fix up by reattempting the load, and zeroing the segment 678 * register if the load fails. 679 * Category 2 we fix up by jumping to do_iret_error. We cannot use the 680 * normal Linux return path in this case because if we use the IRET hypercall 681 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 682 * We distinguish between categories by maintaining a status value in EAX. 683 */ 684ENTRY(xen_failsafe_callback) 685 pushl %eax 686 movl $1, %eax 6871: mov 4(%esp), %ds 6882: mov 8(%esp), %es 6893: mov 12(%esp), %fs 6904: mov 16(%esp), %gs 691 /* EAX == 0 => Category 1 (Bad segment) 692 EAX != 0 => Category 2 (Bad IRET) */ 693 testl %eax, %eax 694 popl %eax 695 lea 16(%esp), %esp 696 jz 5f 697 jmp iret_exc 6985: pushl $-1 /* orig_ax = -1 => not a system call */ 699 SAVE_ALL 700 jmp ret_from_exception 701 702.section .fixup, "ax" 7036: xorl %eax, %eax 704 movl %eax, 4(%esp) 705 jmp 1b 7067: xorl %eax, %eax 707 movl %eax, 8(%esp) 708 jmp 2b 7098: xorl %eax, %eax 710 movl %eax, 12(%esp) 711 jmp 3b 7129: xorl %eax, %eax 713 movl %eax, 16(%esp) 714 jmp 4b 715.previous 716 _ASM_EXTABLE(1b, 6b) 717 _ASM_EXTABLE(2b, 7b) 718 _ASM_EXTABLE(3b, 8b) 719 _ASM_EXTABLE(4b, 9b) 720ENDPROC(xen_failsafe_callback) 721 722BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 723 xen_evtchn_do_upcall) 724 725#endif /* CONFIG_XEN */ 726 727#if IS_ENABLED(CONFIG_HYPERV) 728 729BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 730 hyperv_vector_handler) 731 732#endif /* CONFIG_HYPERV */ 733 734#ifdef CONFIG_FUNCTION_TRACER 735#ifdef CONFIG_DYNAMIC_FTRACE 736 737ENTRY(mcount) 738 ret 739END(mcount) 740 741ENTRY(ftrace_caller) 742 pushl %eax 743 pushl %ecx 744 pushl %edx 745 pushl $0 /* Pass NULL as regs pointer */ 746 movl 4*4(%esp), %eax 747 movl 0x4(%ebp), %edx 748 movl function_trace_op, %ecx 749 subl $MCOUNT_INSN_SIZE, %eax 750 751.globl ftrace_call 752ftrace_call: 753 call ftrace_stub 754 755 addl $4, %esp /* skip NULL pointer */ 756 popl %edx 757 popl %ecx 758 popl %eax 759ftrace_ret: 760#ifdef CONFIG_FUNCTION_GRAPH_TRACER 761.globl ftrace_graph_call 762ftrace_graph_call: 763 jmp ftrace_stub 764#endif 765 766.globl ftrace_stub 767ftrace_stub: 768 ret 769END(ftrace_caller) 770 771ENTRY(ftrace_regs_caller) 772 pushf /* push flags before compare (in cs location) */ 773 774 /* 775 * i386 does not save SS and ESP when coming from kernel. 776 * Instead, to get sp, ®s->sp is used (see ptrace.h). 777 * Unfortunately, that means eflags must be at the same location 778 * as the current return ip is. We move the return ip into the 779 * ip location, and move flags into the return ip location. 780 */ 781 pushl 4(%esp) /* save return ip into ip slot */ 782 783 pushl $0 /* Load 0 into orig_ax */ 784 pushl %gs 785 pushl %fs 786 pushl %es 787 pushl %ds 788 pushl %eax 789 pushl %ebp 790 pushl %edi 791 pushl %esi 792 pushl %edx 793 pushl %ecx 794 pushl %ebx 795 796 movl 13*4(%esp), %eax /* Get the saved flags */ 797 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ 798 /* clobbering return ip */ 799 movl $__KERNEL_CS, 13*4(%esp) 800 801 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 802 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 803 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 804 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 805 pushl %esp /* Save pt_regs as 4th parameter */ 806 807GLOBAL(ftrace_regs_call) 808 call ftrace_stub 809 810 addl $4, %esp /* Skip pt_regs */ 811 movl 14*4(%esp), %eax /* Move flags back into cs */ 812 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ 813 movl 12*4(%esp), %eax /* Get return ip from regs->ip */ 814 movl %eax, 14*4(%esp) /* Put return ip back for ret */ 815 816 popl %ebx 817 popl %ecx 818 popl %edx 819 popl %esi 820 popl %edi 821 popl %ebp 822 popl %eax 823 popl %ds 824 popl %es 825 popl %fs 826 popl %gs 827 addl $8, %esp /* Skip orig_ax and ip */ 828 popf /* Pop flags at end (no addl to corrupt flags) */ 829 jmp ftrace_ret 830 831 popf 832 jmp ftrace_stub 833#else /* ! CONFIG_DYNAMIC_FTRACE */ 834 835ENTRY(mcount) 836 cmpl $__PAGE_OFFSET, %esp 837 jb ftrace_stub /* Paging not enabled yet? */ 838 839 cmpl $ftrace_stub, ftrace_trace_function 840 jnz trace 841#ifdef CONFIG_FUNCTION_GRAPH_TRACER 842 cmpl $ftrace_stub, ftrace_graph_return 843 jnz ftrace_graph_caller 844 845 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 846 jnz ftrace_graph_caller 847#endif 848.globl ftrace_stub 849ftrace_stub: 850 ret 851 852 /* taken from glibc */ 853trace: 854 pushl %eax 855 pushl %ecx 856 pushl %edx 857 movl 0xc(%esp), %eax 858 movl 0x4(%ebp), %edx 859 subl $MCOUNT_INSN_SIZE, %eax 860 861 call *ftrace_trace_function 862 863 popl %edx 864 popl %ecx 865 popl %eax 866 jmp ftrace_stub 867END(mcount) 868#endif /* CONFIG_DYNAMIC_FTRACE */ 869#endif /* CONFIG_FUNCTION_TRACER */ 870 871#ifdef CONFIG_FUNCTION_GRAPH_TRACER 872ENTRY(ftrace_graph_caller) 873 pushl %eax 874 pushl %ecx 875 pushl %edx 876 movl 0xc(%esp), %eax 877 lea 0x4(%ebp), %edx 878 movl (%ebp), %ecx 879 subl $MCOUNT_INSN_SIZE, %eax 880 call prepare_ftrace_return 881 popl %edx 882 popl %ecx 883 popl %eax 884 ret 885END(ftrace_graph_caller) 886 887.globl return_to_handler 888return_to_handler: 889 pushl %eax 890 pushl %edx 891 movl %ebp, %eax 892 call ftrace_return_to_handler 893 movl %eax, %ecx 894 popl %edx 895 popl %eax 896 jmp *%ecx 897#endif 898 899#ifdef CONFIG_TRACING 900ENTRY(trace_page_fault) 901 ASM_CLAC 902 pushl $trace_do_page_fault 903 jmp error_code 904END(trace_page_fault) 905#endif 906 907ENTRY(page_fault) 908 ASM_CLAC 909 pushl $do_page_fault 910 ALIGN 911error_code: 912 /* the function address is in %gs's slot on the stack */ 913 pushl %fs 914 pushl %es 915 pushl %ds 916 pushl %eax 917 pushl %ebp 918 pushl %edi 919 pushl %esi 920 pushl %edx 921 pushl %ecx 922 pushl %ebx 923 cld 924 movl $(__KERNEL_PERCPU), %ecx 925 movl %ecx, %fs 926 UNWIND_ESPFIX_STACK 927 GS_TO_REG %ecx 928 movl PT_GS(%esp), %edi # get the function address 929 movl PT_ORIG_EAX(%esp), %edx # get the error code 930 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 931 REG_TO_PTGS %ecx 932 SET_KERNEL_GS %ecx 933 movl $(__USER_DS), %ecx 934 movl %ecx, %ds 935 movl %ecx, %es 936 TRACE_IRQS_OFF 937 movl %esp, %eax # pt_regs pointer 938 call *%edi 939 jmp ret_from_exception 940END(page_fault) 941 942/* 943 * Debug traps and NMI can happen at the one SYSENTER instruction 944 * that sets up the real kernel stack. Check here, since we can't 945 * allow the wrong stack to be used. 946 * 947 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 948 * already pushed 3 words if it hits on the sysenter instruction: 949 * eflags, cs and eip. 950 * 951 * We just load the right stack, and push the three (known) values 952 * by hand onto the new stack - while updating the return eip past 953 * the instruction that would have done it for sysenter. 954 */ 955.macro FIX_STACK offset ok label 956 cmpw $__KERNEL_CS, 4(%esp) 957 jne \ok 958\label: 959 movl TSS_sysenter_sp0 + \offset(%esp), %esp 960 pushfl 961 pushl $__KERNEL_CS 962 pushl $sysenter_past_esp 963.endm 964 965ENTRY(debug) 966 ASM_CLAC 967 cmpl $entry_SYSENTER_32, (%esp) 968 jne debug_stack_correct 969 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 970debug_stack_correct: 971 pushl $-1 # mark this as an int 972 SAVE_ALL 973 TRACE_IRQS_OFF 974 xorl %edx, %edx # error code 0 975 movl %esp, %eax # pt_regs pointer 976 call do_debug 977 jmp ret_from_exception 978END(debug) 979 980/* 981 * NMI is doubly nasty. It can happen _while_ we're handling 982 * a debug fault, and the debug fault hasn't yet been able to 983 * clear up the stack. So we first check whether we got an 984 * NMI on the sysenter entry path, but after that we need to 985 * check whether we got an NMI on the debug path where the debug 986 * fault happened on the sysenter path. 987 */ 988ENTRY(nmi) 989 ASM_CLAC 990#ifdef CONFIG_X86_ESPFIX32 991 pushl %eax 992 movl %ss, %eax 993 cmpw $__ESPFIX_SS, %ax 994 popl %eax 995 je nmi_espfix_stack 996#endif 997 cmpl $entry_SYSENTER_32, (%esp) 998 je nmi_stack_fixup 999 pushl %eax 1000 movl %esp, %eax 1001 /* 1002 * Do not access memory above the end of our stack page, 1003 * it might not exist. 1004 */ 1005 andl $(THREAD_SIZE-1), %eax 1006 cmpl $(THREAD_SIZE-20), %eax 1007 popl %eax 1008 jae nmi_stack_correct 1009 cmpl $entry_SYSENTER_32, 12(%esp) 1010 je nmi_debug_stack_check 1011nmi_stack_correct: 1012 pushl %eax 1013 SAVE_ALL 1014 xorl %edx, %edx # zero error code 1015 movl %esp, %eax # pt_regs pointer 1016 call do_nmi 1017 jmp restore_all_notrace 1018 1019nmi_stack_fixup: 1020 FIX_STACK 12, nmi_stack_correct, 1 1021 jmp nmi_stack_correct 1022 1023nmi_debug_stack_check: 1024 cmpw $__KERNEL_CS, 16(%esp) 1025 jne nmi_stack_correct 1026 cmpl $debug, (%esp) 1027 jb nmi_stack_correct 1028 cmpl $debug_esp_fix_insn, (%esp) 1029 ja nmi_stack_correct 1030 FIX_STACK 24, nmi_stack_correct, 1 1031 jmp nmi_stack_correct 1032 1033#ifdef CONFIG_X86_ESPFIX32 1034nmi_espfix_stack: 1035 /* 1036 * create the pointer to lss back 1037 */ 1038 pushl %ss 1039 pushl %esp 1040 addl $4, (%esp) 1041 /* copy the iret frame of 12 bytes */ 1042 .rept 3 1043 pushl 16(%esp) 1044 .endr 1045 pushl %eax 1046 SAVE_ALL 1047 FIXUP_ESPFIX_STACK # %eax == %esp 1048 xorl %edx, %edx # zero error code 1049 call do_nmi 1050 RESTORE_REGS 1051 lss 12+4(%esp), %esp # back to espfix stack 1052 jmp irq_return 1053#endif 1054END(nmi) 1055 1056ENTRY(int3) 1057 ASM_CLAC 1058 pushl $-1 # mark this as an int 1059 SAVE_ALL 1060 TRACE_IRQS_OFF 1061 xorl %edx, %edx # zero error code 1062 movl %esp, %eax # pt_regs pointer 1063 call do_int3 1064 jmp ret_from_exception 1065END(int3) 1066 1067ENTRY(general_protection) 1068 pushl $do_general_protection 1069 jmp error_code 1070END(general_protection) 1071 1072#ifdef CONFIG_KVM_GUEST 1073ENTRY(async_page_fault) 1074 ASM_CLAC 1075 pushl $do_async_page_fault 1076 jmp error_code 1077END(async_page_fault) 1078#endif 1079