1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Asm versions of Xen pv-ops, suitable for direct use. 4 * 5 * We only bother with direct forms (ie, vcpu in percpu data) of the 6 * operations here; the indirect forms are better handled in C. 7 */ 8 9#include <asm/errno.h> 10#include <asm/asm-offsets.h> 11#include <asm/percpu.h> 12#include <asm/processor-flags.h> 13#include <asm/segment.h> 14#include <asm/thread_info.h> 15#include <asm/asm.h> 16#include <asm/frame.h> 17#include <asm/unwind_hints.h> 18 19#include <xen/interface/xen.h> 20 21#include <linux/init.h> 22#include <linux/linkage.h> 23 24/* 25 * Enable events. This clears the event mask and tests the pending 26 * event status with one and operation. If there are pending events, 27 * then enter the hypervisor to get them handled. 28 */ 29SYM_FUNC_START(xen_irq_enable_direct) 30 FRAME_BEGIN 31 /* Unmask events */ 32 movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 33 34 /* 35 * Preempt here doesn't matter because that will deal with any 36 * pending interrupts. The pending check may end up being run 37 * on the wrong CPU, but that doesn't hurt. 38 */ 39 40 /* Test for pending */ 41 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending 42 jz 1f 43 44 call check_events 451: 46 FRAME_END 47 ret 48SYM_FUNC_END(xen_irq_enable_direct) 49 50 51/* 52 * Disabling events is simply a matter of making the event mask 53 * non-zero. 54 */ 55SYM_FUNC_START(xen_irq_disable_direct) 56 movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 57 ret 58SYM_FUNC_END(xen_irq_disable_direct) 59 60/* 61 * (xen_)save_fl is used to get the current interrupt enable status. 62 * Callers expect the status to be in X86_EFLAGS_IF, and other bits 63 * may be set in the return value. We take advantage of this by 64 * making sure that X86_EFLAGS_IF has the right value (and other bits 65 * in that byte are 0), but other bits in the return value are 66 * undefined. We need to toggle the state of the bit, because Xen and 67 * x86 use opposite senses (mask vs enable). 68 */ 69SYM_FUNC_START(xen_save_fl_direct) 70 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 71 setz %ah 72 addb %ah, %ah 73 ret 74SYM_FUNC_END(xen_save_fl_direct) 75 76/* 77 * Force an event check by making a hypercall, but preserve regs 78 * before making the call. 79 */ 80SYM_FUNC_START(check_events) 81 FRAME_BEGIN 82 push %rax 83 push %rcx 84 push %rdx 85 push %rsi 86 push %rdi 87 push %r8 88 push %r9 89 push %r10 90 push %r11 91 call xen_force_evtchn_callback 92 pop %r11 93 pop %r10 94 pop %r9 95 pop %r8 96 pop %rdi 97 pop %rsi 98 pop %rdx 99 pop %rcx 100 pop %rax 101 FRAME_END 102 ret 103SYM_FUNC_END(check_events) 104 105SYM_FUNC_START(xen_read_cr2) 106 FRAME_BEGIN 107 _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX 108 _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX 109 FRAME_END 110 ret 111SYM_FUNC_END(xen_read_cr2); 112 113SYM_FUNC_START(xen_read_cr2_direct) 114 FRAME_BEGIN 115 _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX 116 FRAME_END 117 ret 118SYM_FUNC_END(xen_read_cr2_direct); 119 120.macro xen_pv_trap name 121SYM_CODE_START(xen_\name) 122 UNWIND_HINT_EMPTY 123 pop %rcx 124 pop %r11 125 jmp \name 126SYM_CODE_END(xen_\name) 127_ASM_NOKPROBE(xen_\name) 128.endm 129 130xen_pv_trap asm_exc_divide_error 131xen_pv_trap asm_xenpv_exc_debug 132xen_pv_trap asm_exc_int3 133xen_pv_trap asm_xenpv_exc_nmi 134xen_pv_trap asm_exc_overflow 135xen_pv_trap asm_exc_bounds 136xen_pv_trap asm_exc_invalid_op 137xen_pv_trap asm_exc_device_not_available 138xen_pv_trap asm_xenpv_exc_double_fault 139xen_pv_trap asm_exc_coproc_segment_overrun 140xen_pv_trap asm_exc_invalid_tss 141xen_pv_trap asm_exc_segment_not_present 142xen_pv_trap asm_exc_stack_segment 143xen_pv_trap asm_exc_general_protection 144xen_pv_trap asm_exc_page_fault 145xen_pv_trap asm_exc_spurious_interrupt_bug 146xen_pv_trap asm_exc_coprocessor_error 147xen_pv_trap asm_exc_alignment_check 148#ifdef CONFIG_X86_MCE 149xen_pv_trap asm_xenpv_exc_machine_check 150#endif /* CONFIG_X86_MCE */ 151xen_pv_trap asm_exc_simd_coprocessor_error 152#ifdef CONFIG_IA32_EMULATION 153xen_pv_trap entry_INT80_compat 154#endif 155xen_pv_trap asm_exc_xen_unknown_trap 156xen_pv_trap asm_exc_xen_hypervisor_callback 157 158 __INIT 159SYM_CODE_START(xen_early_idt_handler_array) 160 i = 0 161 .rept NUM_EXCEPTION_VECTORS 162 UNWIND_HINT_EMPTY 163 pop %rcx 164 pop %r11 165 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE 166 i = i + 1 167 .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc 168 .endr 169SYM_CODE_END(xen_early_idt_handler_array) 170 __FINIT 171 172hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 173/* 174 * Xen64 iret frame: 175 * 176 * ss 177 * rsp 178 * rflags 179 * cs 180 * rip <-- standard iret frame 181 * 182 * flags 183 * 184 * rcx } 185 * r11 }<-- pushed by hypercall page 186 * rsp->rax } 187 */ 188SYM_CODE_START(xen_iret) 189 UNWIND_HINT_EMPTY 190 pushq $0 191 jmp hypercall_iret 192SYM_CODE_END(xen_iret) 193 194/* 195 * Xen handles syscall callbacks much like ordinary exceptions, which 196 * means we have: 197 * - kernel gs 198 * - kernel rsp 199 * - an iret-like stack frame on the stack (including rcx and r11): 200 * ss 201 * rsp 202 * rflags 203 * cs 204 * rip 205 * r11 206 * rsp->rcx 207 */ 208 209/* Normal 64-bit system call target */ 210SYM_CODE_START(xen_syscall_target) 211 UNWIND_HINT_EMPTY 212 popq %rcx 213 popq %r11 214 215 /* 216 * Neither Xen nor the kernel really knows what the old SS and 217 * CS were. The kernel expects __USER_DS and __USER_CS, so 218 * report those values even though Xen will guess its own values. 219 */ 220 movq $__USER_DS, 4*8(%rsp) 221 movq $__USER_CS, 1*8(%rsp) 222 223 jmp entry_SYSCALL_64_after_hwframe 224SYM_CODE_END(xen_syscall_target) 225 226#ifdef CONFIG_IA32_EMULATION 227 228/* 32-bit compat syscall target */ 229SYM_CODE_START(xen_syscall32_target) 230 UNWIND_HINT_EMPTY 231 popq %rcx 232 popq %r11 233 234 /* 235 * Neither Xen nor the kernel really knows what the old SS and 236 * CS were. The kernel expects __USER32_DS and __USER32_CS, so 237 * report those values even though Xen will guess its own values. 238 */ 239 movq $__USER32_DS, 4*8(%rsp) 240 movq $__USER32_CS, 1*8(%rsp) 241 242 jmp entry_SYSCALL_compat_after_hwframe 243SYM_CODE_END(xen_syscall32_target) 244 245/* 32-bit compat sysenter target */ 246SYM_CODE_START(xen_sysenter_target) 247 UNWIND_HINT_EMPTY 248 /* 249 * NB: Xen is polite and clears TF from EFLAGS for us. This means 250 * that we don't need to guard against single step exceptions here. 251 */ 252 popq %rcx 253 popq %r11 254 255 /* 256 * Neither Xen nor the kernel really knows what the old SS and 257 * CS were. The kernel expects __USER32_DS and __USER32_CS, so 258 * report those values even though Xen will guess its own values. 259 */ 260 movq $__USER32_DS, 4*8(%rsp) 261 movq $__USER32_CS, 1*8(%rsp) 262 263 jmp entry_SYSENTER_compat_after_hwframe 264SYM_CODE_END(xen_sysenter_target) 265 266#else /* !CONFIG_IA32_EMULATION */ 267 268SYM_CODE_START(xen_syscall32_target) 269SYM_CODE_START(xen_sysenter_target) 270 UNWIND_HINT_EMPTY 271 lea 16(%rsp), %rsp /* strip %rcx, %r11 */ 272 mov $-ENOSYS, %rax 273 pushq $0 274 jmp hypercall_iret 275SYM_CODE_END(xen_sysenter_target) 276SYM_CODE_END(xen_syscall32_target) 277 278#endif /* CONFIG_IA32_EMULATION */ 279