xref: /linux/arch/x86/xen/xen-asm.S (revision 1623bc27a85a93e82194c8d077eccc464efa67db) 
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Asm versions of Xen pv-ops, suitable for direct use.
4 *
5 * We only bother with direct forms (ie, vcpu in percpu data) of the
6 * operations here; the indirect forms are better handled in C.
7 */
8
9#include <asm/errno.h>
10#include <asm/asm-offsets.h>
11#include <asm/percpu.h>
12#include <asm/processor-flags.h>
13#include <asm/segment.h>
14#include <asm/thread_info.h>
15#include <asm/asm.h>
16#include <asm/frame.h>
17#include <asm/unwind_hints.h>
18
19#include <xen/interface/xen.h>
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <linux/objtool.h>
24#include <../entry/calling.h>
25
26.pushsection .noinstr.text, "ax"
27/*
28 * PV hypercall interface to the hypervisor.
29 *
30 * Called via inline asm(), so better preserve %rcx and %r11.
31 *
32 * Input:
33 *	%eax: hypercall number
34 *	%rdi, %rsi, %rdx, %r10, %r8: args 1..5 for the hypercall
35 * Output: %rax
36 */
37SYM_FUNC_START(xen_hypercall_pv)
38	ANNOTATE_NOENDBR
39	push %rcx
40	push %r11
41	UNWIND_HINT_SAVE
42	syscall
43	UNWIND_HINT_RESTORE
44	pop %r11
45	pop %rcx
46	RET
47SYM_FUNC_END(xen_hypercall_pv)
48
49/*
50 * Disabling events is simply a matter of making the event mask
51 * non-zero.
52 */
53SYM_FUNC_START(xen_irq_disable_direct)
54	movb $1, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
55	RET
56SYM_FUNC_END(xen_irq_disable_direct)
57
58/*
59 * Force an event check by making a hypercall, but preserve regs
60 * before making the call.
61 */
62SYM_FUNC_START(check_events)
63	FRAME_BEGIN
64	push %rax
65	push %rcx
66	push %rdx
67	push %rsi
68	push %rdi
69	push %r8
70	push %r9
71	push %r10
72	push %r11
73	call xen_force_evtchn_callback
74	pop %r11
75	pop %r10
76	pop %r9
77	pop %r8
78	pop %rdi
79	pop %rsi
80	pop %rdx
81	pop %rcx
82	pop %rax
83	FRAME_END
84	RET
85SYM_FUNC_END(check_events)
86
87/*
88 * Enable events.  This clears the event mask and tests the pending
89 * event status with one and operation.  If there are pending events,
90 * then enter the hypervisor to get them handled.
91 */
92SYM_FUNC_START(xen_irq_enable_direct)
93	FRAME_BEGIN
94	/* Unmask events */
95	movb $0, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
96
97	/*
98	 * Preempt here doesn't matter because that will deal with any
99	 * pending interrupts.  The pending check may end up being run
100	 * on the wrong CPU, but that doesn't hurt.
101	 */
102
103	/* Test for pending */
104	testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_pending)
105	jz 1f
106
107	call check_events
1081:
109	FRAME_END
110	RET
111SYM_FUNC_END(xen_irq_enable_direct)
112
113/*
114 * (xen_)save_fl is used to get the current interrupt enable status.
115 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
116 * may be set in the return value.  We take advantage of this by
117 * making sure that X86_EFLAGS_IF has the right value (and other bits
118 * in that byte are 0), but other bits in the return value are
119 * undefined.  We need to toggle the state of the bit, because Xen and
120 * x86 use opposite senses (mask vs enable).
121 */
122SYM_FUNC_START(xen_save_fl_direct)
123	testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
124	setz %ah
125	addb %ah, %ah
126	RET
127SYM_FUNC_END(xen_save_fl_direct)
128
129SYM_FUNC_START(xen_read_cr2)
130	FRAME_BEGIN
131	_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
132	_ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
133	FRAME_END
134	RET
135SYM_FUNC_END(xen_read_cr2);
136
137SYM_FUNC_START(xen_read_cr2_direct)
138	FRAME_BEGIN
139	_ASM_MOV PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_arch_cr2), %_ASM_AX
140	FRAME_END
141	RET
142SYM_FUNC_END(xen_read_cr2_direct);
143.popsection
144
145.macro xen_pv_trap name
146SYM_CODE_START(xen_\name)
147	UNWIND_HINT_ENTRY
148	ENDBR
149	pop %rcx
150	pop %r11
151	jmp  \name
152SYM_CODE_END(xen_\name)
153_ASM_NOKPROBE(xen_\name)
154.endm
155
156xen_pv_trap asm_exc_divide_error
157xen_pv_trap asm_xenpv_exc_debug
158xen_pv_trap asm_exc_int3
159xen_pv_trap asm_xenpv_exc_nmi
160xen_pv_trap asm_exc_overflow
161xen_pv_trap asm_exc_bounds
162xen_pv_trap asm_exc_invalid_op
163xen_pv_trap asm_exc_device_not_available
164xen_pv_trap asm_xenpv_exc_double_fault
165xen_pv_trap asm_exc_coproc_segment_overrun
166xen_pv_trap asm_exc_invalid_tss
167xen_pv_trap asm_exc_segment_not_present
168xen_pv_trap asm_exc_stack_segment
169xen_pv_trap asm_exc_general_protection
170xen_pv_trap asm_exc_page_fault
171xen_pv_trap asm_exc_spurious_interrupt_bug
172xen_pv_trap asm_exc_coprocessor_error
173xen_pv_trap asm_exc_alignment_check
174#ifdef CONFIG_X86_CET
175xen_pv_trap asm_exc_control_protection
176#endif
177#ifdef CONFIG_X86_MCE
178xen_pv_trap asm_xenpv_exc_machine_check
179#endif /* CONFIG_X86_MCE */
180xen_pv_trap asm_exc_simd_coprocessor_error
181#ifdef CONFIG_IA32_EMULATION
182xen_pv_trap asm_int80_emulation
183#endif
184xen_pv_trap asm_exc_xen_unknown_trap
185xen_pv_trap asm_exc_xen_hypervisor_callback
186
187	__INIT
188SYM_CODE_START(xen_early_idt_handler_array)
189	i = 0
190	.rept NUM_EXCEPTION_VECTORS
191	UNWIND_HINT_UNDEFINED
192	ENDBR
193	pop %rcx
194	pop %r11
195	jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
196	i = i + 1
197	.fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
198	.endr
199SYM_CODE_END(xen_early_idt_handler_array)
200	__FINIT
201
202/*
203 * Xen64 iret frame:
204 *
205 *	ss
206 *	rsp
207 *	rflags
208 *	cs
209 *	rip		<-- standard iret frame
210 *
211 *	flags		<-- xen_iret must push from here on
212 *
213 *	rcx
214 *	r11
215 * rsp->rax
216 */
217.macro xen_hypercall_iret
218	pushq $0	/* Flags */
219	push %rcx
220	push %r11
221	push %rax
222	mov  $__HYPERVISOR_iret, %eax
223	syscall		/* Do the IRET. */
224#ifdef CONFIG_MITIGATION_SLS
225	int3
226#endif
227.endm
228
229SYM_CODE_START(xen_iret)
230	UNWIND_HINT_UNDEFINED
231	ANNOTATE_NOENDBR
232	xen_hypercall_iret
233SYM_CODE_END(xen_iret)
234
235/*
236 * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
237 * also the kernel stack.  Reusing swapgs_restore_regs_and_return_to_usermode()
238 * in XEN pv would cause %rsp to move up to the top of the kernel stack and
239 * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
240 * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
241 * frame at the same address is useless.
242 */
243SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
244	UNWIND_HINT_REGS
245	POP_REGS
246
247	/* stackleak_erase() can work safely on the kernel stack. */
248	STACKLEAK_ERASE_NOCLOBBER
249
250	addq	$8, %rsp	/* skip regs->orig_ax */
251	jmp xen_iret
252SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
253
254/*
255 * Xen handles syscall callbacks much like ordinary exceptions, which
256 * means we have:
257 * - kernel gs
258 * - kernel rsp
259 * - an iret-like stack frame on the stack (including rcx and r11):
260 *	ss
261 *	rsp
262 *	rflags
263 *	cs
264 *	rip
265 *	r11
266 * rsp->rcx
267 */
268
269/* Normal 64-bit system call target */
270SYM_CODE_START(xen_entry_SYSCALL_64)
271	UNWIND_HINT_ENTRY
272	ENDBR
273	popq %rcx
274	popq %r11
275
276	/*
277	 * Neither Xen nor the kernel really knows what the old SS and
278	 * CS were.  The kernel expects __USER_DS and __USER_CS, so
279	 * report those values even though Xen will guess its own values.
280	 */
281	movq $__USER_DS, 4*8(%rsp)
282	movq $__USER_CS, 1*8(%rsp)
283
284	jmp entry_SYSCALL_64_after_hwframe
285SYM_CODE_END(xen_entry_SYSCALL_64)
286
287#ifdef CONFIG_IA32_EMULATION
288
289/* 32-bit compat syscall target */
290SYM_CODE_START(xen_entry_SYSCALL_compat)
291	UNWIND_HINT_ENTRY
292	ENDBR
293	popq %rcx
294	popq %r11
295
296	/*
297	 * Neither Xen nor the kernel really knows what the old SS and
298	 * CS were.  The kernel expects __USER_DS and __USER32_CS, so
299	 * report those values even though Xen will guess its own values.
300	 */
301	movq $__USER_DS, 4*8(%rsp)
302	movq $__USER32_CS, 1*8(%rsp)
303
304	jmp entry_SYSCALL_compat_after_hwframe
305SYM_CODE_END(xen_entry_SYSCALL_compat)
306
307/* 32-bit compat sysenter target */
308SYM_CODE_START(xen_entry_SYSENTER_compat)
309	UNWIND_HINT_ENTRY
310	ENDBR
311	/*
312	 * NB: Xen is polite and clears TF from EFLAGS for us.  This means
313	 * that we don't need to guard against single step exceptions here.
314	 */
315	popq %rcx
316	popq %r11
317
318	/*
319	 * Neither Xen nor the kernel really knows what the old SS and
320	 * CS were.  The kernel expects __USER_DS and __USER32_CS, so
321	 * report those values even though Xen will guess its own values.
322	 */
323	movq $__USER_DS, 4*8(%rsp)
324	movq $__USER32_CS, 1*8(%rsp)
325
326	jmp entry_SYSENTER_compat_after_hwframe
327SYM_CODE_END(xen_entry_SYSENTER_compat)
328
329#else /* !CONFIG_IA32_EMULATION */
330
331SYM_CODE_START(xen_entry_SYSCALL_compat)
332SYM_CODE_START(xen_entry_SYSENTER_compat)
333	UNWIND_HINT_ENTRY
334	ENDBR
335	lea 16(%rsp), %rsp	/* strip %rcx, %r11 */
336	mov $-ENOSYS, %rax
337	xen_hypercall_iret
338SYM_CODE_END(xen_entry_SYSENTER_compat)
339SYM_CODE_END(xen_entry_SYSCALL_compat)
340
341#endif	/* CONFIG_IA32_EMULATION */
342