xref: /freebsd/sys/amd64/vmm/intel/vmx_support.S (revision 78b9f0095b4af3aca6c931b2c7b009ddb8a05125) 
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31
32#include <machine/asmacros.h>
33#include <machine/specialreg.h>
34
35#include "vmx_assym.h"
36
37#ifdef SMP
38#define	LK	lock ;
39#else
40#define	LK
41#endif
42
43/* Be friendly to DTrace FBT's prologue/epilogue pattern matching */
44#define VENTER  push %rbp ; mov %rsp,%rbp
45#define VLEAVE  pop %rbp
46
47/*
48 * Save the guest context.
49 */
50#define	VMX_GUEST_SAVE							\
51	movq	%rdi,VMXCTX_GUEST_RDI(%rsp);				\
52	movq	%rsi,VMXCTX_GUEST_RSI(%rsp);				\
53	movq	%rdx,VMXCTX_GUEST_RDX(%rsp);				\
54	movq	%rcx,VMXCTX_GUEST_RCX(%rsp);				\
55	movq	%r8,VMXCTX_GUEST_R8(%rsp);				\
56	movq	%r9,VMXCTX_GUEST_R9(%rsp);				\
57	movq	%rax,VMXCTX_GUEST_RAX(%rsp);				\
58	movq	%rbx,VMXCTX_GUEST_RBX(%rsp);				\
59	movq	%rbp,VMXCTX_GUEST_RBP(%rsp);				\
60	movq	%r10,VMXCTX_GUEST_R10(%rsp);				\
61	movq	%r11,VMXCTX_GUEST_R11(%rsp);				\
62	movq	%r12,VMXCTX_GUEST_R12(%rsp);				\
63	movq	%r13,VMXCTX_GUEST_R13(%rsp);				\
64	movq	%r14,VMXCTX_GUEST_R14(%rsp);				\
65	movq	%r15,VMXCTX_GUEST_R15(%rsp);				\
66	movq	%cr2,%rdi;						\
67	movq	%rdi,VMXCTX_GUEST_CR2(%rsp);				\
68	movq	%rsp,%rdi;
69
70/*
71 * Assumes that %rdi holds a pointer to the 'vmxctx'.
72 *
73 * On "return" all registers are updated to reflect guest state. The two
74 * exceptions are %rip and %rsp. These registers are atomically switched
75 * by hardware from the guest area of the vmcs.
76 *
77 * We modify %rsp to point to the 'vmxctx' so we can use it to restore
78 * host context in case of an error with 'vmlaunch' or 'vmresume'.
79 */
80#define	VMX_GUEST_RESTORE						\
81	movq	%rdi,%rsp;						\
82	movq	VMXCTX_GUEST_CR2(%rdi),%rsi;				\
83	movq	%rsi,%cr2;						\
84	movq	VMXCTX_GUEST_RSI(%rdi),%rsi;				\
85	movq	VMXCTX_GUEST_RDX(%rdi),%rdx;				\
86	movq	VMXCTX_GUEST_RCX(%rdi),%rcx;				\
87	movq	VMXCTX_GUEST_R8(%rdi),%r8;				\
88	movq	VMXCTX_GUEST_R9(%rdi),%r9;				\
89	movq	VMXCTX_GUEST_RAX(%rdi),%rax;				\
90	movq	VMXCTX_GUEST_RBX(%rdi),%rbx;				\
91	movq	VMXCTX_GUEST_RBP(%rdi),%rbp;				\
92	movq	VMXCTX_GUEST_R10(%rdi),%r10;				\
93	movq	VMXCTX_GUEST_R11(%rdi),%r11;				\
94	movq	VMXCTX_GUEST_R12(%rdi),%r12;				\
95	movq	VMXCTX_GUEST_R13(%rdi),%r13;				\
96	movq	VMXCTX_GUEST_R14(%rdi),%r14;				\
97	movq	VMXCTX_GUEST_R15(%rdi),%r15;				\
98	movq	VMXCTX_GUEST_RDI(%rdi),%rdi; /* restore rdi the last */
99
100/*
101 * Clobber the remaining registers with guest contents so they can't
102 * be misused.
103 */
104#define	VMX_GUEST_CLOBBER						\
105	xor	%rax, %rax;						\
106	xor	%rcx, %rcx;						\
107	xor	%rdx, %rdx;						\
108	xor	%rsi, %rsi;						\
109	xor	%r8, %r8;						\
110	xor	%r9, %r9;						\
111	xor	%r10, %r10;						\
112	xor	%r11, %r11;
113
114/*
115 * Save and restore the host context.
116 *
117 * Assumes that %rdi holds a pointer to the 'vmxctx'.
118 */
119#define	VMX_HOST_SAVE							\
120	movq    %r15, VMXCTX_HOST_R15(%rdi);				\
121	movq    %r14, VMXCTX_HOST_R14(%rdi);				\
122	movq    %r13, VMXCTX_HOST_R13(%rdi);				\
123	movq    %r12, VMXCTX_HOST_R12(%rdi);				\
124	movq    %rbp, VMXCTX_HOST_RBP(%rdi);				\
125	movq    %rsp, VMXCTX_HOST_RSP(%rdi);				\
126	movq    %rbx, VMXCTX_HOST_RBX(%rdi);				\
127
128#define	VMX_HOST_RESTORE						\
129	movq	VMXCTX_HOST_R15(%rdi), %r15;				\
130	movq	VMXCTX_HOST_R14(%rdi), %r14;				\
131	movq	VMXCTX_HOST_R13(%rdi), %r13;				\
132	movq	VMXCTX_HOST_R12(%rdi), %r12;				\
133	movq	VMXCTX_HOST_RBP(%rdi), %rbp;				\
134	movq	VMXCTX_HOST_RSP(%rdi), %rsp;				\
135	movq	VMXCTX_HOST_RBX(%rdi), %rbx;				\
136
137/*
138 * vmx_enter_guest(struct vmxctx *vmxctx, int launched)
139 * %rdi: pointer to the 'vmxctx'
140 * %rsi: pointer to the 'vmx'
141 * %edx: launch state of the VMCS
142 * Interrupts must be disabled on entry.
143 */
144ENTRY(vmx_enter_guest)
145	VENTER
146	/*
147	 * Save host state before doing anything else.
148	 */
149	VMX_HOST_SAVE
150
151	/*
152	 * Activate guest pmap on this cpu.
153	 */
154	movq	VMXCTX_PMAP(%rdi), %r11
155	movl	PCPU(CPUID), %eax
156	LK btsl	%eax, PM_ACTIVE(%r11)
157
158	/*
159	 * If 'vmx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen'
160	 * then we must invalidate all mappings associated with this EPTP.
161	 */
162	movq	PM_EPTGEN(%r11), %r10
163	cmpq	%r10, VMX_EPTGEN(%rsi, %rax, 8)
164	je	guest_restore
165
166	/* Refresh 'vmx->eptgen[curcpu]' */
167	movq	%r10, VMX_EPTGEN(%rsi, %rax, 8)
168
169	/* Setup the invept descriptor on the host stack */
170	mov	%rsp, %r11
171	movq	VMX_EPTP(%rsi), %rax
172	movq	%rax, -16(%r11)
173	movq	$0x0, -8(%r11)
174	mov	$0x1, %eax		/* Single context invalidate */
175	invept	-16(%r11), %rax
176	jbe	invept_error		/* Check invept instruction error */
177
178guest_restore:
179
180	/*
181	 * Flush L1D cache if requested.  Use IA32_FLUSH_CMD MSR if available,
182	 * otherwise load enough of the data from the zero_region to flush
183	 * existing L1D content.
184	 */
185#define	L1D_FLUSH_SIZE	(64 * 1024)
186	movl	%edx, %r8d
187	cmpb	$0, guest_l1d_flush(%rip)
188	je	after_l1d
189	movq	vmx_msr_flush_cmd(%rip), %rax
190	testq	%rax, %rax
191	jz	1f
192	movq	%rax, %rdx
193	shrq	$32, %rdx
194	movl	$MSR_IA32_FLUSH_CMD, %ecx
195	wrmsr
196	jmp	after_l1d
1971:	movq	$KERNBASE, %r9
198	movq	$-L1D_FLUSH_SIZE, %rcx
199	/*
200	 * pass 1: Preload TLB.
201	 * Kernel text is mapped using superpages.  TLB preload is
202	 * done for the benefit of older CPUs which split 2M page
203	 * into 4k TLB entries.
204	 */
2052:	movb	L1D_FLUSH_SIZE(%r9, %rcx), %al
206	addq	$PAGE_SIZE, %rcx
207	jne	2b
208	xorl	%eax, %eax
209	cpuid
210	movq	$-L1D_FLUSH_SIZE, %rcx
211	/* pass 2: Read each cache line */
2123:	movb	L1D_FLUSH_SIZE(%r9, %rcx), %al
213	addq	$64, %rcx
214	jne	3b
215	lfence
216#undef	L1D_FLUSH_SIZE
217after_l1d:
218	cmpl	$0, %r8d
219	je	do_launch
220	VMX_GUEST_RESTORE
221	vmresume
222	/*
223	 * In the common case 'vmresume' returns back to the host through
224	 * 'vmx_exit_guest' with %rsp pointing to 'vmxctx'.
225	 *
226	 * If there is an error we return VMX_VMRESUME_ERROR to the caller.
227	 */
228	movq	%rsp, %rdi		/* point %rdi back to 'vmxctx' */
229	movl	$VMX_VMRESUME_ERROR, %eax
230	jmp	decode_inst_error
231
232do_launch:
233	VMX_GUEST_RESTORE
234	vmlaunch
235	/*
236	 * In the common case 'vmlaunch' returns back to the host through
237	 * 'vmx_exit_guest' with %rsp pointing to 'vmxctx'.
238	 *
239	 * If there is an error we return VMX_VMLAUNCH_ERROR to the caller.
240	 */
241	movq	%rsp, %rdi		/* point %rdi back to 'vmxctx' */
242	movl	$VMX_VMLAUNCH_ERROR, %eax
243	jmp	decode_inst_error
244
245invept_error:
246	movl	$VMX_INVEPT_ERROR, %eax
247	jmp	decode_inst_error
248
249decode_inst_error:
250	movl	$VM_FAIL_VALID, %r11d
251	jz	inst_error
252	movl	$VM_FAIL_INVALID, %r11d
253inst_error:
254	movl	%r11d, VMXCTX_INST_FAIL_STATUS(%rdi)
255
256	/*
257	 * The return value is already populated in %eax so we cannot use
258	 * it as a scratch register beyond this point.
259	 */
260
261	/*
262	 * Deactivate guest pmap from this cpu.
263	 */
264	movq	VMXCTX_PMAP(%rdi), %r11
265	movl	PCPU(CPUID), %r10d
266	LK btrl	%r10d, PM_ACTIVE(%r11)
267
268	VMX_HOST_RESTORE
269	VLEAVE
270	ret
271
272/*
273 * Non-error VM-exit from the guest. Make this a label so it can
274 * be used by C code when setting up the VMCS.
275 * The VMCS-restored %rsp points to the struct vmxctx
276 */
277	ALIGN_TEXT
278	.globl	vmx_exit_guest_flush_rsb
279vmx_exit_guest_flush_rsb:
280	/*
281	 * Save guest state that is not automatically saved in the vmcs.
282	 */
283	VMX_GUEST_SAVE
284
285	/*
286	 * Deactivate guest pmap from this cpu.
287	 */
288	movq	VMXCTX_PMAP(%rdi), %r11
289	movl	PCPU(CPUID), %r10d
290	LK btrl	%r10d, PM_ACTIVE(%r11)
291
292	VMX_HOST_RESTORE
293
294	VMX_GUEST_CLOBBER
295
296	/*
297	 * To prevent malicious branch target predictions from
298	 * affecting the host, overwrite all entries in the RSB upon
299	 * exiting a guest.
300	 */
301	mov	$16, %ecx	/* 16 iterations, two calls per loop */
302	mov	%rsp, %rax
3030:	call	2f		/* create an RSB entry. */
3041:	pause
305	call	1b		/* capture rogue speculation. */
3062:	call	2f		/* create an RSB entry. */
3071:	pause
308	call	1b		/* capture rogue speculation. */
3092:	sub	$1, %ecx
310	jnz	0b
311	mov	%rax, %rsp
312
313	/*
314	 * This will return to the caller of 'vmx_enter_guest()' with a return
315	 * value of VMX_GUEST_VMEXIT.
316	 */
317	movl	$VMX_GUEST_VMEXIT, %eax
318	VLEAVE
319	ret
320
321	.globl	vmx_exit_guest
322vmx_exit_guest:
323	/*
324	 * Save guest state that is not automatically saved in the vmcs.
325	 */
326	VMX_GUEST_SAVE
327
328	/*
329	 * Deactivate guest pmap from this cpu.
330	 */
331	movq	VMXCTX_PMAP(%rdi), %r11
332	movl	PCPU(CPUID), %r10d
333	LK btrl	%r10d, PM_ACTIVE(%r11)
334
335	VMX_HOST_RESTORE
336
337	VMX_GUEST_CLOBBER
338
339	/*
340	 * This will return to the caller of 'vmx_enter_guest()' with a return
341	 * value of VMX_GUEST_VMEXIT.
342	 */
343	movl	$VMX_GUEST_VMEXIT, %eax
344	VLEAVE
345	ret
346END(vmx_enter_guest)
347
348/*
349 * %rdi = interrupt handler entry point
350 *
351 * Calling sequence described in the "Instruction Set Reference" for the "INT"
352 * instruction in Intel SDM, Vol 2.
353 */
354ENTRY(vmx_call_isr)
355	VENTER
356	mov	%rsp, %r11			/* save %rsp */
357	and	$~0xf, %rsp			/* align on 16-byte boundary */
358	pushq	$KERNEL_SS			/* %ss */
359	pushq	%r11				/* %rsp */
360	pushfq					/* %rflags */
361	pushq	$KERNEL_CS			/* %cs */
362	cli					/* disable interrupts */
363	callq	*%rdi				/* push %rip and call isr */
364	VLEAVE
365	ret
366END(vmx_call_isr)
367