xref: /freebsd/sys/amd64/vmm/intel/vmx_support.S (revision c30578feeb827ff09e753685632415ece27c127d)

/*-
 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
 *
 * Copyright (c) 2011 NetApp, Inc.
 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD$
 */

#include <machine/asmacros.h>
#include <machine/specialreg.h>

#include "vmx_assym.h"

#ifdef SMP
#define	LK	lock ;
#else
#define	LK
#endif

/* Be friendly to DTrace FBT's prologue/epilogue pattern matching */
#define VENTER  push %rbp ; mov %rsp,%rbp
#define VLEAVE  pop %rbp

/*
 * Save the guest context.
 */
#define	VMX_GUEST_SAVE							\
	movq	%rdi,VMXCTX_GUEST_RDI(%rsp);				\
	movq	%rsi,VMXCTX_GUEST_RSI(%rsp);				\
	movq	%rdx,VMXCTX_GUEST_RDX(%rsp);				\
	movq	%rcx,VMXCTX_GUEST_RCX(%rsp);				\
	movq	%r8,VMXCTX_GUEST_R8(%rsp);				\
	movq	%r9,VMXCTX_GUEST_R9(%rsp);				\
	movq	%rax,VMXCTX_GUEST_RAX(%rsp);				\
	movq	%rbx,VMXCTX_GUEST_RBX(%rsp);				\
	movq	%rbp,VMXCTX_GUEST_RBP(%rsp);				\
	movq	%r10,VMXCTX_GUEST_R10(%rsp);				\
	movq	%r11,VMXCTX_GUEST_R11(%rsp);				\
	movq	%r12,VMXCTX_GUEST_R12(%rsp);				\
	movq	%r13,VMXCTX_GUEST_R13(%rsp);				\
	movq	%r14,VMXCTX_GUEST_R14(%rsp);				\
	movq	%r15,VMXCTX_GUEST_R15(%rsp);				\
	movq	%cr2,%rdi;						\
	movq	%rdi,VMXCTX_GUEST_CR2(%rsp);				\
	movq	%rsp,%rdi;

/*
 * Assumes that %rdi holds a pointer to the 'vmxctx'.
 *
 * On "return" all registers are updated to reflect guest state. The two
 * exceptions are %rip and %rsp. These registers are atomically switched
 * by hardware from the guest area of the vmcs.
 *
 * We modify %rsp to point to the 'vmxctx' so we can use it to restore
 * host context in case of an error with 'vmlaunch' or 'vmresume'.
 */
#define	VMX_GUEST_RESTORE						\
	movq	%rdi,%rsp;						\
	movq	VMXCTX_GUEST_CR2(%rdi),%rsi;				\
	movq	%rsi,%cr2;						\
	movq	VMXCTX_GUEST_RSI(%rdi),%rsi;				\
	movq	VMXCTX_GUEST_RDX(%rdi),%rdx;				\
	movq	VMXCTX_GUEST_RCX(%rdi),%rcx;				\
	movq	VMXCTX_GUEST_R8(%rdi),%r8;				\
	movq	VMXCTX_GUEST_R9(%rdi),%r9;				\
	movq	VMXCTX_GUEST_RAX(%rdi),%rax;				\
	movq	VMXCTX_GUEST_RBX(%rdi),%rbx;				\
	movq	VMXCTX_GUEST_RBP(%rdi),%rbp;				\
	movq	VMXCTX_GUEST_R10(%rdi),%r10;				\
	movq	VMXCTX_GUEST_R11(%rdi),%r11;				\
	movq	VMXCTX_GUEST_R12(%rdi),%r12;				\
	movq	VMXCTX_GUEST_R13(%rdi),%r13;				\
	movq	VMXCTX_GUEST_R14(%rdi),%r14;				\
	movq	VMXCTX_GUEST_R15(%rdi),%r15;				\
	movq	VMXCTX_GUEST_RDI(%rdi),%rdi; /* restore rdi the last */

/*
 * Clobber the remaining registers with guest contents so they can't
 * be misused.
 */
#define	VMX_GUEST_CLOBBER						\
	xor	%rax, %rax;						\
	xor	%rcx, %rcx;						\
	xor	%rdx, %rdx;						\
	xor	%rsi, %rsi;						\
	xor	%r8, %r8;						\
	xor	%r9, %r9;						\
	xor	%r10, %r10;						\
	xor	%r11, %r11;

/*
 * Save and restore the host context.
 *
 * Assumes that %rdi holds a pointer to the 'vmxctx'.
 */
#define	VMX_HOST_SAVE							\
	movq    %r15, VMXCTX_HOST_R15(%rdi);				\
	movq    %r14, VMXCTX_HOST_R14(%rdi);				\
	movq    %r13, VMXCTX_HOST_R13(%rdi);				\
	movq    %r12, VMXCTX_HOST_R12(%rdi);				\
	movq    %rbp, VMXCTX_HOST_RBP(%rdi);				\
	movq    %rsp, VMXCTX_HOST_RSP(%rdi);				\
	movq    %rbx, VMXCTX_HOST_RBX(%rdi);				\

#define	VMX_HOST_RESTORE						\
	movq	VMXCTX_HOST_R15(%rdi), %r15;				\
	movq	VMXCTX_HOST_R14(%rdi), %r14;				\
	movq	VMXCTX_HOST_R13(%rdi), %r13;				\
	movq	VMXCTX_HOST_R12(%rdi), %r12;				\
	movq	VMXCTX_HOST_RBP(%rdi), %rbp;				\
	movq	VMXCTX_HOST_RSP(%rdi), %rsp;				\
	movq	VMXCTX_HOST_RBX(%rdi), %rbx;				\

/*
 * vmx_enter_guest(struct vmxctx *vmxctx, int launched)
 * %rdi: pointer to the 'vmxctx'
 * %rsi: pointer to the 'vmx'
 * %edx: launch state of the VMCS
 * Interrupts must be disabled on entry.
 */
ENTRY(vmx_enter_guest)
	VENTER
	/*
	 * Save host state before doing anything else.
	 */
	VMX_HOST_SAVE

	/*
	 * Activate guest pmap on this cpu.
	 */
	movq	VMXCTX_PMAP(%rdi), %r11
	movl	PCPU(CPUID), %eax
	LK btsl	%eax, PM_ACTIVE(%r11)

	/*
	 * If 'vmx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen'
	 * then we must invalidate all mappings associated with this EPTP.
	 */
	movq	PM_EPTGEN(%r11), %r10
	cmpq	%r10, VMX_EPTGEN(%rsi, %rax, 8)
	je	guest_restore

	/* Refresh 'vmx->eptgen[curcpu]' */
	movq	%r10, VMX_EPTGEN(%rsi, %rax, 8)

	/* Setup the invept descriptor on the host stack */
	mov	%rsp, %r11
	movq	VMX_EPTP(%rsi), %rax
	movq	%rax, -16(%r11)
	movq	$0x0, -8(%r11)
	mov	$0x1, %eax		/* Single context invalidate */
	invept	-16(%r11), %rax
	jbe	invept_error		/* Check invept instruction error */

guest_restore:

	/*
	 * Flush L1D cache if requested.  Use IA32_FLUSH_CMD MSR if available,
	 * otherwise load enough of the data from the zero_region to flush
	 * existing L1D content.
	 */
#define	L1D_FLUSH_SIZE	(64 * 1024)
	movl	%edx, %r8d
	cmpb	$0, guest_l1d_flush(%rip)
	je	after_l1d
	movq	vmx_msr_flush_cmd(%rip), %rax
	testq	%rax, %rax
	jz	1f
	movq	%rax, %rdx
	shrq	$32, %rdx
	movl	$MSR_IA32_FLUSH_CMD, %ecx
	wrmsr
	jmp	after_l1d
1:	movq	$KERNBASE, %r9
	movq	$-L1D_FLUSH_SIZE, %rcx
	/*
	 * pass 1: Preload TLB.
	 * Kernel text is mapped using superpages.  TLB preload is
	 * done for the benefit of older CPUs which split 2M page
	 * into 4k TLB entries.
	 */
2:	movb	L1D_FLUSH_SIZE(%r9, %rcx), %al
	addq	$PAGE_SIZE, %rcx
	jne	2b
	xorl	%eax, %eax
	cpuid
	movq	$-L1D_FLUSH_SIZE, %rcx
	/* pass 2: Read each cache line */
3:	movb	L1D_FLUSH_SIZE(%r9, %rcx), %al
	addq	$64, %rcx
	jne	3b
	lfence
#undef	L1D_FLUSH_SIZE
after_l1d:
	cmpl	$0, %r8d
	je	do_launch
	VMX_GUEST_RESTORE
	vmresume
	/*
	 * In the common case 'vmresume' returns back to the host through
	 * 'vmx_exit_guest' with %rsp pointing to 'vmxctx'.
	 *
	 * If there is an error we return VMX_VMRESUME_ERROR to the caller.
	 */
	movq	%rsp, %rdi		/* point %rdi back to 'vmxctx' */
	movl	$VMX_VMRESUME_ERROR, %eax
	jmp	decode_inst_error

do_launch:
	VMX_GUEST_RESTORE
	vmlaunch
	/*
	 * In the common case 'vmlaunch' returns back to the host through
	 * 'vmx_exit_guest' with %rsp pointing to 'vmxctx'.
	 *
	 * If there is an error we return VMX_VMLAUNCH_ERROR to the caller.
	 */
	movq	%rsp, %rdi		/* point %rdi back to 'vmxctx' */
	movl	$VMX_VMLAUNCH_ERROR, %eax
	jmp	decode_inst_error

invept_error:
	movl	$VMX_INVEPT_ERROR, %eax
	jmp	decode_inst_error

decode_inst_error:
	movl	$VM_FAIL_VALID, %r11d
	jz	inst_error
	movl	$VM_FAIL_INVALID, %r11d
inst_error:
	movl	%r11d, VMXCTX_INST_FAIL_STATUS(%rdi)

	/*
	 * The return value is already populated in %eax so we cannot use
	 * it as a scratch register beyond this point.
	 */

	/*
	 * Deactivate guest pmap from this cpu.
	 */
	movq	VMXCTX_PMAP(%rdi), %r11
	movl	PCPU(CPUID), %r10d
	LK btrl	%r10d, PM_ACTIVE(%r11)

	VMX_HOST_RESTORE
	VLEAVE
	ret

/*
 * Non-error VM-exit from the guest. Make this a label so it can
 * be used by C code when setting up the VMCS.
 * The VMCS-restored %rsp points to the struct vmxctx
 */
	ALIGN_TEXT
	.globl	vmx_exit_guest_flush_rsb
vmx_exit_guest_flush_rsb:
	/*
	 * Save guest state that is not automatically saved in the vmcs.
	 */
	VMX_GUEST_SAVE

	/*
	 * Deactivate guest pmap from this cpu.
	 */
	movq	VMXCTX_PMAP(%rdi), %r11
	movl	PCPU(CPUID), %r10d
	LK btrl	%r10d, PM_ACTIVE(%r11)

	VMX_HOST_RESTORE

	VMX_GUEST_CLOBBER

	/*
	 * To prevent malicious branch target predictions from
	 * affecting the host, overwrite all entries in the RSB upon
	 * exiting a guest.
	 */
	mov	$16, %ecx	/* 16 iterations, two calls per loop */
	mov	%rsp, %rax
0:	call	2f		/* create an RSB entry. */
1:	pause
	call	1b		/* capture rogue speculation. */
2:	call	2f		/* create an RSB entry. */
1:	pause
	call	1b		/* capture rogue speculation. */
2:	sub	$1, %ecx
	jnz	0b
	mov	%rax, %rsp

	/*
	 * This will return to the caller of 'vmx_enter_guest()' with a return
	 * value of VMX_GUEST_VMEXIT.
	 */
	movl	$VMX_GUEST_VMEXIT, %eax
	VLEAVE
	ret

	.globl	vmx_exit_guest
vmx_exit_guest:
	/*
	 * Save guest state that is not automatically saved in the vmcs.
	 */
	VMX_GUEST_SAVE

	/*
	 * Deactivate guest pmap from this cpu.
	 */
	movq	VMXCTX_PMAP(%rdi), %r11
	movl	PCPU(CPUID), %r10d
	LK btrl	%r10d, PM_ACTIVE(%r11)

	VMX_HOST_RESTORE

	VMX_GUEST_CLOBBER

	/*
	 * This will return to the caller of 'vmx_enter_guest()' with a return
	 * value of VMX_GUEST_VMEXIT.
	 */
	movl	$VMX_GUEST_VMEXIT, %eax
	VLEAVE
	ret
END(vmx_enter_guest)

/*
 * %rdi = interrupt handler entry point
 *
 * Calling sequence described in the "Instruction Set Reference" for the "INT"
 * instruction in Intel SDM, Vol 2.
 */
ENTRY(vmx_call_isr)
	VENTER
	mov	%rsp, %r11			/* save %rsp */
	and	$~0xf, %rsp			/* align on 16-byte boundary */
	pushq	$KERNEL_SS			/* %ss */
	pushq	%r11				/* %rsp */
	pushfq					/* %rflags */
	pushq	$KERNEL_CS			/* %cs */
	cli					/* disable interrupts */
	callq	*%rdi				/* push %rip and call isr */
	VLEAVE
	ret
END(vmx_call_isr)