1/* SPDX-License-Identifier: GPL-2.0 */ 2#include <linux/linkage.h> 3#include <asm/asm.h> 4#include <asm/bitsperlong.h> 5#include <asm/kvm_vcpu_regs.h> 6#include <asm/nospec-branch.h> 7#include <asm/percpu.h> 8#include <asm/segment.h> 9#include "kvm-asm-offsets.h" 10#include "run_flags.h" 11 12#define WORD_SIZE (BITS_PER_LONG / 8) 13 14#define VCPU_RAX __VCPU_REGS_RAX * WORD_SIZE 15#define VCPU_RCX __VCPU_REGS_RCX * WORD_SIZE 16#define VCPU_RDX __VCPU_REGS_RDX * WORD_SIZE 17#define VCPU_RBX __VCPU_REGS_RBX * WORD_SIZE 18/* Intentionally omit RSP as it's context switched by hardware */ 19#define VCPU_RBP __VCPU_REGS_RBP * WORD_SIZE 20#define VCPU_RSI __VCPU_REGS_RSI * WORD_SIZE 21#define VCPU_RDI __VCPU_REGS_RDI * WORD_SIZE 22 23#ifdef CONFIG_X86_64 24#define VCPU_R8 __VCPU_REGS_R8 * WORD_SIZE 25#define VCPU_R9 __VCPU_REGS_R9 * WORD_SIZE 26#define VCPU_R10 __VCPU_REGS_R10 * WORD_SIZE 27#define VCPU_R11 __VCPU_REGS_R11 * WORD_SIZE 28#define VCPU_R12 __VCPU_REGS_R12 * WORD_SIZE 29#define VCPU_R13 __VCPU_REGS_R13 * WORD_SIZE 30#define VCPU_R14 __VCPU_REGS_R14 * WORD_SIZE 31#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE 32#endif 33 34.section .noinstr.text, "ax" 35 36/** 37 * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode 38 * @vmx: struct vcpu_vmx * 39 * @regs: unsigned long * (to guest registers) 40 * @flags: VMX_RUN_VMRESUME: use VMRESUME instead of VMLAUNCH 41 * VMX_RUN_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl 42 * VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO: vCPU can access host MMIO 43 * 44 * Returns: 45 * 0 on VM-Exit, 1 on VM-Fail 46 */ 47SYM_FUNC_START(__vmx_vcpu_run) 48 push %_ASM_BP 49 mov %_ASM_SP, %_ASM_BP 50#ifdef CONFIG_X86_64 51 push %r15 52 push %r14 53 push %r13 54 push %r12 55#else 56 push %edi 57 push %esi 58#endif 59 push %_ASM_BX 60 61 /* Save @vmx for SPEC_CTRL handling */ 62 push %_ASM_ARG1 63 64 /* Save @flags (used for VMLAUNCH vs. VMRESUME and mitigations). */ 65 push %_ASM_ARG3 66 67 /* 68 * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and 69 * @regs is needed after VM-Exit to save the guest's register values. 70 */ 71 push %_ASM_ARG2 72 73 lea (%_ASM_SP), %_ASM_ARG2 74 call vmx_update_host_rsp 75 76 ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL 77 78 /* 79 * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the 80 * host's, write the MSR. 81 * 82 * IMPORTANT: To avoid RSB underflow attacks and any other nastiness, 83 * there must not be any returns or indirect branches between this code 84 * and vmentry. 85 */ 86 mov 2*WORD_SIZE(%_ASM_SP), %_ASM_DI 87#ifdef CONFIG_X86_64 88 mov VMX_spec_ctrl(%rdi), %rdx 89 cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx 90 je .Lspec_ctrl_done 91 movl %edx, %eax 92 shr $32, %rdx 93#else 94 mov VMX_spec_ctrl(%edi), %eax 95 mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx 96 xor %eax, %ecx 97 mov VMX_spec_ctrl + 4(%edi), %edx 98 mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edi 99 xor %edx, %edi 100 or %edi, %ecx 101 je .Lspec_ctrl_done 102#endif 103 mov $MSR_IA32_SPEC_CTRL, %ecx 104 wrmsr 105 106.Lspec_ctrl_done: 107 108 /* 109 * Since vmentry is serializing on affected CPUs, there's no need for 110 * an LFENCE to stop speculation from skipping the wrmsr. 111 */ 112 113 /* Load @regs to RAX. */ 114 mov (%_ASM_SP), %_ASM_AX 115 116 /* Load guest registers. Don't clobber flags. */ 117 mov VCPU_RCX(%_ASM_AX), %_ASM_CX 118 mov VCPU_RDX(%_ASM_AX), %_ASM_DX 119 mov VCPU_RBX(%_ASM_AX), %_ASM_BX 120 mov VCPU_RBP(%_ASM_AX), %_ASM_BP 121 mov VCPU_RSI(%_ASM_AX), %_ASM_SI 122 mov VCPU_RDI(%_ASM_AX), %_ASM_DI 123#ifdef CONFIG_X86_64 124 mov VCPU_R8 (%_ASM_AX), %r8 125 mov VCPU_R9 (%_ASM_AX), %r9 126 mov VCPU_R10(%_ASM_AX), %r10 127 mov VCPU_R11(%_ASM_AX), %r11 128 mov VCPU_R12(%_ASM_AX), %r12 129 mov VCPU_R13(%_ASM_AX), %r13 130 mov VCPU_R14(%_ASM_AX), %r14 131 mov VCPU_R15(%_ASM_AX), %r15 132#endif 133 /* Load guest RAX. This kills the @regs pointer! */ 134 mov VCPU_RAX(%_ASM_AX), %_ASM_AX 135 136 /* 137 * Note, ALTERNATIVE_2 works in reverse order. If CLEAR_CPU_BUF_VM is 138 * enabled, do VERW unconditionally. If CPU_BUF_VM_MMIO is enabled, 139 * check @flags to see if the vCPU has access to host MMIO, and if so, 140 * do VERW. Else, do nothing (no mitigations needed/enabled). 141 */ 142 ALTERNATIVE_2 "", \ 143 __stringify(testl $VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO, WORD_SIZE(%_ASM_SP); \ 144 jz .Lskip_mmio_verw; \ 145 VERW; \ 146 .Lskip_mmio_verw:), \ 147 X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO, \ 148 __stringify(VERW), X86_FEATURE_CLEAR_CPU_BUF_VM 149 150 /* Check @flags to see if VMLAUNCH or VMRESUME is needed. */ 151 testl $VMX_RUN_VMRESUME, WORD_SIZE(%_ASM_SP) 152 jz .Lvmlaunch 153 154 /* 155 * After a successful VMRESUME/VMLAUNCH, control flow "magically" 156 * resumes below at 'vmx_vmexit' due to the VMCS HOST_RIP setting. 157 * So this isn't a typical function and objtool needs to be told to 158 * save the unwind state here and restore it below. 159 */ 160 UNWIND_HINT_SAVE 161 162/* 163 * If VMRESUME/VMLAUNCH and corresponding vmexit succeed, execution resumes at 164 * the 'vmx_vmexit' label below. 165 */ 166.Lvmresume: 167 vmresume 168 jmp .Lvmfail 169 170.Lvmlaunch: 171 vmlaunch 172 jmp .Lvmfail 173 174 _ASM_EXTABLE(.Lvmresume, .Lfixup) 175 _ASM_EXTABLE(.Lvmlaunch, .Lfixup) 176 177SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL) 178 179 /* Restore unwind state from before the VMRESUME/VMLAUNCH. */ 180 UNWIND_HINT_RESTORE 181 ENDBR 182 183 /* Temporarily save guest's RAX. */ 184 push %_ASM_AX 185 186 /* Reload @regs to RAX. */ 187 mov WORD_SIZE(%_ASM_SP), %_ASM_AX 188 189 /* Save all guest registers, including RAX from the stack */ 190 pop VCPU_RAX(%_ASM_AX) 191 mov %_ASM_CX, VCPU_RCX(%_ASM_AX) 192 mov %_ASM_DX, VCPU_RDX(%_ASM_AX) 193 mov %_ASM_BX, VCPU_RBX(%_ASM_AX) 194 mov %_ASM_BP, VCPU_RBP(%_ASM_AX) 195 mov %_ASM_SI, VCPU_RSI(%_ASM_AX) 196 mov %_ASM_DI, VCPU_RDI(%_ASM_AX) 197#ifdef CONFIG_X86_64 198 mov %r8, VCPU_R8 (%_ASM_AX) 199 mov %r9, VCPU_R9 (%_ASM_AX) 200 mov %r10, VCPU_R10(%_ASM_AX) 201 mov %r11, VCPU_R11(%_ASM_AX) 202 mov %r12, VCPU_R12(%_ASM_AX) 203 mov %r13, VCPU_R13(%_ASM_AX) 204 mov %r14, VCPU_R14(%_ASM_AX) 205 mov %r15, VCPU_R15(%_ASM_AX) 206#endif 207 208 /* Clear return value to indicate VM-Exit (as opposed to VM-Fail). */ 209 xor %ebx, %ebx 210 211.Lclear_regs: 212 /* Discard @regs. The register is irrelevant, it just can't be RBX. */ 213 pop %_ASM_AX 214 215 /* 216 * Clear all general purpose registers except RSP and RBX to prevent 217 * speculative use of the guest's values, even those that are reloaded 218 * via the stack. In theory, an L1 cache miss when restoring registers 219 * could lead to speculative execution with the guest's values. 220 * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially 221 * free. RSP and RBX are exempt as RSP is restored by hardware during 222 * VM-Exit and RBX is explicitly loaded with 0 or 1 to hold the return 223 * value. 224 */ 225 xor %eax, %eax 226 xor %ecx, %ecx 227 xor %edx, %edx 228 xor %ebp, %ebp 229 xor %esi, %esi 230 xor %edi, %edi 231#ifdef CONFIG_X86_64 232 xor %r8d, %r8d 233 xor %r9d, %r9d 234 xor %r10d, %r10d 235 xor %r11d, %r11d 236 xor %r12d, %r12d 237 xor %r13d, %r13d 238 xor %r14d, %r14d 239 xor %r15d, %r15d 240#endif 241 242 /* 243 * IMPORTANT: RSB filling and SPEC_CTRL handling must be done before 244 * the first unbalanced RET after vmexit! 245 * 246 * For retpoline or IBRS, RSB filling is needed to prevent poisoned RSB 247 * entries and (in some cases) RSB underflow. 248 * 249 * eIBRS has its own protection against poisoned RSB, so it doesn't 250 * need the RSB filling sequence. But it does need to be enabled, and a 251 * single call to retire, before the first unbalanced RET. 252 */ 253 254 FILL_RETURN_BUFFER %_ASM_CX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT,\ 255 X86_FEATURE_RSB_VMEXIT_LITE 256 257 pop %_ASM_ARG2 /* @flags */ 258 pop %_ASM_ARG1 /* @vmx */ 259 260 call vmx_spec_ctrl_restore_host 261 262 CLEAR_BRANCH_HISTORY_VMEXIT 263 264 /* Put return value in AX */ 265 mov %_ASM_BX, %_ASM_AX 266 267 pop %_ASM_BX 268#ifdef CONFIG_X86_64 269 pop %r12 270 pop %r13 271 pop %r14 272 pop %r15 273#else 274 pop %esi 275 pop %edi 276#endif 277 pop %_ASM_BP 278 RET 279 280.Lfixup: 281 cmpb $0, _ASM_RIP(virt_rebooting) 282 jne .Lvmfail 283 ud2 284.Lvmfail: 285 /* VM-Fail: set return value to 1 */ 286 mov $1, %_ASM_BX 287 jmp .Lclear_regs 288 289SYM_FUNC_END(__vmx_vcpu_run) 290 291#ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 292 293/** 294 * vmread_error_trampoline - Trampoline from inline asm to vmread_error() 295 * @field: VMCS field encoding that failed 296 * @fault: %true if the VMREAD faulted, %false if it failed 297 * 298 * Save and restore volatile registers across a call to vmread_error(). Note, 299 * all parameters are passed on the stack. 300 */ 301SYM_FUNC_START(vmread_error_trampoline) 302 push %_ASM_BP 303 mov %_ASM_SP, %_ASM_BP 304 305 push %_ASM_AX 306 push %_ASM_CX 307 push %_ASM_DX 308#ifdef CONFIG_X86_64 309 push %rdi 310 push %rsi 311 push %r8 312 push %r9 313 push %r10 314 push %r11 315#endif 316 317 /* Load @field and @fault to arg1 and arg2 respectively. */ 318 mov 3*WORD_SIZE(%_ASM_BP), %_ASM_ARG2 319 mov 2*WORD_SIZE(%_ASM_BP), %_ASM_ARG1 320 321 call vmread_error_trampoline2 322 323 /* Zero out @fault, which will be popped into the result register. */ 324 _ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP) 325 326#ifdef CONFIG_X86_64 327 pop %r11 328 pop %r10 329 pop %r9 330 pop %r8 331 pop %rsi 332 pop %rdi 333#endif 334 pop %_ASM_DX 335 pop %_ASM_CX 336 pop %_ASM_AX 337 pop %_ASM_BP 338 339 RET 340SYM_FUNC_END(vmread_error_trampoline) 341#endif 342