1/* SPDX-License-Identifier: GPL-2.0 */ 2#include <linux/linkage.h> 3#include <asm/asm.h> 4#include <asm/bitsperlong.h> 5#include <asm/kvm_vcpu_regs.h> 6#include <asm/nospec-branch.h> 7#include <asm/percpu.h> 8#include <asm/segment.h> 9#include "kvm-asm-offsets.h" 10#include "run_flags.h" 11 12#define WORD_SIZE (BITS_PER_LONG / 8) 13 14#define VCPU_RAX __VCPU_REGS_RAX * WORD_SIZE 15#define VCPU_RCX __VCPU_REGS_RCX * WORD_SIZE 16#define VCPU_RDX __VCPU_REGS_RDX * WORD_SIZE 17#define VCPU_RBX __VCPU_REGS_RBX * WORD_SIZE 18/* Intentionally omit RSP as it's context switched by hardware */ 19#define VCPU_RBP __VCPU_REGS_RBP * WORD_SIZE 20#define VCPU_RSI __VCPU_REGS_RSI * WORD_SIZE 21#define VCPU_RDI __VCPU_REGS_RDI * WORD_SIZE 22 23#ifdef CONFIG_X86_64 24#define VCPU_R8 __VCPU_REGS_R8 * WORD_SIZE 25#define VCPU_R9 __VCPU_REGS_R9 * WORD_SIZE 26#define VCPU_R10 __VCPU_REGS_R10 * WORD_SIZE 27#define VCPU_R11 __VCPU_REGS_R11 * WORD_SIZE 28#define VCPU_R12 __VCPU_REGS_R12 * WORD_SIZE 29#define VCPU_R13 __VCPU_REGS_R13 * WORD_SIZE 30#define VCPU_R14 __VCPU_REGS_R14 * WORD_SIZE 31#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE 32#endif 33 34.macro VMX_DO_EVENT_IRQOFF call_insn call_target 35 /* 36 * Unconditionally create a stack frame, getting the correct RSP on the 37 * stack (for x86-64) would take two instructions anyways, and RBP can 38 * be used to restore RSP to make objtool happy (see below). 39 */ 40 push %_ASM_BP 41 mov %_ASM_SP, %_ASM_BP 42 43#ifdef CONFIG_X86_64 44 /* 45 * Align RSP to a 16-byte boundary (to emulate CPU behavior) before 46 * creating the synthetic interrupt stack frame for the IRQ/NMI. 47 */ 48 and $-16, %rsp 49 push $__KERNEL_DS 50 push %rbp 51#endif 52 pushf 53 push $__KERNEL_CS 54 \call_insn \call_target 55 56 /* 57 * "Restore" RSP from RBP, even though IRET has already unwound RSP to 58 * the correct value. objtool doesn't know the callee will IRET and, 59 * without the explicit restore, thinks the stack is getting walloped. 60 * Using an unwind hint is problematic due to x86-64's dynamic alignment. 61 */ 62 mov %_ASM_BP, %_ASM_SP 63 pop %_ASM_BP 64 RET 65.endm 66 67.section .noinstr.text, "ax" 68 69/** 70 * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode 71 * @vmx: struct vcpu_vmx * 72 * @regs: unsigned long * (to guest registers) 73 * @flags: VMX_RUN_VMRESUME: use VMRESUME instead of VMLAUNCH 74 * VMX_RUN_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl 75 * 76 * Returns: 77 * 0 on VM-Exit, 1 on VM-Fail 78 */ 79SYM_FUNC_START(__vmx_vcpu_run) 80 push %_ASM_BP 81 mov %_ASM_SP, %_ASM_BP 82#ifdef CONFIG_X86_64 83 push %r15 84 push %r14 85 push %r13 86 push %r12 87#else 88 push %edi 89 push %esi 90#endif 91 push %_ASM_BX 92 93 /* Save @vmx for SPEC_CTRL handling */ 94 push %_ASM_ARG1 95 96 /* Save @flags for SPEC_CTRL handling */ 97 push %_ASM_ARG3 98 99 /* 100 * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and 101 * @regs is needed after VM-Exit to save the guest's register values. 102 */ 103 push %_ASM_ARG2 104 105 /* Copy @flags to EBX, _ASM_ARG3 is volatile. */ 106 mov %_ASM_ARG3L, %ebx 107 108 lea (%_ASM_SP), %_ASM_ARG2 109 call vmx_update_host_rsp 110 111 ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL 112 113 /* 114 * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the 115 * host's, write the MSR. 116 * 117 * IMPORTANT: To avoid RSB underflow attacks and any other nastiness, 118 * there must not be any returns or indirect branches between this code 119 * and vmentry. 120 */ 121 mov 2*WORD_SIZE(%_ASM_SP), %_ASM_DI 122 movl VMX_spec_ctrl(%_ASM_DI), %edi 123 movl PER_CPU_VAR(x86_spec_ctrl_current), %esi 124 cmp %edi, %esi 125 je .Lspec_ctrl_done 126 mov $MSR_IA32_SPEC_CTRL, %ecx 127 xor %edx, %edx 128 mov %edi, %eax 129 wrmsr 130 131.Lspec_ctrl_done: 132 133 /* 134 * Since vmentry is serializing on affected CPUs, there's no need for 135 * an LFENCE to stop speculation from skipping the wrmsr. 136 */ 137 138 /* Load @regs to RAX. */ 139 mov (%_ASM_SP), %_ASM_AX 140 141 /* Check if vmlaunch or vmresume is needed */ 142 test $VMX_RUN_VMRESUME, %ebx 143 144 /* Load guest registers. Don't clobber flags. */ 145 mov VCPU_RCX(%_ASM_AX), %_ASM_CX 146 mov VCPU_RDX(%_ASM_AX), %_ASM_DX 147 mov VCPU_RBX(%_ASM_AX), %_ASM_BX 148 mov VCPU_RBP(%_ASM_AX), %_ASM_BP 149 mov VCPU_RSI(%_ASM_AX), %_ASM_SI 150 mov VCPU_RDI(%_ASM_AX), %_ASM_DI 151#ifdef CONFIG_X86_64 152 mov VCPU_R8 (%_ASM_AX), %r8 153 mov VCPU_R9 (%_ASM_AX), %r9 154 mov VCPU_R10(%_ASM_AX), %r10 155 mov VCPU_R11(%_ASM_AX), %r11 156 mov VCPU_R12(%_ASM_AX), %r12 157 mov VCPU_R13(%_ASM_AX), %r13 158 mov VCPU_R14(%_ASM_AX), %r14 159 mov VCPU_R15(%_ASM_AX), %r15 160#endif 161 /* Load guest RAX. This kills the @regs pointer! */ 162 mov VCPU_RAX(%_ASM_AX), %_ASM_AX 163 164 /* Check EFLAGS.ZF from 'test VMX_RUN_VMRESUME' above */ 165 jz .Lvmlaunch 166 167 /* 168 * After a successful VMRESUME/VMLAUNCH, control flow "magically" 169 * resumes below at 'vmx_vmexit' due to the VMCS HOST_RIP setting. 170 * So this isn't a typical function and objtool needs to be told to 171 * save the unwind state here and restore it below. 172 */ 173 UNWIND_HINT_SAVE 174 175/* 176 * If VMRESUME/VMLAUNCH and corresponding vmexit succeed, execution resumes at 177 * the 'vmx_vmexit' label below. 178 */ 179.Lvmresume: 180 vmresume 181 jmp .Lvmfail 182 183.Lvmlaunch: 184 vmlaunch 185 jmp .Lvmfail 186 187 _ASM_EXTABLE(.Lvmresume, .Lfixup) 188 _ASM_EXTABLE(.Lvmlaunch, .Lfixup) 189 190SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL) 191 192 /* Restore unwind state from before the VMRESUME/VMLAUNCH. */ 193 UNWIND_HINT_RESTORE 194 ENDBR 195 196 /* Temporarily save guest's RAX. */ 197 push %_ASM_AX 198 199 /* Reload @regs to RAX. */ 200 mov WORD_SIZE(%_ASM_SP), %_ASM_AX 201 202 /* Save all guest registers, including RAX from the stack */ 203 pop VCPU_RAX(%_ASM_AX) 204 mov %_ASM_CX, VCPU_RCX(%_ASM_AX) 205 mov %_ASM_DX, VCPU_RDX(%_ASM_AX) 206 mov %_ASM_BX, VCPU_RBX(%_ASM_AX) 207 mov %_ASM_BP, VCPU_RBP(%_ASM_AX) 208 mov %_ASM_SI, VCPU_RSI(%_ASM_AX) 209 mov %_ASM_DI, VCPU_RDI(%_ASM_AX) 210#ifdef CONFIG_X86_64 211 mov %r8, VCPU_R8 (%_ASM_AX) 212 mov %r9, VCPU_R9 (%_ASM_AX) 213 mov %r10, VCPU_R10(%_ASM_AX) 214 mov %r11, VCPU_R11(%_ASM_AX) 215 mov %r12, VCPU_R12(%_ASM_AX) 216 mov %r13, VCPU_R13(%_ASM_AX) 217 mov %r14, VCPU_R14(%_ASM_AX) 218 mov %r15, VCPU_R15(%_ASM_AX) 219#endif 220 221 /* Clear return value to indicate VM-Exit (as opposed to VM-Fail). */ 222 xor %ebx, %ebx 223 224.Lclear_regs: 225 /* Discard @regs. The register is irrelevant, it just can't be RBX. */ 226 pop %_ASM_AX 227 228 /* 229 * Clear all general purpose registers except RSP and RBX to prevent 230 * speculative use of the guest's values, even those that are reloaded 231 * via the stack. In theory, an L1 cache miss when restoring registers 232 * could lead to speculative execution with the guest's values. 233 * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially 234 * free. RSP and RBX are exempt as RSP is restored by hardware during 235 * VM-Exit and RBX is explicitly loaded with 0 or 1 to hold the return 236 * value. 237 */ 238 xor %eax, %eax 239 xor %ecx, %ecx 240 xor %edx, %edx 241 xor %ebp, %ebp 242 xor %esi, %esi 243 xor %edi, %edi 244#ifdef CONFIG_X86_64 245 xor %r8d, %r8d 246 xor %r9d, %r9d 247 xor %r10d, %r10d 248 xor %r11d, %r11d 249 xor %r12d, %r12d 250 xor %r13d, %r13d 251 xor %r14d, %r14d 252 xor %r15d, %r15d 253#endif 254 255 /* 256 * IMPORTANT: RSB filling and SPEC_CTRL handling must be done before 257 * the first unbalanced RET after vmexit! 258 * 259 * For retpoline or IBRS, RSB filling is needed to prevent poisoned RSB 260 * entries and (in some cases) RSB underflow. 261 * 262 * eIBRS has its own protection against poisoned RSB, so it doesn't 263 * need the RSB filling sequence. But it does need to be enabled, and a 264 * single call to retire, before the first unbalanced RET. 265 */ 266 267 FILL_RETURN_BUFFER %_ASM_CX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT,\ 268 X86_FEATURE_RSB_VMEXIT_LITE 269 270 pop %_ASM_ARG2 /* @flags */ 271 pop %_ASM_ARG1 /* @vmx */ 272 273 call vmx_spec_ctrl_restore_host 274 275 /* Put return value in AX */ 276 mov %_ASM_BX, %_ASM_AX 277 278 pop %_ASM_BX 279#ifdef CONFIG_X86_64 280 pop %r12 281 pop %r13 282 pop %r14 283 pop %r15 284#else 285 pop %esi 286 pop %edi 287#endif 288 pop %_ASM_BP 289 RET 290 291.Lfixup: 292 cmpb $0, _ASM_RIP(kvm_rebooting) 293 jne .Lvmfail 294 ud2 295.Lvmfail: 296 /* VM-Fail: set return value to 1 */ 297 mov $1, %_ASM_BX 298 jmp .Lclear_regs 299 300SYM_FUNC_END(__vmx_vcpu_run) 301 302SYM_FUNC_START(vmx_do_nmi_irqoff) 303 VMX_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx 304SYM_FUNC_END(vmx_do_nmi_irqoff) 305 306#ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 307 308/** 309 * vmread_error_trampoline - Trampoline from inline asm to vmread_error() 310 * @field: VMCS field encoding that failed 311 * @fault: %true if the VMREAD faulted, %false if it failed 312 * 313 * Save and restore volatile registers across a call to vmread_error(). Note, 314 * all parameters are passed on the stack. 315 */ 316SYM_FUNC_START(vmread_error_trampoline) 317 push %_ASM_BP 318 mov %_ASM_SP, %_ASM_BP 319 320 push %_ASM_AX 321 push %_ASM_CX 322 push %_ASM_DX 323#ifdef CONFIG_X86_64 324 push %rdi 325 push %rsi 326 push %r8 327 push %r9 328 push %r10 329 push %r11 330#endif 331 332 /* Load @field and @fault to arg1 and arg2 respectively. */ 333 mov 3*WORD_SIZE(%_ASM_BP), %_ASM_ARG2 334 mov 2*WORD_SIZE(%_ASM_BP), %_ASM_ARG1 335 336 call vmread_error_trampoline2 337 338 /* Zero out @fault, which will be popped into the result register. */ 339 _ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP) 340 341#ifdef CONFIG_X86_64 342 pop %r11 343 pop %r10 344 pop %r9 345 pop %r8 346 pop %rsi 347 pop %rdi 348#endif 349 pop %_ASM_DX 350 pop %_ASM_CX 351 pop %_ASM_AX 352 pop %_ASM_BP 353 354 RET 355SYM_FUNC_END(vmread_error_trampoline) 356#endif 357 358.section .text, "ax" 359 360SYM_FUNC_START(vmx_do_interrupt_irqoff) 361 VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 362SYM_FUNC_END(vmx_do_interrupt_irqoff) 363