1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __KVM_X86_VMX_INSN_H 3 #define __KVM_X86_VMX_INSN_H 4 5 #include <linux/nospec.h> 6 7 #include <asm/vmx.h> 8 9 #include "vmx_onhyperv.h" 10 #include "vmcs.h" 11 #include "../x86.h" 12 13 void vmread_error(unsigned long field); 14 void vmwrite_error(unsigned long field, unsigned long value); 15 void vmclear_error(struct vmcs *vmcs, u64 phys_addr); 16 void vmptrld_error(struct vmcs *vmcs, u64 phys_addr); 17 void invvpid_error(unsigned long ext, u16 vpid, gva_t gva); 18 void invept_error(unsigned long ext, u64 eptp); 19 20 #ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 21 /* 22 * The VMREAD error trampoline _always_ uses the stack to pass parameters, even 23 * for 64-bit targets. Preserving all registers allows the VMREAD inline asm 24 * blob to avoid clobbering GPRs, which in turn allows the compiler to better 25 * optimize sequences of VMREADs. 26 * 27 * Declare the trampoline as an opaque label as it's not safe to call from C 28 * code; there is no way to tell the compiler to pass params on the stack for 29 * 64-bit targets. 30 * 31 * void vmread_error_trampoline(unsigned long field, bool fault); 32 */ 33 extern unsigned long vmread_error_trampoline; 34 35 /* 36 * The second VMREAD error trampoline, called from the assembly trampoline, 37 * exists primarily to enable instrumentation for the VM-Fail path. 38 */ 39 void vmread_error_trampoline2(unsigned long field, bool fault); 40 41 #endif 42 43 static __always_inline void vmcs_check16(unsigned long field) 44 { 45 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 46 "16-bit accessor invalid for 64-bit field"); 47 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 48 "16-bit accessor invalid for 64-bit high field"); 49 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 50 "16-bit accessor invalid for 32-bit field"); 51 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 52 "16-bit accessor invalid for natural width field"); 53 } 54 55 static __always_inline void vmcs_check32(unsigned long field) 56 { 57 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 58 "32-bit accessor invalid for 16-bit field"); 59 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 60 "32-bit accessor invalid for 64-bit field"); 61 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 62 "32-bit accessor invalid for 64-bit high field"); 63 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 64 "32-bit accessor invalid for natural width field"); 65 } 66 67 static __always_inline void vmcs_check64(unsigned long field) 68 { 69 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 70 "64-bit accessor invalid for 16-bit field"); 71 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 72 "64-bit accessor invalid for 64-bit high field"); 73 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 74 "64-bit accessor invalid for 32-bit field"); 75 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, 76 "64-bit accessor invalid for natural width field"); 77 } 78 79 static __always_inline void vmcs_checkl(unsigned long field) 80 { 81 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, 82 "Natural width accessor invalid for 16-bit field"); 83 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, 84 "Natural width accessor invalid for 64-bit field"); 85 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, 86 "Natural width accessor invalid for 64-bit high field"); 87 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, 88 "Natural width accessor invalid for 32-bit field"); 89 } 90 91 static __always_inline unsigned long __vmcs_readl(unsigned long field) 92 { 93 unsigned long value; 94 95 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 96 97 asm_goto_output("1: vmread %[field], %[output]\n\t" 98 "jna %l[do_fail]\n\t" 99 100 _ASM_EXTABLE(1b, %l[do_exception]) 101 102 : [output] "=r" (value) 103 : [field] "r" (field) 104 : "cc" 105 : do_fail, do_exception); 106 107 return value; 108 109 do_fail: 110 instrumentation_begin(); 111 vmread_error(field); 112 instrumentation_end(); 113 return 0; 114 115 do_exception: 116 kvm_spurious_fault(); 117 return 0; 118 119 #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 120 121 asm volatile("1: vmread %[field], %[output]\n\t" 122 "ja 3f\n\t" 123 124 /* 125 * VMREAD failed. Push '0' for @fault, push the failing 126 * @field, and bounce through the trampoline to preserve 127 * volatile registers. 128 */ 129 "xorl %k[output], %k[output]\n\t" 130 "2:\n\t" 131 "push %[output]\n\t" 132 "push %[field]\n\t" 133 "call vmread_error_trampoline\n\t" 134 135 /* 136 * Unwind the stack. Note, the trampoline zeros out the 137 * memory for @fault so that the result is '0' on error. 138 */ 139 "pop %[field]\n\t" 140 "pop %[output]\n\t" 141 "3:\n\t" 142 143 /* VMREAD faulted. As above, except push '1' for @fault. */ 144 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %[output]) 145 146 : ASM_CALL_CONSTRAINT, [output] "=&r" (value) 147 : [field] "r" (field) 148 : "cc"); 149 return value; 150 151 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 152 } 153 154 static __always_inline u16 vmcs_read16(unsigned long field) 155 { 156 vmcs_check16(field); 157 if (kvm_is_using_evmcs()) 158 return evmcs_read16(field); 159 return __vmcs_readl(field); 160 } 161 162 static __always_inline u32 vmcs_read32(unsigned long field) 163 { 164 vmcs_check32(field); 165 if (kvm_is_using_evmcs()) 166 return evmcs_read32(field); 167 return __vmcs_readl(field); 168 } 169 170 static __always_inline u64 vmcs_read64(unsigned long field) 171 { 172 vmcs_check64(field); 173 if (kvm_is_using_evmcs()) 174 return evmcs_read64(field); 175 #ifdef CONFIG_X86_64 176 return __vmcs_readl(field); 177 #else 178 return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32); 179 #endif 180 } 181 182 static __always_inline unsigned long vmcs_readl(unsigned long field) 183 { 184 vmcs_checkl(field); 185 if (kvm_is_using_evmcs()) 186 return evmcs_read64(field); 187 return __vmcs_readl(field); 188 } 189 190 #define vmx_asm1(insn, op1, error_args...) \ 191 do { \ 192 asm goto("1: " __stringify(insn) " %0\n\t" \ 193 "jna %l[error]\n\t" \ 194 _ASM_EXTABLE(1b, %l[fault]) \ 195 : : op1 : "cc" : error, fault); \ 196 return; \ 197 error: \ 198 instrumentation_begin(); \ 199 insn##_error(error_args); \ 200 instrumentation_end(); \ 201 return; \ 202 fault: \ 203 kvm_spurious_fault(); \ 204 } while (0) 205 206 #define vmx_asm2(insn, op1, op2, error_args...) \ 207 do { \ 208 asm goto("1: " __stringify(insn) " %1, %0\n\t" \ 209 "jna %l[error]\n\t" \ 210 _ASM_EXTABLE(1b, %l[fault]) \ 211 : : op1, op2 : "cc" : error, fault); \ 212 return; \ 213 error: \ 214 instrumentation_begin(); \ 215 insn##_error(error_args); \ 216 instrumentation_end(); \ 217 return; \ 218 fault: \ 219 kvm_spurious_fault(); \ 220 } while (0) 221 222 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value) 223 { 224 vmx_asm2(vmwrite, "r" (field), ASM_INPUT_RM (value), field, value); 225 } 226 227 static __always_inline void vmcs_write16(unsigned long field, u16 value) 228 { 229 vmcs_check16(field); 230 if (kvm_is_using_evmcs()) 231 return evmcs_write16(field, value); 232 233 __vmcs_writel(field, value); 234 } 235 236 static __always_inline void vmcs_write32(unsigned long field, u32 value) 237 { 238 vmcs_check32(field); 239 if (kvm_is_using_evmcs()) 240 return evmcs_write32(field, value); 241 242 __vmcs_writel(field, value); 243 } 244 245 static __always_inline void vmcs_write64(unsigned long field, u64 value) 246 { 247 vmcs_check64(field); 248 if (kvm_is_using_evmcs()) 249 return evmcs_write64(field, value); 250 251 __vmcs_writel(field, value); 252 #ifndef CONFIG_X86_64 253 __vmcs_writel(field+1, value >> 32); 254 #endif 255 } 256 257 static __always_inline void vmcs_writel(unsigned long field, unsigned long value) 258 { 259 vmcs_checkl(field); 260 if (kvm_is_using_evmcs()) 261 return evmcs_write64(field, value); 262 263 __vmcs_writel(field, value); 264 } 265 266 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask) 267 { 268 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, 269 "vmcs_clear_bits does not support 64-bit fields"); 270 if (kvm_is_using_evmcs()) 271 return evmcs_write32(field, evmcs_read32(field) & ~mask); 272 273 __vmcs_writel(field, __vmcs_readl(field) & ~mask); 274 } 275 276 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask) 277 { 278 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, 279 "vmcs_set_bits does not support 64-bit fields"); 280 if (kvm_is_using_evmcs()) 281 return evmcs_write32(field, evmcs_read32(field) | mask); 282 283 __vmcs_writel(field, __vmcs_readl(field) | mask); 284 } 285 286 static inline void vmcs_clear(struct vmcs *vmcs) 287 { 288 u64 phys_addr = __pa(vmcs); 289 290 vmx_asm1(vmclear, "m"(phys_addr), vmcs, phys_addr); 291 } 292 293 static inline void vmcs_load(struct vmcs *vmcs) 294 { 295 u64 phys_addr = __pa(vmcs); 296 297 if (kvm_is_using_evmcs()) 298 return evmcs_load(phys_addr); 299 300 vmx_asm1(vmptrld, "m"(phys_addr), vmcs, phys_addr); 301 } 302 303 static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva) 304 { 305 struct { 306 u64 vpid : 16; 307 u64 rsvd : 48; 308 u64 gva; 309 } operand = { vpid, 0, gva }; 310 311 vmx_asm2(invvpid, "r"(ext), "m"(operand), ext, vpid, gva); 312 } 313 314 static inline void __invept(unsigned long ext, u64 eptp) 315 { 316 struct { 317 u64 eptp; 318 u64 reserved_0; 319 } operand = { eptp, 0 }; 320 vmx_asm2(invept, "r"(ext), "m"(operand), ext, eptp); 321 } 322 323 static inline void vpid_sync_vcpu_single(int vpid) 324 { 325 if (vpid == 0) 326 return; 327 328 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0); 329 } 330 331 static inline void vpid_sync_vcpu_global(void) 332 { 333 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0); 334 } 335 336 static inline void vpid_sync_context(int vpid) 337 { 338 if (cpu_has_vmx_invvpid_single()) 339 vpid_sync_vcpu_single(vpid); 340 else if (vpid != 0) 341 vpid_sync_vcpu_global(); 342 } 343 344 static inline void vpid_sync_vcpu_addr(int vpid, gva_t addr) 345 { 346 if (vpid == 0) 347 return; 348 349 if (cpu_has_vmx_invvpid_individual_addr()) 350 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr); 351 else 352 vpid_sync_context(vpid); 353 } 354 355 static inline void ept_sync_global(void) 356 { 357 __invept(VMX_EPT_EXTENT_GLOBAL, 0); 358 } 359 360 static inline void ept_sync_context(u64 eptp) 361 { 362 if (cpu_has_vmx_invept_context()) 363 __invept(VMX_EPT_EXTENT_CONTEXT, eptp); 364 else 365 ept_sync_global(); 366 } 367 368 #endif /* __KVM_X86_VMX_INSN_H */ 369