1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __KVM_X86_VMX_INSN_H
3 #define __KVM_X86_VMX_INSN_H
4
5 #include <linux/nospec.h>
6
7 #include <asm/vmx.h>
8
9 #include "vmx_onhyperv.h"
10 #include "vmcs.h"
11 #include "../x86.h"
12
13 void vmread_error(unsigned long field);
14 void vmwrite_error(unsigned long field, unsigned long value);
15 void vmclear_error(struct vmcs *vmcs, u64 phys_addr);
16 void vmptrld_error(struct vmcs *vmcs, u64 phys_addr);
17 void invvpid_error(unsigned long ext, u16 vpid, gva_t gva);
18 void invept_error(unsigned long ext, u64 eptp);
19
20 #ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
21 /*
22 * The VMREAD error trampoline _always_ uses the stack to pass parameters, even
23 * for 64-bit targets. Preserving all registers allows the VMREAD inline asm
24 * blob to avoid clobbering GPRs, which in turn allows the compiler to better
25 * optimize sequences of VMREADs.
26 *
27 * Declare the trampoline as an opaque label as it's not safe to call from C
28 * code; there is no way to tell the compiler to pass params on the stack for
29 * 64-bit targets.
30 *
31 * void vmread_error_trampoline(unsigned long field, bool fault);
32 */
33 extern unsigned long vmread_error_trampoline;
34
35 /*
36 * The second VMREAD error trampoline, called from the assembly trampoline,
37 * exists primarily to enable instrumentation for the VM-Fail path.
38 */
39 void vmread_error_trampoline2(unsigned long field, bool fault);
40
41 #endif
42
vmcs_check16(unsigned long field)43 static __always_inline void vmcs_check16(unsigned long field)
44 {
45 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
46 "16-bit accessor invalid for 64-bit field");
47 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
48 "16-bit accessor invalid for 64-bit high field");
49 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
50 "16-bit accessor invalid for 32-bit field");
51 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
52 "16-bit accessor invalid for natural width field");
53 }
54
vmcs_check32(unsigned long field)55 static __always_inline void vmcs_check32(unsigned long field)
56 {
57 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
58 "32-bit accessor invalid for 16-bit field");
59 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
60 "32-bit accessor invalid for 64-bit field");
61 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
62 "32-bit accessor invalid for 64-bit high field");
63 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
64 "32-bit accessor invalid for natural width field");
65 }
66
vmcs_check64(unsigned long field)67 static __always_inline void vmcs_check64(unsigned long field)
68 {
69 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
70 "64-bit accessor invalid for 16-bit field");
71 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
72 "64-bit accessor invalid for 64-bit high field");
73 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
74 "64-bit accessor invalid for 32-bit field");
75 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
76 "64-bit accessor invalid for natural width field");
77 }
78
vmcs_checkl(unsigned long field)79 static __always_inline void vmcs_checkl(unsigned long field)
80 {
81 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
82 "Natural width accessor invalid for 16-bit field");
83 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
84 "Natural width accessor invalid for 64-bit field");
85 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
86 "Natural width accessor invalid for 64-bit high field");
87 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
88 "Natural width accessor invalid for 32-bit field");
89 }
90
__vmcs_readl(unsigned long field)91 static __always_inline unsigned long __vmcs_readl(unsigned long field)
92 {
93 unsigned long value;
94
95 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
96
97 asm_goto_output("1: vmread %[field], %[output]\n\t"
98 "jna %l[do_fail]\n\t"
99
100 _ASM_EXTABLE(1b, %l[do_exception])
101
102 : [output] "=r" (value)
103 : [field] "r" (field)
104 : "cc", "memory"
105 : do_fail, do_exception);
106
107 return value;
108
109 do_fail:
110 instrumentation_begin();
111 vmread_error(field);
112 instrumentation_end();
113 return 0;
114
115 do_exception:
116 kvm_spurious_fault();
117 return 0;
118
119 #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
120
121 asm volatile("1: vmread %[field], %[output]\n\t"
122 "ja 3f\n\t"
123
124 /*
125 * VMREAD failed. Push '0' for @fault, push the failing
126 * @field, and bounce through the trampoline to preserve
127 * volatile registers.
128 */
129 "xorl %k[output], %k[output]\n\t"
130 "2:\n\t"
131 "push %[output]\n\t"
132 "push %[field]\n\t"
133 "call vmread_error_trampoline\n\t"
134
135 /*
136 * Unwind the stack. Note, the trampoline zeros out the
137 * memory for @fault so that the result is '0' on error.
138 */
139 "pop %[field]\n\t"
140 "pop %[output]\n\t"
141 "3:\n\t"
142
143 /* VMREAD faulted. As above, except push '1' for @fault. */
144 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %[output])
145
146 : ASM_CALL_CONSTRAINT, [output] "=&r" (value)
147 : [field] "r" (field)
148 : "cc", "memory");
149 return value;
150
151 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
152 }
153
vmcs_read16(unsigned long field)154 static __always_inline u16 vmcs_read16(unsigned long field)
155 {
156 vmcs_check16(field);
157 if (kvm_is_using_evmcs())
158 return evmcs_read16(field);
159 return __vmcs_readl(field);
160 }
161
vmcs_read32(unsigned long field)162 static __always_inline u32 vmcs_read32(unsigned long field)
163 {
164 vmcs_check32(field);
165 if (kvm_is_using_evmcs())
166 return evmcs_read32(field);
167 return __vmcs_readl(field);
168 }
169
vmcs_read64(unsigned long field)170 static __always_inline u64 vmcs_read64(unsigned long field)
171 {
172 vmcs_check64(field);
173 if (kvm_is_using_evmcs())
174 return evmcs_read64(field);
175 #ifdef CONFIG_X86_64
176 return __vmcs_readl(field);
177 #else
178 return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
179 #endif
180 }
181
vmcs_readl(unsigned long field)182 static __always_inline unsigned long vmcs_readl(unsigned long field)
183 {
184 vmcs_checkl(field);
185 if (kvm_is_using_evmcs())
186 return evmcs_read64(field);
187 return __vmcs_readl(field);
188 }
189
190 #define vmx_asm1(insn, op1, error_args...) \
191 do { \
192 asm goto("1: " __stringify(insn) " %0\n\t" \
193 "jna %l[error]\n\t" \
194 _ASM_EXTABLE(1b, %l[fault]) \
195 : : op1 : "cc", "memory" : error, fault); \
196 return; \
197 error: \
198 instrumentation_begin(); \
199 insn##_error(error_args); \
200 instrumentation_end(); \
201 return; \
202 fault: \
203 kvm_spurious_fault(); \
204 } while (0)
205
206 #define vmx_asm2(insn, op1, op2, error_args...) \
207 do { \
208 asm goto("1: " __stringify(insn) " %1, %0\n\t" \
209 "jna %l[error]\n\t" \
210 _ASM_EXTABLE(1b, %l[fault]) \
211 : : op1, op2 : "cc", "memory" : error, fault);\
212 return; \
213 error: \
214 instrumentation_begin(); \
215 insn##_error(error_args); \
216 instrumentation_end(); \
217 return; \
218 fault: \
219 kvm_spurious_fault(); \
220 } while (0)
221
__vmcs_writel(unsigned long field,unsigned long value)222 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
223 {
224 vmx_asm2(vmwrite, "r" (field), ASM_INPUT_RM (value), field, value);
225 }
226
vmcs_write16(unsigned long field,u16 value)227 static __always_inline void vmcs_write16(unsigned long field, u16 value)
228 {
229 vmcs_check16(field);
230 if (kvm_is_using_evmcs())
231 return evmcs_write16(field, value);
232
233 __vmcs_writel(field, value);
234 }
235
vmcs_write32(unsigned long field,u32 value)236 static __always_inline void vmcs_write32(unsigned long field, u32 value)
237 {
238 vmcs_check32(field);
239 if (kvm_is_using_evmcs())
240 return evmcs_write32(field, value);
241
242 __vmcs_writel(field, value);
243 }
244
vmcs_write64(unsigned long field,u64 value)245 static __always_inline void vmcs_write64(unsigned long field, u64 value)
246 {
247 vmcs_check64(field);
248 if (kvm_is_using_evmcs())
249 return evmcs_write64(field, value);
250
251 __vmcs_writel(field, value);
252 #ifndef CONFIG_X86_64
253 __vmcs_writel(field+1, value >> 32);
254 #endif
255 }
256
vmcs_writel(unsigned long field,unsigned long value)257 static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
258 {
259 vmcs_checkl(field);
260 if (kvm_is_using_evmcs())
261 return evmcs_write64(field, value);
262
263 __vmcs_writel(field, value);
264 }
265
vmcs_clear_bits(unsigned long field,u32 mask)266 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
267 {
268 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
269 "vmcs_clear_bits does not support 64-bit fields");
270 if (kvm_is_using_evmcs())
271 return evmcs_write32(field, evmcs_read32(field) & ~mask);
272
273 __vmcs_writel(field, __vmcs_readl(field) & ~mask);
274 }
275
vmcs_set_bits(unsigned long field,u32 mask)276 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
277 {
278 BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
279 "vmcs_set_bits does not support 64-bit fields");
280 if (kvm_is_using_evmcs())
281 return evmcs_write32(field, evmcs_read32(field) | mask);
282
283 __vmcs_writel(field, __vmcs_readl(field) | mask);
284 }
285
vmcs_clear(struct vmcs * vmcs)286 static inline void vmcs_clear(struct vmcs *vmcs)
287 {
288 u64 phys_addr = __pa(vmcs);
289
290 vmx_asm1(vmclear, "m"(phys_addr), vmcs, phys_addr);
291 }
292
vmcs_load(struct vmcs * vmcs)293 static inline void vmcs_load(struct vmcs *vmcs)
294 {
295 u64 phys_addr = __pa(vmcs);
296
297 if (kvm_is_using_evmcs())
298 return evmcs_load(phys_addr);
299
300 vmx_asm1(vmptrld, "m"(phys_addr), vmcs, phys_addr);
301 }
302
__invvpid(unsigned long ext,u16 vpid,gva_t gva)303 static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva)
304 {
305 struct {
306 u64 vpid : 16;
307 u64 rsvd : 48;
308 u64 gva;
309 } operand = { vpid, 0, gva };
310
311 vmx_asm2(invvpid, "r"(ext), "m"(operand), ext, vpid, gva);
312 }
313
__invept(unsigned long ext,u64 eptp)314 static inline void __invept(unsigned long ext, u64 eptp)
315 {
316 struct {
317 u64 eptp;
318 u64 reserved_0;
319 } operand = { eptp, 0 };
320 vmx_asm2(invept, "r"(ext), "m"(operand), ext, eptp);
321 }
322
vpid_sync_vcpu_single(int vpid)323 static inline void vpid_sync_vcpu_single(int vpid)
324 {
325 if (vpid == 0)
326 return;
327
328 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
329 }
330
vpid_sync_vcpu_global(void)331 static inline void vpid_sync_vcpu_global(void)
332 {
333 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
334 }
335
vpid_sync_context(int vpid)336 static inline void vpid_sync_context(int vpid)
337 {
338 if (cpu_has_vmx_invvpid_single())
339 vpid_sync_vcpu_single(vpid);
340 else if (vpid != 0)
341 vpid_sync_vcpu_global();
342 }
343
vpid_sync_vcpu_addr(int vpid,gva_t addr)344 static inline void vpid_sync_vcpu_addr(int vpid, gva_t addr)
345 {
346 if (vpid == 0)
347 return;
348
349 if (cpu_has_vmx_invvpid_individual_addr())
350 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr);
351 else
352 vpid_sync_context(vpid);
353 }
354
ept_sync_global(void)355 static inline void ept_sync_global(void)
356 {
357 __invept(VMX_EPT_EXTENT_GLOBAL, 0);
358 }
359
ept_sync_context(u64 eptp)360 static inline void ept_sync_context(u64 eptp)
361 {
362 if (cpu_has_vmx_invept_context())
363 __invept(VMX_EPT_EXTENT_CONTEXT, eptp);
364 else
365 ept_sync_global();
366 }
367
368 #endif /* __KVM_X86_VMX_INSN_H */
369