1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KVM_GET/SET_* tests 4 * 5 * Copyright (C) 2018, Red Hat, Inc. 6 * 7 * Tests for vCPU state save/restore, including nested guest state. 8 */ 9 #include <fcntl.h> 10 #include <stdio.h> 11 #include <stdlib.h> 12 #include <string.h> 13 #include <sys/ioctl.h> 14 15 #include "test_util.h" 16 17 #include "kvm_util.h" 18 #include "processor.h" 19 #include "vmx.h" 20 #include "svm_util.h" 21 22 void svm_l2_guest_code(void) 23 { 24 GUEST_SYNC(4); 25 /* Exit to L1 */ 26 vmcall(); 27 clgi(); 28 GUEST_SYNC(6); 29 stgi(); 30 /* Done, exit to L1 and never come back. */ 31 vmcall(); 32 } 33 34 static void svm_l1_guest_code(struct svm_test_data *svm) 35 { 36 struct vmcb *vmcb = svm->vmcb; 37 38 GUEST_ASSERT(svm->vmcb_gpa); 39 /* Prepare for L2 execution. */ 40 generic_svm_setup(svm, svm_l2_guest_code); 41 42 vmcb->control.int_ctl |= (V_GIF_ENABLE_MASK | V_GIF_MASK); 43 44 GUEST_SYNC(3); 45 run_guest(vmcb, svm->vmcb_gpa); 46 GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_VMMCALL); 47 GUEST_SYNC(5); 48 vmcb->save.rip += 3; 49 run_guest(vmcb, svm->vmcb_gpa); 50 GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_VMMCALL); 51 GUEST_SYNC(7); 52 } 53 54 void vmx_l2_guest_code(void) 55 { 56 GUEST_SYNC(6); 57 58 /* Exit to L1 */ 59 vmcall(); 60 61 /* L1 has now set up a shadow VMCS for us. */ 62 GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); 63 GUEST_SYNC(10); 64 GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); 65 GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0fffee)); 66 GUEST_SYNC(11); 67 GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0fffee); 68 GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0ffffee)); 69 GUEST_SYNC(12); 70 71 /* Done, exit to L1 and never come back. */ 72 vmcall(); 73 } 74 75 static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) 76 { 77 GUEST_ASSERT(vmx_pages->vmcs_gpa); 78 GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); 79 GUEST_SYNC(3); 80 GUEST_ASSERT(load_vmcs(vmx_pages)); 81 GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); 82 83 GUEST_SYNC(4); 84 GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); 85 86 prepare_vmcs(vmx_pages, vmx_l2_guest_code); 87 88 GUEST_SYNC(5); 89 GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); 90 GUEST_ASSERT(!vmlaunch()); 91 GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); 92 GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); 93 94 /* Check that the launched state is preserved. */ 95 GUEST_ASSERT(vmlaunch()); 96 97 GUEST_ASSERT(!vmresume()); 98 GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); 99 100 GUEST_SYNC(7); 101 GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); 102 103 GUEST_ASSERT(!vmresume()); 104 GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); 105 106 vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + 3); 107 108 vmwrite(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); 109 vmwrite(VMCS_LINK_POINTER, vmx_pages->shadow_vmcs_gpa); 110 111 GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa)); 112 GUEST_ASSERT(vmlaunch()); 113 GUEST_SYNC(8); 114 GUEST_ASSERT(vmlaunch()); 115 GUEST_ASSERT(vmresume()); 116 117 vmwrite(GUEST_RIP, 0xc0ffee); 118 GUEST_SYNC(9); 119 GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); 120 121 GUEST_ASSERT(!vmptrld(vmx_pages->vmcs_gpa)); 122 GUEST_ASSERT(!vmresume()); 123 GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); 124 125 GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa)); 126 GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); 127 GUEST_ASSERT(vmlaunch()); 128 GUEST_ASSERT(vmresume()); 129 GUEST_SYNC(13); 130 GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); 131 GUEST_ASSERT(vmlaunch()); 132 GUEST_ASSERT(vmresume()); 133 } 134 135 static void __attribute__((__flatten__)) guest_code(void *arg) 136 { 137 GUEST_SYNC(1); 138 139 if (this_cpu_has(X86_FEATURE_XSAVE)) { 140 u64 supported_xcr0 = this_cpu_supported_xcr0(); 141 u8 buffer[PAGE_SIZE]; 142 143 memset(buffer, 0xcc, sizeof(buffer)); 144 145 /* 146 * Modify state for all supported xfeatures to take them out of 147 * their "init" state, i.e. to make them show up in XSTATE_BV. 148 * 149 * Note off-by-default features, e.g. AMX, are out of scope for 150 * this particular testcase as they have a different ABI. 151 */ 152 GUEST_ASSERT(supported_xcr0 & XFEATURE_MASK_FP); 153 asm volatile ("fincstp"); 154 155 GUEST_ASSERT(supported_xcr0 & XFEATURE_MASK_SSE); 156 asm volatile ("vmovdqu %0, %%xmm0" :: "m" (buffer)); 157 158 if (supported_xcr0 & XFEATURE_MASK_YMM) 159 asm volatile ("vmovdqu %0, %%ymm0" :: "m" (buffer)); 160 161 if (supported_xcr0 & XFEATURE_MASK_AVX512) { 162 asm volatile ("kmovq %0, %%k1" :: "r" (-1ull)); 163 asm volatile ("vmovupd %0, %%zmm0" :: "m" (buffer)); 164 asm volatile ("vmovupd %0, %%zmm16" :: "m" (buffer)); 165 } 166 167 if (this_cpu_has(X86_FEATURE_MPX)) { 168 u64 bounds[2] = { 10, 0xffffffffull }; 169 u64 output[2] = { }; 170 171 GUEST_ASSERT(supported_xcr0 & XFEATURE_MASK_BNDREGS); 172 GUEST_ASSERT(supported_xcr0 & XFEATURE_MASK_BNDCSR); 173 174 /* 175 * Don't bother trying to get BNDCSR into the INUSE 176 * state. MSR_IA32_BNDCFGS doesn't count as it isn't 177 * managed via XSAVE/XRSTOR, and BNDCFGU can only be 178 * modified by XRSTOR. Stuffing XSTATE_BV in the host 179 * is simpler than doing XRSTOR here in the guest. 180 * 181 * However, temporarily enable MPX in BNDCFGS so that 182 * BNDMOV actually loads BND1. If MPX isn't *fully* 183 * enabled, all MPX instructions are treated as NOPs. 184 * 185 * Hand encode "bndmov (%rax),%bnd1" as support for MPX 186 * mnemonics/registers has been removed from gcc and 187 * clang (and was never fully supported by clang). 188 */ 189 wrmsr(MSR_IA32_BNDCFGS, BIT_ULL(0)); 190 asm volatile (".byte 0x66,0x0f,0x1a,0x08" :: "a" (bounds)); 191 /* 192 * Hand encode "bndmov %bnd1, (%rax)" to sanity check 193 * that BND1 actually got loaded. 194 */ 195 asm volatile (".byte 0x66,0x0f,0x1b,0x08" :: "a" (output)); 196 wrmsr(MSR_IA32_BNDCFGS, 0); 197 198 GUEST_ASSERT_EQ(bounds[0], output[0]); 199 GUEST_ASSERT_EQ(bounds[1], output[1]); 200 } 201 if (this_cpu_has(X86_FEATURE_PKU)) { 202 GUEST_ASSERT(supported_xcr0 & XFEATURE_MASK_PKRU); 203 set_cr4(get_cr4() | X86_CR4_PKE); 204 GUEST_ASSERT(this_cpu_has(X86_FEATURE_OSPKE)); 205 206 wrpkru(-1u); 207 } 208 } 209 210 GUEST_SYNC(2); 211 212 if (arg) { 213 if (this_cpu_has(X86_FEATURE_SVM)) 214 svm_l1_guest_code(arg); 215 else 216 vmx_l1_guest_code(arg); 217 } 218 219 GUEST_DONE(); 220 } 221 222 void svm_check_nested_state(int stage, struct kvm_x86_state *state) 223 { 224 struct vmcb *vmcb = (struct vmcb *)state->nested.data.svm; 225 226 if (kvm_cpu_has(X86_FEATURE_VGIF)) { 227 if (stage == 4) 228 TEST_ASSERT_EQ(!!(vmcb->control.int_ctl & V_GIF_MASK), 1); 229 if (stage == 6) 230 TEST_ASSERT_EQ(!!(vmcb->control.int_ctl & V_GIF_MASK), 0); 231 } 232 233 if (kvm_cpu_has(X86_FEATURE_NRIPS)) { 234 /* 235 * GUEST_SYNC() causes IO emulation in KVM, in which case the 236 * RIP is advanced before exiting to userspace. Hence, the RIP 237 * in the saved state should be the same as nRIP saved by the 238 * CPU in the VMCB. 239 */ 240 if (stage == 6) 241 TEST_ASSERT_EQ(vmcb->control.next_rip, state->regs.rip); 242 } 243 } 244 245 void check_nested_state(int stage, struct kvm_x86_state *state) 246 { 247 if (kvm_has_cap(KVM_CAP_NESTED_STATE) && kvm_cpu_has(X86_FEATURE_SVM)) 248 svm_check_nested_state(stage, state); 249 } 250 251 int main(int argc, char *argv[]) 252 { 253 u64 *xstate_bv, saved_xstate_bv; 254 gva_t nested_gva = 0; 255 struct kvm_cpuid2 empty_cpuid = {}; 256 struct kvm_regs regs1, regs2; 257 struct kvm_vcpu *vcpu, *vcpuN; 258 struct kvm_vm *vm; 259 struct kvm_x86_state *state; 260 struct ucall uc; 261 int stage; 262 263 /* Create VM */ 264 vm = vm_create_with_one_vcpu(&vcpu, guest_code); 265 266 vcpu_regs_get(vcpu, ®s1); 267 268 if (kvm_has_cap(KVM_CAP_NESTED_STATE)) { 269 if (kvm_cpu_has(X86_FEATURE_SVM)) 270 vcpu_alloc_svm(vm, &nested_gva); 271 else if (kvm_cpu_has(X86_FEATURE_VMX)) 272 vcpu_alloc_vmx(vm, &nested_gva); 273 } 274 275 if (!nested_gva) 276 pr_info("will skip nested state checks\n"); 277 278 vcpu_args_set(vcpu, 1, nested_gva); 279 280 for (stage = 1;; stage++) { 281 vcpu_run(vcpu); 282 TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); 283 284 switch (get_ucall(vcpu, &uc)) { 285 case UCALL_ABORT: 286 REPORT_GUEST_ASSERT(uc); 287 /* NOT REACHED */ 288 case UCALL_SYNC: 289 break; 290 case UCALL_DONE: 291 goto done; 292 default: 293 TEST_FAIL("Unknown ucall %lu", uc.cmd); 294 } 295 296 /* UCALL_SYNC is handled here. */ 297 TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && 298 uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", 299 stage, (ulong)uc.args[1]); 300 301 state = vcpu_save_state(vcpu); 302 memset(®s1, 0, sizeof(regs1)); 303 vcpu_regs_get(vcpu, ®s1); 304 305 kvm_vm_release(vm); 306 307 check_nested_state(stage, state); 308 309 /* Restore state in a new VM. */ 310 vcpu = vm_recreate_with_one_vcpu(vm); 311 vcpu_load_state(vcpu, state); 312 313 /* 314 * Restore XSAVE state in a dummy vCPU, first without doing 315 * KVM_SET_CPUID2, and then with an empty guest CPUID. Except 316 * for off-by-default xfeatures, e.g. AMX, KVM is supposed to 317 * allow KVM_SET_XSAVE regardless of guest CPUID. Manually 318 * load only XSAVE state, MSRs in particular have a much more 319 * convoluted ABI. 320 * 321 * Load two versions of XSAVE state: one with the actual guest 322 * XSAVE state, and one with all supported features forced "on" 323 * in xstate_bv, e.g. to ensure that KVM allows loading all 324 * supported features, even if something goes awry in saving 325 * the original snapshot. 326 */ 327 xstate_bv = (void *)&((u8 *)state->xsave->region)[512]; 328 saved_xstate_bv = *xstate_bv; 329 330 vcpuN = __vm_vcpu_add(vm, vcpu->id + 1); 331 vcpu_xsave_set(vcpuN, state->xsave); 332 *xstate_bv = kvm_cpu_supported_xcr0(); 333 vcpu_xsave_set(vcpuN, state->xsave); 334 335 vcpu_init_cpuid(vcpuN, &empty_cpuid); 336 vcpu_xsave_set(vcpuN, state->xsave); 337 *xstate_bv = saved_xstate_bv; 338 vcpu_xsave_set(vcpuN, state->xsave); 339 340 kvm_x86_state_cleanup(state); 341 342 memset(®s2, 0, sizeof(regs2)); 343 vcpu_regs_get(vcpu, ®s2); 344 TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), 345 "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", 346 (ulong) regs2.rdi, (ulong) regs2.rsi); 347 } 348 349 done: 350 kvm_vm_free(vm); 351 } 352