1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kvm_host.h> 3 4 #include "lapic.h" 5 #include "mmu.h" 6 #include "regs.h" 7 #include "x86.h" 8 9 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) 10 { 11 /* Can't read the RIP when guest state is protected, just return 0 */ 12 if (vcpu->arch.guest_state_protected) 13 return 0; 14 15 if (is_64_bit_mode(vcpu)) 16 return kvm_rip_read(vcpu); 17 return (u32)(kvm_get_segment_base(vcpu, VCPU_SREG_CS) + 18 kvm_rip_read(vcpu)); 19 } 20 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_linear_rip); 21 22 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip) 23 { 24 return kvm_get_linear_rip(vcpu) == linear_rip; 25 } 26 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_is_linear_rip); 27 28 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) 29 { 30 unsigned long rflags; 31 32 rflags = kvm_x86_call(get_rflags)(vcpu); 33 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 34 rflags &= ~X86_EFLAGS_TF; 35 return rflags; 36 } 37 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_rflags); 38 39 void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 40 { 41 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && 42 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) 43 rflags |= X86_EFLAGS_TF; 44 kvm_x86_call(set_rflags)(vcpu, rflags); 45 } 46 47 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 48 { 49 __kvm_set_rflags(vcpu, rflags); 50 kvm_make_request(KVM_REQ_EVENT, vcpu); 51 } 52 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_rflags); 53 54 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 55 { 56 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { 57 /* 58 * We are here if userspace calls get_regs() in the middle of 59 * instruction emulation. Registers state needs to be copied 60 * back from emulation context to vcpu. Userspace shouldn't do 61 * that usually, but some bad designed PV devices (vmware 62 * backdoor interface) need this to work 63 */ 64 emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); 65 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 66 } 67 regs->rax = kvm_rax_read_raw(vcpu); 68 regs->rbx = kvm_rbx_read_raw(vcpu); 69 regs->rcx = kvm_rcx_read_raw(vcpu); 70 regs->rdx = kvm_rdx_read_raw(vcpu); 71 regs->rsi = kvm_rsi_read_raw(vcpu); 72 regs->rdi = kvm_rdi_read_raw(vcpu); 73 regs->rsp = kvm_rsp_read(vcpu); 74 regs->rbp = kvm_rbp_read_raw(vcpu); 75 #ifdef CONFIG_X86_64 76 regs->r8 = kvm_r8_read_raw(vcpu); 77 regs->r9 = kvm_r9_read_raw(vcpu); 78 regs->r10 = kvm_r10_read_raw(vcpu); 79 regs->r11 = kvm_r11_read_raw(vcpu); 80 regs->r12 = kvm_r12_read_raw(vcpu); 81 regs->r13 = kvm_r13_read_raw(vcpu); 82 regs->r14 = kvm_r14_read_raw(vcpu); 83 regs->r15 = kvm_r15_read_raw(vcpu); 84 #endif 85 86 regs->rip = kvm_rip_read(vcpu); 87 regs->rflags = kvm_get_rflags(vcpu); 88 } 89 90 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 91 { 92 if (vcpu->kvm->arch.has_protected_state && 93 vcpu->arch.guest_state_protected) 94 return -EINVAL; 95 96 vcpu_load(vcpu); 97 __get_regs(vcpu, regs); 98 vcpu_put(vcpu); 99 return 0; 100 } 101 102 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 103 { 104 vcpu->arch.emulate_regs_need_sync_from_vcpu = true; 105 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 106 107 kvm_rax_write_raw(vcpu, regs->rax); 108 kvm_rbx_write_raw(vcpu, regs->rbx); 109 kvm_rcx_write_raw(vcpu, regs->rcx); 110 kvm_rdx_write_raw(vcpu, regs->rdx); 111 kvm_rsi_write_raw(vcpu, regs->rsi); 112 kvm_rdi_write_raw(vcpu, regs->rdi); 113 kvm_rsp_write(vcpu, regs->rsp); 114 kvm_rbp_write_raw(vcpu, regs->rbp); 115 #ifdef CONFIG_X86_64 116 kvm_r8_write_raw(vcpu, regs->r8); 117 kvm_r9_write_raw(vcpu, regs->r9); 118 kvm_r10_write_raw(vcpu, regs->r10); 119 kvm_r11_write_raw(vcpu, regs->r11); 120 kvm_r12_write_raw(vcpu, regs->r12); 121 kvm_r13_write_raw(vcpu, regs->r13); 122 kvm_r14_write_raw(vcpu, regs->r14); 123 kvm_r15_write_raw(vcpu, regs->r15); 124 #endif 125 126 kvm_rip_write(vcpu, regs->rip); 127 kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); 128 129 vcpu->arch.exception.pending = false; 130 vcpu->arch.exception_vmexit.pending = false; 131 132 kvm_make_request(KVM_REQ_EVENT, vcpu); 133 } 134 135 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 136 { 137 if (vcpu->kvm->arch.has_protected_state && 138 vcpu->arch.guest_state_protected) 139 return -EINVAL; 140 141 vcpu_load(vcpu); 142 __set_regs(vcpu, regs); 143 vcpu_put(vcpu); 144 return 0; 145 } 146 147 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) 148 { 149 return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); 150 } 151 152 /* 153 * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise. 154 */ 155 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) 156 { 157 struct kvm_pagewalk *w = &vcpu->arch.gva_walk; 158 gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; 159 gpa_t real_gpa; 160 int i; 161 int ret; 162 u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; 163 164 /* 165 * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated 166 * to an L1 GPA. 167 */ 168 real_gpa = kvm_translate_gpa(vcpu, w, gfn_to_gpa(pdpt_gfn), 169 PFERR_USER_MASK | PFERR_WRITE_MASK | 170 PFERR_GUEST_PAGE_MASK, NULL, 0); 171 if (real_gpa == INVALID_GPA) 172 return 0; 173 174 /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */ 175 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, 176 cr3 & GENMASK(11, 5), sizeof(pdpte)); 177 if (ret < 0) 178 return 0; 179 180 for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { 181 if ((pdpte[i] & PT_PRESENT_MASK) && 182 (pdpte[i] & pdptr_rsvd_bits(vcpu))) { 183 return 0; 184 } 185 } 186 187 /* 188 * Marking VCPU_REG_PDPTR dirty doesn't work for !tdp_enabled. 189 * Shadow page roots need to be reconstructed instead. 190 */ 191 if (!tdp_enabled && memcmp(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs))) 192 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.root_mmu, 193 KVM_MMU_ROOT_CURRENT); 194 195 memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); 196 kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); 197 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); 198 vcpu->arch.pdptrs_from_userspace = false; 199 200 return 1; 201 } 202 EXPORT_SYMBOL_FOR_KVM_INTERNAL(load_pdptrs); 203 204 static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 205 { 206 #ifdef CONFIG_X86_64 207 if (cr0 & 0xffffffff00000000UL) 208 return false; 209 #endif 210 211 if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) 212 return false; 213 214 if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) 215 return false; 216 217 return kvm_x86_call(is_valid_cr0)(vcpu, cr0); 218 } 219 220 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0) 221 { 222 /* 223 * CR0.WP is incorporated into the MMU role, but only for non-nested, 224 * indirect shadow MMUs. If paging is disabled, no updates are needed 225 * as there are no permission bits to emulate. If TDP is enabled, the 226 * MMU's metadata needs to be updated, e.g. so that emulating guest 227 * translations does the right thing, but there's no need to unload the 228 * root as CR0.WP doesn't affect SPTEs. 229 */ 230 if ((cr0 ^ old_cr0) == X86_CR0_WP) { 231 if (!(cr0 & X86_CR0_PG)) 232 return; 233 234 if (tdp_enabled) { 235 kvm_init_mmu(vcpu); 236 return; 237 } 238 } 239 240 if ((cr0 ^ old_cr0) & X86_CR0_PG) { 241 /* 242 * Clearing CR0.PG is defined to flush the TLB from the guest's 243 * perspective. 244 */ 245 if (!(cr0 & X86_CR0_PG)) 246 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 247 /* 248 * Check for async #PF completion events when enabling paging, 249 * as the vCPU may have previously encountered async #PFs (it's 250 * entirely legal for the guest to toggle paging on/off without 251 * waiting for the async #PF queue to drain). 252 */ 253 else if (kvm_pv_async_pf_enabled(vcpu)) 254 kvm_make_request(KVM_REQ_APF_READY, vcpu); 255 } 256 257 if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS) 258 kvm_mmu_reset_context(vcpu); 259 } 260 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr0); 261 262 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 263 { 264 unsigned long old_cr0 = kvm_read_cr0(vcpu); 265 266 if (!kvm_is_valid_cr0(vcpu, cr0)) 267 return 1; 268 269 cr0 |= X86_CR0_ET; 270 271 /* Write to CR0 reserved bits are ignored, even on Intel. */ 272 cr0 &= ~CR0_RESERVED_BITS; 273 274 #ifdef CONFIG_X86_64 275 if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) && 276 (cr0 & X86_CR0_PG)) { 277 int cs_db, cs_l; 278 279 if (!is_pae(vcpu)) 280 return 1; 281 kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); 282 if (cs_l) 283 return 1; 284 } 285 #endif 286 if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && 287 is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && 288 !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) 289 return 1; 290 291 if (!(cr0 & X86_CR0_PG) && 292 (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))) 293 return 1; 294 295 if (!(cr0 & X86_CR0_WP) && kvm_is_cr4_bit_set(vcpu, X86_CR4_CET)) 296 return 1; 297 298 kvm_x86_call(set_cr0)(vcpu, cr0); 299 300 kvm_post_set_cr0(vcpu, old_cr0, cr0); 301 302 return 0; 303 } 304 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr0); 305 306 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) 307 { 308 (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); 309 } 310 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw); 311 312 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) 313 { 314 bool skip_tlb_flush = false; 315 unsigned long pcid = 0; 316 #ifdef CONFIG_X86_64 317 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) { 318 skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH; 319 cr3 &= ~X86_CR3_PCID_NOFLUSH; 320 pcid = cr3 & X86_CR3_PCID_MASK; 321 } 322 #endif 323 324 /* PDPTRs are always reloaded for PAE paging. */ 325 if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) 326 goto handle_tlb_flush; 327 328 /* 329 * Do not condition the GPA check on long mode, this helper is used to 330 * stuff CR3, e.g. for RSM emulation, and there is no guarantee that 331 * the current vCPU mode is accurate. 332 */ 333 if (!kvm_vcpu_is_legal_cr3(vcpu, cr3)) 334 return 1; 335 336 if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) 337 return 1; 338 339 if (cr3 != kvm_read_cr3(vcpu)) 340 kvm_mmu_new_pgd(vcpu, cr3); 341 342 vcpu->arch.cr3 = cr3; 343 kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); 344 /* Do not call post_set_cr3, we do not get here for confidential guests. */ 345 346 handle_tlb_flush: 347 /* 348 * A load of CR3 that flushes the TLB flushes only the current PCID, 349 * even if PCID is disabled, in which case PCID=0 is flushed. It's a 350 * moot point in the end because _disabling_ PCID will flush all PCIDs, 351 * and it's impossible to use a non-zero PCID when PCID is disabled, 352 * i.e. only PCID=0 can be relevant. 353 */ 354 if (!skip_tlb_flush) 355 kvm_invalidate_pcid(vcpu, pcid); 356 357 return 0; 358 } 359 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr3); 360 361 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 362 { 363 return __kvm_is_valid_cr4(vcpu, cr4) && 364 kvm_x86_call(is_valid_cr4)(vcpu, cr4); 365 } 366 367 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) 368 { 369 if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) 370 kvm_mmu_reset_context(vcpu); 371 372 /* 373 * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB 374 * according to the SDM; however, stale prev_roots could be reused 375 * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we 376 * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST 377 * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed, 378 * so fall through. 379 */ 380 if (!tdp_enabled && 381 (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) 382 kvm_mmu_unload(vcpu); 383 384 /* 385 * The TLB has to be flushed for all PCIDs if any of the following 386 * (architecturally required) changes happen: 387 * - CR4.PCIDE is changed from 1 to 0 388 * - CR4.PGE is toggled 389 * 390 * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT. 391 */ 392 if (((cr4 ^ old_cr4) & X86_CR4_PGE) || 393 (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) 394 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 395 396 /* 397 * The TLB has to be flushed for the current PCID if any of the 398 * following (architecturally required) changes happen: 399 * - CR4.SMEP is changed from 0 to 1 400 * - CR4.PAE is toggled 401 */ 402 else if (((cr4 ^ old_cr4) & X86_CR4_PAE) || 403 ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP))) 404 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 405 406 } 407 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr4); 408 409 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 410 { 411 unsigned long old_cr4 = kvm_read_cr4(vcpu); 412 413 if (!kvm_is_valid_cr4(vcpu, cr4)) 414 return 1; 415 416 if (is_long_mode(vcpu)) { 417 if (!(cr4 & X86_CR4_PAE)) 418 return 1; 419 if ((cr4 ^ old_cr4) & X86_CR4_LA57) 420 return 1; 421 } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) 422 && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS) 423 && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) 424 return 1; 425 426 if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { 427 /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ 428 if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) 429 return 1; 430 } 431 432 if ((cr4 & X86_CR4_CET) && !kvm_is_cr0_bit_set(vcpu, X86_CR0_WP)) 433 return 1; 434 435 kvm_x86_call(set_cr4)(vcpu, cr4); 436 437 kvm_post_set_cr4(vcpu, old_cr4, cr4); 438 439 return 0; 440 } 441 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); 442 443 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) 444 { 445 if (cr8 & CR8_RESERVED_BITS) 446 return 1; 447 if (lapic_in_kernel(vcpu)) 448 kvm_lapic_set_tpr(vcpu, cr8); 449 else 450 vcpu->arch.cr8 = cr8; 451 return 0; 452 } 453 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr8); 454 455 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) 456 { 457 if (lapic_in_kernel(vcpu)) 458 return kvm_lapic_get_cr8(vcpu); 459 else 460 return vcpu->arch.cr8; 461 } 462 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_cr8); 463 464 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 465 { 466 struct desc_ptr dt; 467 468 if (vcpu->arch.guest_state_protected) 469 goto skip_protected_regs; 470 471 kvm_handle_exception_payload_quirk(vcpu); 472 473 kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); 474 kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); 475 kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); 476 kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); 477 kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); 478 kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); 479 480 kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); 481 kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); 482 483 kvm_x86_call(get_idt)(vcpu, &dt); 484 sregs->idt.limit = dt.size; 485 sregs->idt.base = dt.address; 486 kvm_x86_call(get_gdt)(vcpu, &dt); 487 sregs->gdt.limit = dt.size; 488 sregs->gdt.base = dt.address; 489 490 sregs->cr2 = vcpu->arch.cr2; 491 sregs->cr3 = kvm_read_cr3(vcpu); 492 493 skip_protected_regs: 494 sregs->cr0 = kvm_read_cr0(vcpu); 495 sregs->cr4 = kvm_read_cr4(vcpu); 496 sregs->cr8 = kvm_get_cr8(vcpu); 497 sregs->efer = vcpu->arch.efer; 498 sregs->apic_base = vcpu->arch.apic_base; 499 } 500 501 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 502 { 503 __get_sregs_common(vcpu, sregs); 504 505 if (vcpu->arch.guest_state_protected) 506 return; 507 508 if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) 509 set_bit(vcpu->arch.interrupt.nr, 510 (unsigned long *)sregs->interrupt_bitmap); 511 } 512 513 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 514 struct kvm_sregs *sregs) 515 { 516 if (vcpu->kvm->arch.has_protected_state && 517 vcpu->arch.guest_state_protected) 518 return -EINVAL; 519 520 vcpu_load(vcpu); 521 __get_sregs(vcpu, sregs); 522 vcpu_put(vcpu); 523 return 0; 524 } 525 526 void kvm_vcpu_ioctl_x86_get_sregs2(struct kvm_vcpu *vcpu, 527 struct kvm_sregs2 *sregs2) 528 { 529 int i; 530 531 __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); 532 533 if (vcpu->arch.guest_state_protected) 534 return; 535 536 if (is_pae_paging(vcpu)) { 537 kvm_vcpu_srcu_read_lock(vcpu); 538 for (i = 0 ; i < 4 ; i++) 539 sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); 540 sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; 541 kvm_vcpu_srcu_read_unlock(vcpu); 542 } 543 } 544 545 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 546 { 547 if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) { 548 /* 549 * When EFER.LME and CR0.PG are set, the processor is in 550 * 64-bit mode (though maybe in a 32-bit code segment). 551 * CR4.PAE and EFER.LMA must be set. 552 */ 553 if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA)) 554 return false; 555 if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3)) 556 return false; 557 } else { 558 /* 559 * Not in 64-bit mode: EFER.LMA is clear and the code 560 * segment cannot be 64-bit. 561 */ 562 if (sregs->efer & EFER_LMA || sregs->cs.l) 563 return false; 564 } 565 566 return kvm_is_valid_cr4(vcpu, sregs->cr4) && 567 kvm_is_valid_cr0(vcpu, sregs->cr0) && 568 kvm_valid_efer(vcpu, sregs->efer); 569 } 570 571 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, 572 int *mmu_reset_needed, bool update_pdptrs) 573 { 574 int idx; 575 struct desc_ptr dt; 576 577 if (!kvm_is_valid_sregs(vcpu, sregs)) 578 return -EINVAL; 579 580 if (kvm_apic_set_base(vcpu, sregs->apic_base, true)) 581 return -EINVAL; 582 583 if (vcpu->arch.guest_state_protected) 584 return 0; 585 586 dt.size = sregs->idt.limit; 587 dt.address = sregs->idt.base; 588 kvm_x86_call(set_idt)(vcpu, &dt); 589 dt.size = sregs->gdt.limit; 590 dt.address = sregs->gdt.base; 591 kvm_x86_call(set_gdt)(vcpu, &dt); 592 593 vcpu->arch.cr2 = sregs->cr2; 594 *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; 595 vcpu->arch.cr3 = sregs->cr3; 596 kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); 597 kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3); 598 599 *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; 600 kvm_x86_call(set_efer)(vcpu, sregs->efer); 601 602 *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; 603 kvm_x86_call(set_cr0)(vcpu, sregs->cr0); 604 605 *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; 606 kvm_x86_call(set_cr4)(vcpu, sregs->cr4); 607 608 if (update_pdptrs) { 609 idx = srcu_read_lock(&vcpu->kvm->srcu); 610 if (is_pae_paging(vcpu)) { 611 load_pdptrs(vcpu, kvm_read_cr3(vcpu)); 612 *mmu_reset_needed = 1; 613 } 614 srcu_read_unlock(&vcpu->kvm->srcu, idx); 615 } 616 617 kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); 618 kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); 619 kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); 620 kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); 621 kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); 622 kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); 623 624 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); 625 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); 626 627 kvm_set_cr8(vcpu, sregs->cr8); 628 629 /* Older userspace won't unhalt the vcpu on reset. */ 630 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && 631 sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && 632 !is_protmode(vcpu)) 633 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 634 635 return 0; 636 } 637 638 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 639 { 640 int pending_vec, max_bits; 641 int mmu_reset_needed = 0; 642 int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); 643 644 if (ret) 645 return ret; 646 647 if (mmu_reset_needed) { 648 kvm_mmu_reset_context(vcpu); 649 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 650 } 651 652 max_bits = KVM_NR_INTERRUPTS; 653 pending_vec = find_first_bit( 654 (const unsigned long *)sregs->interrupt_bitmap, max_bits); 655 656 if (pending_vec < max_bits) { 657 kvm_queue_interrupt(vcpu, pending_vec, false); 658 pr_debug("Set back pending irq %d\n", pending_vec); 659 kvm_make_request(KVM_REQ_EVENT, vcpu); 660 } 661 return 0; 662 } 663 664 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 665 struct kvm_sregs *sregs) 666 { 667 int ret; 668 669 if (vcpu->kvm->arch.has_protected_state && 670 vcpu->arch.guest_state_protected) 671 return -EINVAL; 672 673 vcpu_load(vcpu); 674 ret = __set_sregs(vcpu, sregs); 675 vcpu_put(vcpu); 676 return ret; 677 } 678 679 int kvm_vcpu_ioctl_x86_set_sregs2(struct kvm_vcpu *vcpu, 680 struct kvm_sregs2 *sregs2) 681 { 682 int mmu_reset_needed = 0; 683 bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; 684 bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && 685 !(sregs2->efer & EFER_LMA); 686 int i, ret; 687 688 if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) 689 return -EINVAL; 690 691 if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) 692 return -EINVAL; 693 694 ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, 695 &mmu_reset_needed, !valid_pdptrs); 696 if (ret) 697 return ret; 698 699 if (valid_pdptrs) { 700 for (i = 0; i < 4 ; i++) 701 kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); 702 703 kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); 704 mmu_reset_needed = 1; 705 vcpu->arch.pdptrs_from_userspace = true; 706 } 707 if (mmu_reset_needed) { 708 kvm_mmu_reset_context(vcpu); 709 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 710 } 711 return 0; 712 } 713 714 void kvm_run_sync_regs_to_user(struct kvm_vcpu *vcpu) 715 { 716 BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES); 717 718 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS) 719 __get_regs(vcpu, &vcpu->run->s.regs.regs); 720 721 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS) 722 __get_sregs(vcpu, &vcpu->run->s.regs.sregs); 723 } 724 725 int kvm_run_sync_regs_from_user(struct kvm_vcpu *vcpu) 726 { 727 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) { 728 __set_regs(vcpu, &vcpu->run->s.regs.regs); 729 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS; 730 } 731 732 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) { 733 struct kvm_sregs sregs = vcpu->run->s.regs.sregs; 734 735 if (__set_sregs(vcpu, &sregs)) 736 return -EINVAL; 737 738 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS; 739 } 740 741 return 0; 742 } 743 744 void kvm_update_dr0123(struct kvm_vcpu *vcpu) 745 { 746 int i; 747 748 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { 749 for (i = 0; i < KVM_NR_DB_REGS; i++) 750 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 751 } 752 } 753 754 void kvm_update_dr7(struct kvm_vcpu *vcpu) 755 { 756 unsigned long dr7; 757 758 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 759 dr7 = vcpu->arch.guest_debug_dr7; 760 else 761 dr7 = vcpu->arch.dr7; 762 kvm_x86_call(set_dr7)(vcpu, dr7); 763 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; 764 if (dr7 & DR7_BP_EN_MASK) 765 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; 766 } 767 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); 768 769 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) 770 { 771 u64 fixed = DR6_FIXED_1; 772 773 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) 774 fixed |= DR6_RTM; 775 776 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) 777 fixed |= DR6_BUS_LOCK; 778 return fixed; 779 } 780 781 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) 782 { 783 size_t size = ARRAY_SIZE(vcpu->arch.db); 784 785 switch (dr) { 786 case 0 ... 3: 787 vcpu->arch.db[array_index_nospec(dr, size)] = val; 788 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) 789 vcpu->arch.eff_db[dr] = val; 790 break; 791 case 4: 792 case 6: 793 if (!kvm_dr6_valid(val)) 794 return 1; /* #GP */ 795 vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); 796 break; 797 case 5: 798 default: /* 7 */ 799 if (!kvm_dr7_valid(val)) 800 return 1; /* #GP */ 801 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; 802 kvm_update_dr7(vcpu); 803 break; 804 } 805 806 return 0; 807 } 808 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_dr); 809 810 unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr) 811 { 812 size_t size = ARRAY_SIZE(vcpu->arch.db); 813 814 switch (dr) { 815 case 0 ... 3: 816 return vcpu->arch.db[array_index_nospec(dr, size)]; 817 case 4: 818 case 6: 819 return vcpu->arch.dr6; 820 case 5: 821 default: /* 7 */ 822 return vcpu->arch.dr7; 823 } 824 } 825 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr); 826 827 int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, 828 struct kvm_debugregs *dbgregs) 829 { 830 unsigned int i; 831 832 if (vcpu->kvm->arch.has_protected_state && 833 vcpu->arch.guest_state_protected) 834 return -EINVAL; 835 836 kvm_handle_exception_payload_quirk(vcpu); 837 838 memset(dbgregs, 0, sizeof(*dbgregs)); 839 840 BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db)); 841 for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) 842 dbgregs->db[i] = vcpu->arch.db[i]; 843 844 dbgregs->dr6 = vcpu->arch.dr6; 845 dbgregs->dr7 = vcpu->arch.dr7; 846 return 0; 847 } 848 849 int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, 850 struct kvm_debugregs *dbgregs) 851 { 852 unsigned int i; 853 854 if (vcpu->kvm->arch.has_protected_state && 855 vcpu->arch.guest_state_protected) 856 return -EINVAL; 857 858 if (dbgregs->flags) 859 return -EINVAL; 860 861 if (!kvm_dr6_valid(dbgregs->dr6)) 862 return -EINVAL; 863 if (!kvm_dr7_valid(dbgregs->dr7)) 864 return -EINVAL; 865 866 for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) 867 vcpu->arch.db[i] = dbgregs->db[i]; 868 869 kvm_update_dr0123(vcpu); 870 vcpu->arch.dr6 = dbgregs->dr6; 871 vcpu->arch.dr7 = dbgregs->dr7; 872 kvm_update_dr7(vcpu); 873 874 return 0; 875 } 876