1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/moduleparam.h> 3 4 #include "x86_ops.h" 5 #include "vmx.h" 6 #include "mmu.h" 7 #include "nested.h" 8 #include "pmu.h" 9 #include "posted_intr.h" 10 #include "tdx.h" 11 #include "tdx_arch.h" 12 13 #ifdef CONFIG_KVM_INTEL_TDX 14 static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); 15 16 static void vt_disable_virtualization_cpu(void) 17 { 18 /* Note, TDX *and* VMX need to be disabled if TDX is enabled. */ 19 if (enable_tdx) 20 tdx_disable_virtualization_cpu(); 21 vmx_disable_virtualization_cpu(); 22 } 23 24 static __init int vt_hardware_setup(void) 25 { 26 int ret; 27 28 ret = vmx_hardware_setup(); 29 if (ret) 30 return ret; 31 32 return enable_tdx ? tdx_hardware_setup() : 0; 33 } 34 35 static void vt_hardware_unsetup(void) 36 { 37 if (enable_tdx) 38 tdx_hardware_unsetup(); 39 40 vmx_hardware_unsetup(); 41 } 42 43 static int vt_vm_init(struct kvm *kvm) 44 { 45 if (is_td(kvm)) 46 return tdx_vm_init(kvm); 47 48 return vmx_vm_init(kvm); 49 } 50 51 static void vt_vm_pre_destroy(struct kvm *kvm) 52 { 53 if (is_td(kvm)) 54 return tdx_mmu_release_hkid(kvm); 55 } 56 57 static void vt_vm_destroy(struct kvm *kvm) 58 { 59 if (is_td(kvm)) 60 return tdx_vm_destroy(kvm); 61 62 vmx_vm_destroy(kvm); 63 } 64 65 static int vt_vcpu_precreate(struct kvm *kvm) 66 { 67 if (is_td(kvm)) 68 return 0; 69 70 return vmx_vcpu_precreate(kvm); 71 } 72 73 static int vt_vcpu_create(struct kvm_vcpu *vcpu) 74 { 75 if (is_td_vcpu(vcpu)) 76 return tdx_vcpu_create(vcpu); 77 78 return vmx_vcpu_create(vcpu); 79 } 80 81 static void vt_vcpu_free(struct kvm_vcpu *vcpu) 82 { 83 if (is_td_vcpu(vcpu)) { 84 tdx_vcpu_free(vcpu); 85 return; 86 } 87 88 vmx_vcpu_free(vcpu); 89 } 90 91 static void vt_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 92 { 93 if (is_td_vcpu(vcpu)) { 94 tdx_vcpu_reset(vcpu, init_event); 95 return; 96 } 97 98 vmx_vcpu_reset(vcpu, init_event); 99 } 100 101 static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 102 { 103 if (is_td_vcpu(vcpu)) { 104 tdx_vcpu_load(vcpu, cpu); 105 return; 106 } 107 108 vmx_vcpu_load(vcpu, cpu); 109 } 110 111 static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 112 { 113 /* 114 * Basic TDX does not support feature PML. KVM does not enable PML in 115 * TD's VMCS, nor does it allocate or flush PML buffer for TDX. 116 */ 117 if (WARN_ON_ONCE(is_td_vcpu(vcpu))) 118 return; 119 120 vmx_update_cpu_dirty_logging(vcpu); 121 } 122 123 static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 124 { 125 if (is_td_vcpu(vcpu)) { 126 tdx_prepare_switch_to_guest(vcpu); 127 return; 128 } 129 130 vmx_prepare_switch_to_guest(vcpu); 131 } 132 133 static void vt_vcpu_put(struct kvm_vcpu *vcpu) 134 { 135 if (is_td_vcpu(vcpu)) { 136 tdx_vcpu_put(vcpu); 137 return; 138 } 139 140 vmx_vcpu_put(vcpu); 141 } 142 143 static bool vt_vcpu_needs_initialization(struct kvm_vcpu *vcpu) 144 { 145 return is_td_vcpu(vcpu) && 146 tdx_vcpu_needs_initialization(vcpu); 147 } 148 149 static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) 150 { 151 if (is_td_vcpu(vcpu)) 152 return tdx_vcpu_run(vcpu, run_flags); 153 154 return vmx_vcpu_run(vcpu, run_flags); 155 } 156 157 static int vt_handle_exit(struct kvm_vcpu *vcpu, 158 enum exit_fastpath_completion fastpath) 159 { 160 if (is_td_vcpu(vcpu)) 161 return tdx_handle_exit(vcpu, fastpath); 162 163 return vmx_handle_exit(vcpu, fastpath); 164 } 165 166 static bool vt_unhandleable_emulation_required(struct kvm_vcpu *vcpu) 167 { 168 if (is_td_vcpu(vcpu)) { 169 WARN_ON_ONCE(to_vt(vcpu)->emulation_required); 170 return false; 171 } 172 173 return vmx_unhandleable_emulation_required(vcpu); 174 } 175 176 static int vt_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 177 { 178 if (unlikely(is_td_vcpu(vcpu))) 179 return tdx_set_msr(vcpu, msr_info); 180 181 return vmx_set_msr(vcpu, msr_info); 182 } 183 184 /* 185 * The kvm parameter can be NULL (module initialization, or invocation before 186 * VM creation). Be sure to check the kvm parameter before using it. 187 */ 188 static bool vt_has_emulated_msr(struct kvm *kvm, u32 index) 189 { 190 if (kvm && is_td(kvm)) 191 return tdx_has_emulated_msr(index); 192 193 return vmx_has_emulated_msr(kvm, index); 194 } 195 196 static int vt_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 197 { 198 if (unlikely(is_td_vcpu(vcpu))) 199 return tdx_get_msr(vcpu, msr_info); 200 201 return vmx_get_msr(vcpu, msr_info); 202 } 203 204 static void vt_recalc_intercepts(struct kvm_vcpu *vcpu) 205 { 206 /* 207 * TDX doesn't allow VMM to configure interception of instructions or 208 * MSR accesses. TDX guest requests MSR accesses by calling TDVMCALL. 209 * The MSR filters will be applied when handling the TDVMCALL for 210 * RDMSR/WRMSR if the userspace has set any. 211 */ 212 if (is_td_vcpu(vcpu)) 213 return; 214 215 vmx_recalc_intercepts(vcpu); 216 } 217 218 static int vt_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) 219 { 220 if (is_td_vcpu(vcpu)) 221 return tdx_complete_emulated_msr(vcpu, err); 222 223 return vmx_complete_emulated_msr(vcpu, err); 224 } 225 226 #ifdef CONFIG_KVM_SMM 227 static int vt_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 228 { 229 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 230 return 0; 231 232 return vmx_smi_allowed(vcpu, for_injection); 233 } 234 235 static int vt_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 236 { 237 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 238 return 0; 239 240 return vmx_enter_smm(vcpu, smram); 241 } 242 243 static int vt_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 244 { 245 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 246 return 0; 247 248 return vmx_leave_smm(vcpu, smram); 249 } 250 251 static void vt_enable_smi_window(struct kvm_vcpu *vcpu) 252 { 253 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 254 return; 255 256 /* RSM will cause a vmexit anyway. */ 257 vmx_enable_smi_window(vcpu); 258 } 259 #endif 260 261 static int vt_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 262 void *insn, int insn_len) 263 { 264 /* 265 * For TDX, this can only be triggered for MMIO emulation. Let the 266 * guest retry after installing the SPTE with suppress #VE bit cleared, 267 * so that the guest will receive #VE when retry. The guest is expected 268 * to call TDG.VP.VMCALL<MMIO> to request VMM to do MMIO emulation on 269 * #VE. 270 */ 271 if (is_td_vcpu(vcpu)) 272 return X86EMUL_RETRY_INSTR; 273 274 return vmx_check_emulate_instruction(vcpu, emul_type, insn, insn_len); 275 } 276 277 static bool vt_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 278 { 279 /* 280 * INIT and SIPI are always blocked for TDX, i.e., INIT handling and 281 * the OP vcpu_deliver_sipi_vector() won't be called. 282 */ 283 if (is_td_vcpu(vcpu)) 284 return true; 285 286 return vmx_apic_init_signal_blocked(vcpu); 287 } 288 289 static void vt_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 290 { 291 /* Only x2APIC mode is supported for TD. */ 292 if (is_td_vcpu(vcpu)) 293 return; 294 295 return vmx_set_virtual_apic_mode(vcpu); 296 } 297 298 static void vt_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 299 { 300 if (is_td_vcpu(vcpu)) 301 return; 302 303 return vmx_hwapic_isr_update(vcpu, max_isr); 304 } 305 306 static int vt_sync_pir_to_irr(struct kvm_vcpu *vcpu) 307 { 308 if (is_td_vcpu(vcpu)) 309 return -1; 310 311 return vmx_sync_pir_to_irr(vcpu); 312 } 313 314 static void vt_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 315 int trig_mode, int vector) 316 { 317 if (is_td_vcpu(apic->vcpu)) { 318 tdx_deliver_interrupt(apic, delivery_mode, trig_mode, 319 vector); 320 return; 321 } 322 323 vmx_deliver_interrupt(apic, delivery_mode, trig_mode, vector); 324 } 325 326 static void vt_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 327 { 328 if (is_td_vcpu(vcpu)) 329 return; 330 331 vmx_vcpu_after_set_cpuid(vcpu); 332 } 333 334 static void vt_update_exception_bitmap(struct kvm_vcpu *vcpu) 335 { 336 if (is_td_vcpu(vcpu)) 337 return; 338 339 vmx_update_exception_bitmap(vcpu); 340 } 341 342 static u64 vt_get_segment_base(struct kvm_vcpu *vcpu, int seg) 343 { 344 if (is_td_vcpu(vcpu)) 345 return 0; 346 347 return vmx_get_segment_base(vcpu, seg); 348 } 349 350 static void vt_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, 351 int seg) 352 { 353 if (is_td_vcpu(vcpu)) { 354 memset(var, 0, sizeof(*var)); 355 return; 356 } 357 358 vmx_get_segment(vcpu, var, seg); 359 } 360 361 static void vt_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, 362 int seg) 363 { 364 if (is_td_vcpu(vcpu)) 365 return; 366 367 vmx_set_segment(vcpu, var, seg); 368 } 369 370 static int vt_get_cpl(struct kvm_vcpu *vcpu) 371 { 372 if (is_td_vcpu(vcpu)) 373 return 0; 374 375 return vmx_get_cpl(vcpu); 376 } 377 378 static int vt_get_cpl_no_cache(struct kvm_vcpu *vcpu) 379 { 380 if (is_td_vcpu(vcpu)) 381 return 0; 382 383 return vmx_get_cpl_no_cache(vcpu); 384 } 385 386 static void vt_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 387 { 388 if (is_td_vcpu(vcpu)) { 389 *db = 0; 390 *l = 0; 391 return; 392 } 393 394 vmx_get_cs_db_l_bits(vcpu, db, l); 395 } 396 397 static bool vt_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 398 { 399 if (is_td_vcpu(vcpu)) 400 return true; 401 402 return vmx_is_valid_cr0(vcpu, cr0); 403 } 404 405 static void vt_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 406 { 407 if (is_td_vcpu(vcpu)) 408 return; 409 410 vmx_set_cr0(vcpu, cr0); 411 } 412 413 static bool vt_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 414 { 415 if (is_td_vcpu(vcpu)) 416 return true; 417 418 return vmx_is_valid_cr4(vcpu, cr4); 419 } 420 421 static void vt_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 422 { 423 if (is_td_vcpu(vcpu)) 424 return; 425 426 vmx_set_cr4(vcpu, cr4); 427 } 428 429 static int vt_set_efer(struct kvm_vcpu *vcpu, u64 efer) 430 { 431 if (is_td_vcpu(vcpu)) 432 return 0; 433 434 return vmx_set_efer(vcpu, efer); 435 } 436 437 static void vt_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 438 { 439 if (is_td_vcpu(vcpu)) { 440 memset(dt, 0, sizeof(*dt)); 441 return; 442 } 443 444 vmx_get_idt(vcpu, dt); 445 } 446 447 static void vt_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 448 { 449 if (is_td_vcpu(vcpu)) 450 return; 451 452 vmx_set_idt(vcpu, dt); 453 } 454 455 static void vt_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 456 { 457 if (is_td_vcpu(vcpu)) { 458 memset(dt, 0, sizeof(*dt)); 459 return; 460 } 461 462 vmx_get_gdt(vcpu, dt); 463 } 464 465 static void vt_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 466 { 467 if (is_td_vcpu(vcpu)) 468 return; 469 470 vmx_set_gdt(vcpu, dt); 471 } 472 473 static void vt_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 474 { 475 if (is_td_vcpu(vcpu)) 476 return; 477 478 vmx_set_dr7(vcpu, val); 479 } 480 481 static void vt_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 482 { 483 /* 484 * MOV-DR exiting is always cleared for TD guest, even in debug mode. 485 * Thus KVM_DEBUGREG_WONT_EXIT can never be set and it should never 486 * reach here for TD vcpu. 487 */ 488 if (is_td_vcpu(vcpu)) 489 return; 490 491 vmx_sync_dirty_debug_regs(vcpu); 492 } 493 494 static void vt_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 495 { 496 if (WARN_ON_ONCE(is_td_vcpu(vcpu))) 497 return; 498 499 vmx_cache_reg(vcpu, reg); 500 } 501 502 static unsigned long vt_get_rflags(struct kvm_vcpu *vcpu) 503 { 504 if (is_td_vcpu(vcpu)) 505 return 0; 506 507 return vmx_get_rflags(vcpu); 508 } 509 510 static void vt_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 511 { 512 if (is_td_vcpu(vcpu)) 513 return; 514 515 vmx_set_rflags(vcpu, rflags); 516 } 517 518 static bool vt_get_if_flag(struct kvm_vcpu *vcpu) 519 { 520 if (is_td_vcpu(vcpu)) 521 return false; 522 523 return vmx_get_if_flag(vcpu); 524 } 525 526 static void vt_flush_tlb_all(struct kvm_vcpu *vcpu) 527 { 528 if (is_td_vcpu(vcpu)) { 529 tdx_flush_tlb_all(vcpu); 530 return; 531 } 532 533 vmx_flush_tlb_all(vcpu); 534 } 535 536 static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) 537 { 538 if (is_td_vcpu(vcpu)) { 539 tdx_flush_tlb_current(vcpu); 540 return; 541 } 542 543 vmx_flush_tlb_current(vcpu); 544 } 545 546 static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) 547 { 548 if (is_td_vcpu(vcpu)) 549 return; 550 551 vmx_flush_tlb_gva(vcpu, addr, full); 552 } 553 554 static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) 555 { 556 if (is_td_vcpu(vcpu)) 557 return; 558 559 vmx_flush_tlb_guest(vcpu); 560 } 561 562 static void vt_inject_nmi(struct kvm_vcpu *vcpu) 563 { 564 if (is_td_vcpu(vcpu)) { 565 tdx_inject_nmi(vcpu); 566 return; 567 } 568 569 vmx_inject_nmi(vcpu); 570 } 571 572 static int vt_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 573 { 574 /* 575 * The TDX module manages NMI windows and NMI reinjection, and hides NMI 576 * blocking, all KVM can do is throw an NMI over the wall. 577 */ 578 if (is_td_vcpu(vcpu)) 579 return true; 580 581 return vmx_nmi_allowed(vcpu, for_injection); 582 } 583 584 static bool vt_get_nmi_mask(struct kvm_vcpu *vcpu) 585 { 586 /* 587 * KVM can't get NMI blocking status for TDX guest, assume NMIs are 588 * always unmasked. 589 */ 590 if (is_td_vcpu(vcpu)) 591 return false; 592 593 return vmx_get_nmi_mask(vcpu); 594 } 595 596 static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 597 { 598 if (is_td_vcpu(vcpu)) 599 return; 600 601 vmx_set_nmi_mask(vcpu, masked); 602 } 603 604 static void vt_enable_nmi_window(struct kvm_vcpu *vcpu) 605 { 606 /* Refer to the comments in tdx_inject_nmi(). */ 607 if (is_td_vcpu(vcpu)) 608 return; 609 610 vmx_enable_nmi_window(vcpu); 611 } 612 613 static void vt_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 614 int pgd_level) 615 { 616 if (is_td_vcpu(vcpu)) { 617 tdx_load_mmu_pgd(vcpu, root_hpa, pgd_level); 618 return; 619 } 620 621 vmx_load_mmu_pgd(vcpu, root_hpa, pgd_level); 622 } 623 624 static void vt_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 625 { 626 if (is_td_vcpu(vcpu)) 627 return; 628 629 vmx_set_interrupt_shadow(vcpu, mask); 630 } 631 632 static u32 vt_get_interrupt_shadow(struct kvm_vcpu *vcpu) 633 { 634 if (is_td_vcpu(vcpu)) 635 return 0; 636 637 return vmx_get_interrupt_shadow(vcpu); 638 } 639 640 static void vt_patch_hypercall(struct kvm_vcpu *vcpu, 641 unsigned char *hypercall) 642 { 643 /* 644 * Because guest memory is protected, guest can't be patched. TD kernel 645 * is modified to use TDG.VP.VMCALL for hypercall. 646 */ 647 if (is_td_vcpu(vcpu)) 648 return; 649 650 vmx_patch_hypercall(vcpu, hypercall); 651 } 652 653 static void vt_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 654 { 655 if (is_td_vcpu(vcpu)) 656 return; 657 658 vmx_inject_irq(vcpu, reinjected); 659 } 660 661 static void vt_inject_exception(struct kvm_vcpu *vcpu) 662 { 663 if (is_td_vcpu(vcpu)) 664 return; 665 666 vmx_inject_exception(vcpu); 667 } 668 669 static void vt_cancel_injection(struct kvm_vcpu *vcpu) 670 { 671 if (is_td_vcpu(vcpu)) 672 return; 673 674 vmx_cancel_injection(vcpu); 675 } 676 677 static int vt_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 678 { 679 if (is_td_vcpu(vcpu)) 680 return tdx_interrupt_allowed(vcpu); 681 682 return vmx_interrupt_allowed(vcpu, for_injection); 683 } 684 685 static void vt_enable_irq_window(struct kvm_vcpu *vcpu) 686 { 687 if (is_td_vcpu(vcpu)) 688 return; 689 690 vmx_enable_irq_window(vcpu); 691 } 692 693 static void vt_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) 694 { 695 *intr_info = 0; 696 *error_code = 0; 697 698 if (is_td_vcpu(vcpu)) 699 return; 700 701 vmx_get_entry_info(vcpu, intr_info, error_code); 702 } 703 704 static void vt_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 705 u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) 706 { 707 if (is_td_vcpu(vcpu)) { 708 tdx_get_exit_info(vcpu, reason, info1, info2, intr_info, 709 error_code); 710 return; 711 } 712 713 vmx_get_exit_info(vcpu, reason, info1, info2, intr_info, error_code); 714 } 715 716 static void vt_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 717 { 718 if (is_td_vcpu(vcpu)) 719 return; 720 721 vmx_update_cr8_intercept(vcpu, tpr, irr); 722 } 723 724 static void vt_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 725 { 726 if (is_td_vcpu(vcpu)) 727 return; 728 729 vmx_set_apic_access_page_addr(vcpu); 730 } 731 732 static void vt_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 733 { 734 if (is_td_vcpu(vcpu)) { 735 KVM_BUG_ON(!kvm_vcpu_apicv_active(vcpu), vcpu->kvm); 736 return; 737 } 738 739 vmx_refresh_apicv_exec_ctrl(vcpu); 740 } 741 742 static void vt_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 743 { 744 if (is_td_vcpu(vcpu)) 745 return; 746 747 vmx_load_eoi_exitmap(vcpu, eoi_exit_bitmap); 748 } 749 750 static int vt_set_tss_addr(struct kvm *kvm, unsigned int addr) 751 { 752 if (is_td(kvm)) 753 return 0; 754 755 return vmx_set_tss_addr(kvm, addr); 756 } 757 758 static int vt_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 759 { 760 if (is_td(kvm)) 761 return 0; 762 763 return vmx_set_identity_map_addr(kvm, ident_addr); 764 } 765 766 static bool vt_tdp_has_smep(struct kvm *kvm) 767 { 768 if (is_td(kvm)) 769 return false; 770 771 return vmx_tdp_has_smep(kvm); 772 } 773 774 static u64 vt_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 775 { 776 /* TDX doesn't support L2 guest at the moment. */ 777 if (is_td_vcpu(vcpu)) 778 return 0; 779 780 return vmx_get_l2_tsc_offset(vcpu); 781 } 782 783 static u64 vt_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 784 { 785 /* TDX doesn't support L2 guest at the moment. */ 786 if (is_td_vcpu(vcpu)) 787 return 0; 788 789 return vmx_get_l2_tsc_multiplier(vcpu); 790 } 791 792 static void vt_write_tsc_offset(struct kvm_vcpu *vcpu) 793 { 794 /* In TDX, tsc offset can't be changed. */ 795 if (is_td_vcpu(vcpu)) 796 return; 797 798 vmx_write_tsc_offset(vcpu); 799 } 800 801 static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu) 802 { 803 /* In TDX, tsc multiplier can't be changed. */ 804 if (is_td_vcpu(vcpu)) 805 return; 806 807 vmx_write_tsc_multiplier(vcpu); 808 } 809 810 #ifdef CONFIG_X86_64 811 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 812 bool *expired) 813 { 814 /* VMX-preemption timer isn't available for TDX. */ 815 if (is_td_vcpu(vcpu)) 816 return -EINVAL; 817 818 return vmx_set_hv_timer(vcpu, guest_deadline_tsc, expired); 819 } 820 821 static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu) 822 { 823 /* VMX-preemption timer can't be set. See vt_set_hv_timer(). */ 824 if (is_td_vcpu(vcpu)) 825 return; 826 827 vmx_cancel_hv_timer(vcpu); 828 } 829 #endif 830 831 static void vt_setup_mce(struct kvm_vcpu *vcpu) 832 { 833 if (is_td_vcpu(vcpu)) 834 return; 835 836 vmx_setup_mce(vcpu); 837 } 838 839 static int vt_mem_enc_ioctl(struct kvm *kvm, void __user *argp) 840 { 841 if (!is_td(kvm)) 842 return -ENOTTY; 843 844 return tdx_vm_ioctl(kvm, argp); 845 } 846 847 static int vt_vcpu_mem_enc_ioctl(struct kvm_vcpu *vcpu, void __user *argp) 848 { 849 if (!is_td_vcpu(vcpu)) 850 return -EINVAL; 851 852 return tdx_vcpu_ioctl(vcpu, argp); 853 } 854 855 static int vt_vcpu_mem_enc_unlocked_ioctl(struct kvm_vcpu *vcpu, void __user *argp) 856 { 857 if (!is_td_vcpu(vcpu)) 858 return -EINVAL; 859 860 return tdx_vcpu_unlocked_ioctl(vcpu, argp); 861 } 862 863 static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, 864 bool is_private) 865 { 866 if (is_td(kvm)) 867 return tdx_gmem_max_mapping_level(kvm, pfn, is_private); 868 869 return 0; 870 } 871 872 #define vt_op(name) vt_##name 873 #define vt_op_tdx_only(name) vt_##name 874 #else /* CONFIG_KVM_INTEL_TDX */ 875 #define vt_op(name) vmx_##name 876 #define vt_op_tdx_only(name) NULL 877 #endif /* CONFIG_KVM_INTEL_TDX */ 878 879 #define VMX_REQUIRED_APICV_INHIBITS \ 880 (BIT(APICV_INHIBIT_REASON_DISABLED) | \ 881 BIT(APICV_INHIBIT_REASON_ABSENT) | \ 882 BIT(APICV_INHIBIT_REASON_HYPERV) | \ 883 BIT(APICV_INHIBIT_REASON_BLOCKIRQ) | \ 884 BIT(APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED) | \ 885 BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) | \ 886 BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED)) 887 888 struct kvm_x86_ops vt_x86_ops __initdata = { 889 .name = KBUILD_MODNAME, 890 891 .check_processor_compatibility = vmx_check_processor_compat, 892 893 .hardware_unsetup = vt_op(hardware_unsetup), 894 895 .enable_virtualization_cpu = vmx_enable_virtualization_cpu, 896 .disable_virtualization_cpu = vt_op(disable_virtualization_cpu), 897 .emergency_disable_virtualization_cpu = vmx_emergency_disable_virtualization_cpu, 898 899 .has_emulated_msr = vt_op(has_emulated_msr), 900 901 .vm_size = sizeof(struct kvm_vmx), 902 903 .vm_init = vt_op(vm_init), 904 .vm_destroy = vt_op(vm_destroy), 905 .vm_pre_destroy = vt_op_tdx_only(vm_pre_destroy), 906 907 .vcpu_precreate = vt_op(vcpu_precreate), 908 .vcpu_create = vt_op(vcpu_create), 909 .vcpu_free = vt_op(vcpu_free), 910 .vcpu_reset = vt_op(vcpu_reset), 911 912 .prepare_switch_to_guest = vt_op(prepare_switch_to_guest), 913 .vcpu_load = vt_op(vcpu_load), 914 .vcpu_put = vt_op(vcpu_put), 915 916 .HOST_OWNED_DEBUGCTL = VMX_HOST_OWNED_DEBUGCTL_BITS, 917 918 .update_exception_bitmap = vt_op(update_exception_bitmap), 919 .get_feature_msr = vmx_get_feature_msr, 920 .get_msr = vt_op(get_msr), 921 .set_msr = vt_op(set_msr), 922 923 .get_segment_base = vt_op(get_segment_base), 924 .get_segment = vt_op(get_segment), 925 .set_segment = vt_op(set_segment), 926 .get_cpl = vt_op(get_cpl), 927 .get_cpl_no_cache = vt_op(get_cpl_no_cache), 928 .get_cs_db_l_bits = vt_op(get_cs_db_l_bits), 929 .is_valid_cr0 = vt_op(is_valid_cr0), 930 .set_cr0 = vt_op(set_cr0), 931 .is_valid_cr4 = vt_op(is_valid_cr4), 932 .set_cr4 = vt_op(set_cr4), 933 .set_efer = vt_op(set_efer), 934 .get_idt = vt_op(get_idt), 935 .set_idt = vt_op(set_idt), 936 .get_gdt = vt_op(get_gdt), 937 .set_gdt = vt_op(set_gdt), 938 .set_dr7 = vt_op(set_dr7), 939 .sync_dirty_debug_regs = vt_op(sync_dirty_debug_regs), 940 .cache_reg = vt_op(cache_reg), 941 .get_rflags = vt_op(get_rflags), 942 .set_rflags = vt_op(set_rflags), 943 .get_if_flag = vt_op(get_if_flag), 944 945 .flush_tlb_all = vt_op(flush_tlb_all), 946 .flush_tlb_current = vt_op(flush_tlb_current), 947 .flush_tlb_gva = vt_op(flush_tlb_gva), 948 .flush_tlb_guest = vt_op(flush_tlb_guest), 949 950 .vcpu_needs_initialization = vt_op_tdx_only(vcpu_needs_initialization), 951 .vcpu_run = vt_op(vcpu_run), 952 .handle_exit = vt_op(handle_exit), 953 .skip_emulated_instruction = vmx_skip_emulated_instruction, 954 .update_emulated_instruction = vmx_update_emulated_instruction, 955 .unhandleable_emulation_required = vt_op(unhandleable_emulation_required), 956 .set_interrupt_shadow = vt_op(set_interrupt_shadow), 957 .get_interrupt_shadow = vt_op(get_interrupt_shadow), 958 .patch_hypercall = vt_op(patch_hypercall), 959 .inject_irq = vt_op(inject_irq), 960 .inject_nmi = vt_op(inject_nmi), 961 .inject_exception = vt_op(inject_exception), 962 .cancel_injection = vt_op(cancel_injection), 963 .interrupt_allowed = vt_op(interrupt_allowed), 964 .nmi_allowed = vt_op(nmi_allowed), 965 .get_nmi_mask = vt_op(get_nmi_mask), 966 .set_nmi_mask = vt_op(set_nmi_mask), 967 .enable_nmi_window = vt_op(enable_nmi_window), 968 .enable_irq_window = vt_op(enable_irq_window), 969 .update_cr8_intercept = vt_op(update_cr8_intercept), 970 971 .x2apic_icr_is_split = false, 972 .set_virtual_apic_mode = vt_op(set_virtual_apic_mode), 973 .set_apic_access_page_addr = vt_op(set_apic_access_page_addr), 974 .refresh_apicv_exec_ctrl = vt_op(refresh_apicv_exec_ctrl), 975 .load_eoi_exitmap = vt_op(load_eoi_exitmap), 976 .apicv_pre_state_restore = pi_apicv_pre_state_restore, 977 .required_apicv_inhibits = VMX_REQUIRED_APICV_INHIBITS, 978 .hwapic_isr_update = vt_op(hwapic_isr_update), 979 .sync_pir_to_irr = vt_op(sync_pir_to_irr), 980 .deliver_interrupt = vt_op(deliver_interrupt), 981 .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, 982 983 .set_tss_addr = vt_op(set_tss_addr), 984 .set_identity_map_addr = vt_op(set_identity_map_addr), 985 .get_mt_mask = vmx_get_mt_mask, 986 .tdp_has_smep = vt_op(tdp_has_smep), 987 988 .get_exit_info = vt_op(get_exit_info), 989 .get_entry_info = vt_op(get_entry_info), 990 991 .vcpu_after_set_cpuid = vt_op(vcpu_after_set_cpuid), 992 993 .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, 994 995 .get_l2_tsc_offset = vt_op(get_l2_tsc_offset), 996 .get_l2_tsc_multiplier = vt_op(get_l2_tsc_multiplier), 997 .write_tsc_offset = vt_op(write_tsc_offset), 998 .write_tsc_multiplier = vt_op(write_tsc_multiplier), 999 1000 .load_mmu_pgd = vt_op(load_mmu_pgd), 1001 1002 .check_intercept = vmx_check_intercept, 1003 .handle_exit_irqoff = vmx_handle_exit_irqoff, 1004 1005 .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), 1006 1007 .pi_update_irte = vmx_pi_update_irte, 1008 .pi_start_bypass = vmx_pi_start_bypass, 1009 1010 #ifdef CONFIG_X86_64 1011 .set_hv_timer = vt_op(set_hv_timer), 1012 .cancel_hv_timer = vt_op(cancel_hv_timer), 1013 #endif 1014 1015 .setup_mce = vt_op(setup_mce), 1016 1017 #ifdef CONFIG_KVM_SMM 1018 .smi_allowed = vt_op(smi_allowed), 1019 .enter_smm = vt_op(enter_smm), 1020 .leave_smm = vt_op(leave_smm), 1021 .enable_smi_window = vt_op(enable_smi_window), 1022 #endif 1023 1024 .check_emulate_instruction = vt_op(check_emulate_instruction), 1025 .apic_init_signal_blocked = vt_op(apic_init_signal_blocked), 1026 .migrate_timers = vmx_migrate_timers, 1027 1028 .recalc_intercepts = vt_op(recalc_intercepts), 1029 .complete_emulated_msr = vt_op(complete_emulated_msr), 1030 1031 .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, 1032 1033 .get_untagged_addr = vmx_get_untagged_addr, 1034 1035 .mem_enc_ioctl = vt_op_tdx_only(mem_enc_ioctl), 1036 .vcpu_mem_enc_ioctl = vt_op_tdx_only(vcpu_mem_enc_ioctl), 1037 .vcpu_mem_enc_unlocked_ioctl = vt_op_tdx_only(vcpu_mem_enc_unlocked_ioctl), 1038 1039 .gmem_max_mapping_level = vt_op_tdx_only(gmem_max_mapping_level) 1040 }; 1041 1042 struct kvm_x86_init_ops vt_init_ops __initdata = { 1043 .hardware_setup = vt_op(hardware_setup), 1044 .handle_intel_pt_intr = NULL, 1045 1046 .runtime_ops = &vt_x86_ops, 1047 .pmu_ops = &intel_pmu_ops, 1048 .nested_ops = &vmx_nested_ops, 1049 }; 1050 1051 static void __exit vt_exit(void) 1052 { 1053 kvm_exit(); 1054 vmx_exit(); 1055 } 1056 module_exit(vt_exit); 1057 1058 static int __init vt_init(void) 1059 { 1060 unsigned vcpu_size, vcpu_align; 1061 int r; 1062 1063 r = vmx_init(); 1064 if (r) 1065 return r; 1066 1067 /* 1068 * TDX and VMX have different vCPU structures. Calculate the 1069 * maximum size/align so that kvm_init() can use the larger 1070 * values to create the kmem_vcpu_cache. 1071 */ 1072 vcpu_size = sizeof(struct vcpu_vmx); 1073 vcpu_align = __alignof__(struct vcpu_vmx); 1074 if (enable_tdx) { 1075 vcpu_size = max_t(unsigned, vcpu_size, 1076 sizeof(struct vcpu_tdx)); 1077 vcpu_align = max_t(unsigned, vcpu_align, 1078 __alignof__(struct vcpu_tdx)); 1079 kvm_caps.supported_vm_types |= BIT(KVM_X86_TDX_VM); 1080 } 1081 1082 /* 1083 * Common KVM initialization _must_ come last, after this, /dev/kvm is 1084 * exposed to userspace! 1085 */ 1086 r = kvm_init(vcpu_size, vcpu_align, THIS_MODULE); 1087 if (r) 1088 goto err_kvm_init; 1089 1090 return 0; 1091 1092 err_kvm_init: 1093 vmx_exit(); 1094 return r; 1095 } 1096 module_init(vt_init); 1097