1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/moduleparam.h> 3 4 #include "x86_ops.h" 5 #include "vmx.h" 6 #include "mmu.h" 7 #include "nested.h" 8 #include "pmu.h" 9 #include "posted_intr.h" 10 #include "tdx.h" 11 #include "tdx_arch.h" 12 13 #ifdef CONFIG_KVM_INTEL_TDX 14 static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); 15 16 static void vt_disable_virtualization_cpu(void) 17 { 18 /* Note, TDX *and* VMX need to be disabled if TDX is enabled. */ 19 if (enable_tdx) 20 tdx_disable_virtualization_cpu(); 21 vmx_disable_virtualization_cpu(); 22 } 23 24 static __init int vt_hardware_setup(void) 25 { 26 int ret; 27 28 ret = vmx_hardware_setup(); 29 if (ret) 30 return ret; 31 32 return enable_tdx ? tdx_hardware_setup() : 0; 33 } 34 35 static void vt_hardware_unsetup(void) 36 { 37 if (enable_tdx) 38 tdx_hardware_unsetup(); 39 40 vmx_hardware_unsetup(); 41 } 42 43 static int vt_vm_init(struct kvm *kvm) 44 { 45 if (is_td(kvm)) 46 return tdx_vm_init(kvm); 47 48 return vmx_vm_init(kvm); 49 } 50 51 static void vt_vm_pre_destroy(struct kvm *kvm) 52 { 53 if (is_td(kvm)) 54 return tdx_mmu_release_hkid(kvm); 55 } 56 57 static void vt_vm_destroy(struct kvm *kvm) 58 { 59 if (is_td(kvm)) 60 return tdx_vm_destroy(kvm); 61 62 vmx_vm_destroy(kvm); 63 } 64 65 static int vt_vcpu_precreate(struct kvm *kvm) 66 { 67 if (is_td(kvm)) 68 return 0; 69 70 return vmx_vcpu_precreate(kvm); 71 } 72 73 static int vt_vcpu_create(struct kvm_vcpu *vcpu) 74 { 75 if (is_td_vcpu(vcpu)) 76 return tdx_vcpu_create(vcpu); 77 78 return vmx_vcpu_create(vcpu); 79 } 80 81 static void vt_vcpu_free(struct kvm_vcpu *vcpu) 82 { 83 if (is_td_vcpu(vcpu)) { 84 tdx_vcpu_free(vcpu); 85 return; 86 } 87 88 vmx_vcpu_free(vcpu); 89 } 90 91 static void vt_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 92 { 93 if (is_td_vcpu(vcpu)) { 94 tdx_vcpu_reset(vcpu, init_event); 95 return; 96 } 97 98 vmx_vcpu_reset(vcpu, init_event); 99 } 100 101 static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 102 { 103 if (is_td_vcpu(vcpu)) { 104 tdx_vcpu_load(vcpu, cpu); 105 return; 106 } 107 108 vmx_vcpu_load(vcpu, cpu); 109 } 110 111 static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 112 { 113 /* 114 * Basic TDX does not support feature PML. KVM does not enable PML in 115 * TD's VMCS, nor does it allocate or flush PML buffer for TDX. 116 */ 117 if (WARN_ON_ONCE(is_td_vcpu(vcpu))) 118 return; 119 120 vmx_update_cpu_dirty_logging(vcpu); 121 } 122 123 static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 124 { 125 if (is_td_vcpu(vcpu)) { 126 tdx_prepare_switch_to_guest(vcpu); 127 return; 128 } 129 130 vmx_prepare_switch_to_guest(vcpu); 131 } 132 133 static void vt_vcpu_put(struct kvm_vcpu *vcpu) 134 { 135 if (is_td_vcpu(vcpu)) { 136 tdx_vcpu_put(vcpu); 137 return; 138 } 139 140 vmx_vcpu_put(vcpu); 141 } 142 143 static int vt_vcpu_pre_run(struct kvm_vcpu *vcpu) 144 { 145 if (is_td_vcpu(vcpu)) 146 return tdx_vcpu_pre_run(vcpu); 147 148 return vmx_vcpu_pre_run(vcpu); 149 } 150 151 static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) 152 { 153 if (is_td_vcpu(vcpu)) 154 return tdx_vcpu_run(vcpu, run_flags); 155 156 return vmx_vcpu_run(vcpu, run_flags); 157 } 158 159 static int vt_handle_exit(struct kvm_vcpu *vcpu, 160 enum exit_fastpath_completion fastpath) 161 { 162 if (is_td_vcpu(vcpu)) 163 return tdx_handle_exit(vcpu, fastpath); 164 165 return vmx_handle_exit(vcpu, fastpath); 166 } 167 168 static int vt_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 169 { 170 if (unlikely(is_td_vcpu(vcpu))) 171 return tdx_set_msr(vcpu, msr_info); 172 173 return vmx_set_msr(vcpu, msr_info); 174 } 175 176 /* 177 * The kvm parameter can be NULL (module initialization, or invocation before 178 * VM creation). Be sure to check the kvm parameter before using it. 179 */ 180 static bool vt_has_emulated_msr(struct kvm *kvm, u32 index) 181 { 182 if (kvm && is_td(kvm)) 183 return tdx_has_emulated_msr(index); 184 185 return vmx_has_emulated_msr(kvm, index); 186 } 187 188 static int vt_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 189 { 190 if (unlikely(is_td_vcpu(vcpu))) 191 return tdx_get_msr(vcpu, msr_info); 192 193 return vmx_get_msr(vcpu, msr_info); 194 } 195 196 static void vt_recalc_intercepts(struct kvm_vcpu *vcpu) 197 { 198 /* 199 * TDX doesn't allow VMM to configure interception of instructions or 200 * MSR accesses. TDX guest requests MSR accesses by calling TDVMCALL. 201 * The MSR filters will be applied when handling the TDVMCALL for 202 * RDMSR/WRMSR if the userspace has set any. 203 */ 204 if (is_td_vcpu(vcpu)) 205 return; 206 207 vmx_recalc_intercepts(vcpu); 208 } 209 210 static int vt_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) 211 { 212 if (is_td_vcpu(vcpu)) 213 return tdx_complete_emulated_msr(vcpu, err); 214 215 return vmx_complete_emulated_msr(vcpu, err); 216 } 217 218 #ifdef CONFIG_KVM_SMM 219 static int vt_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 220 { 221 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 222 return 0; 223 224 return vmx_smi_allowed(vcpu, for_injection); 225 } 226 227 static int vt_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 228 { 229 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 230 return 0; 231 232 return vmx_enter_smm(vcpu, smram); 233 } 234 235 static int vt_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 236 { 237 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 238 return 0; 239 240 return vmx_leave_smm(vcpu, smram); 241 } 242 243 static void vt_enable_smi_window(struct kvm_vcpu *vcpu) 244 { 245 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 246 return; 247 248 /* RSM will cause a vmexit anyway. */ 249 vmx_enable_smi_window(vcpu); 250 } 251 #endif 252 253 static int vt_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 254 void *insn, int insn_len) 255 { 256 /* 257 * For TDX, this can only be triggered for MMIO emulation. Let the 258 * guest retry after installing the SPTE with suppress #VE bit cleared, 259 * so that the guest will receive #VE when retry. The guest is expected 260 * to call TDG.VP.VMCALL<MMIO> to request VMM to do MMIO emulation on 261 * #VE. 262 */ 263 if (is_td_vcpu(vcpu)) 264 return X86EMUL_RETRY_INSTR; 265 266 return vmx_check_emulate_instruction(vcpu, emul_type, insn, insn_len); 267 } 268 269 static bool vt_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 270 { 271 /* 272 * INIT and SIPI are always blocked for TDX, i.e., INIT handling and 273 * the OP vcpu_deliver_sipi_vector() won't be called. 274 */ 275 if (is_td_vcpu(vcpu)) 276 return true; 277 278 return vmx_apic_init_signal_blocked(vcpu); 279 } 280 281 static void vt_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 282 { 283 /* Only x2APIC mode is supported for TD. */ 284 if (is_td_vcpu(vcpu)) 285 return; 286 287 return vmx_set_virtual_apic_mode(vcpu); 288 } 289 290 static void vt_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 291 { 292 if (is_td_vcpu(vcpu)) 293 return; 294 295 return vmx_hwapic_isr_update(vcpu, max_isr); 296 } 297 298 static int vt_sync_pir_to_irr(struct kvm_vcpu *vcpu) 299 { 300 if (is_td_vcpu(vcpu)) 301 return -1; 302 303 return vmx_sync_pir_to_irr(vcpu); 304 } 305 306 static void vt_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 307 int trig_mode, int vector) 308 { 309 if (is_td_vcpu(apic->vcpu)) { 310 tdx_deliver_interrupt(apic, delivery_mode, trig_mode, 311 vector); 312 return; 313 } 314 315 vmx_deliver_interrupt(apic, delivery_mode, trig_mode, vector); 316 } 317 318 static void vt_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 319 { 320 if (is_td_vcpu(vcpu)) 321 return; 322 323 vmx_vcpu_after_set_cpuid(vcpu); 324 } 325 326 static void vt_update_exception_bitmap(struct kvm_vcpu *vcpu) 327 { 328 if (is_td_vcpu(vcpu)) 329 return; 330 331 vmx_update_exception_bitmap(vcpu); 332 } 333 334 static u64 vt_get_segment_base(struct kvm_vcpu *vcpu, int seg) 335 { 336 if (is_td_vcpu(vcpu)) 337 return 0; 338 339 return vmx_get_segment_base(vcpu, seg); 340 } 341 342 static void vt_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, 343 int seg) 344 { 345 if (is_td_vcpu(vcpu)) { 346 memset(var, 0, sizeof(*var)); 347 return; 348 } 349 350 vmx_get_segment(vcpu, var, seg); 351 } 352 353 static void vt_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, 354 int seg) 355 { 356 if (is_td_vcpu(vcpu)) 357 return; 358 359 vmx_set_segment(vcpu, var, seg); 360 } 361 362 static int vt_get_cpl(struct kvm_vcpu *vcpu) 363 { 364 if (is_td_vcpu(vcpu)) 365 return 0; 366 367 return vmx_get_cpl(vcpu); 368 } 369 370 static int vt_get_cpl_no_cache(struct kvm_vcpu *vcpu) 371 { 372 if (is_td_vcpu(vcpu)) 373 return 0; 374 375 return vmx_get_cpl_no_cache(vcpu); 376 } 377 378 static void vt_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 379 { 380 if (is_td_vcpu(vcpu)) { 381 *db = 0; 382 *l = 0; 383 return; 384 } 385 386 vmx_get_cs_db_l_bits(vcpu, db, l); 387 } 388 389 static bool vt_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 390 { 391 if (is_td_vcpu(vcpu)) 392 return true; 393 394 return vmx_is_valid_cr0(vcpu, cr0); 395 } 396 397 static void vt_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 398 { 399 if (is_td_vcpu(vcpu)) 400 return; 401 402 vmx_set_cr0(vcpu, cr0); 403 } 404 405 static bool vt_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 406 { 407 if (is_td_vcpu(vcpu)) 408 return true; 409 410 return vmx_is_valid_cr4(vcpu, cr4); 411 } 412 413 static void vt_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 414 { 415 if (is_td_vcpu(vcpu)) 416 return; 417 418 vmx_set_cr4(vcpu, cr4); 419 } 420 421 static int vt_set_efer(struct kvm_vcpu *vcpu, u64 efer) 422 { 423 if (is_td_vcpu(vcpu)) 424 return 0; 425 426 return vmx_set_efer(vcpu, efer); 427 } 428 429 static void vt_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 430 { 431 if (is_td_vcpu(vcpu)) { 432 memset(dt, 0, sizeof(*dt)); 433 return; 434 } 435 436 vmx_get_idt(vcpu, dt); 437 } 438 439 static void vt_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 440 { 441 if (is_td_vcpu(vcpu)) 442 return; 443 444 vmx_set_idt(vcpu, dt); 445 } 446 447 static void vt_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 448 { 449 if (is_td_vcpu(vcpu)) { 450 memset(dt, 0, sizeof(*dt)); 451 return; 452 } 453 454 vmx_get_gdt(vcpu, dt); 455 } 456 457 static void vt_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 458 { 459 if (is_td_vcpu(vcpu)) 460 return; 461 462 vmx_set_gdt(vcpu, dt); 463 } 464 465 static void vt_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 466 { 467 if (is_td_vcpu(vcpu)) 468 return; 469 470 vmx_set_dr7(vcpu, val); 471 } 472 473 static void vt_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 474 { 475 /* 476 * MOV-DR exiting is always cleared for TD guest, even in debug mode. 477 * Thus KVM_DEBUGREG_WONT_EXIT can never be set and it should never 478 * reach here for TD vcpu. 479 */ 480 if (is_td_vcpu(vcpu)) 481 return; 482 483 vmx_sync_dirty_debug_regs(vcpu); 484 } 485 486 static void vt_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 487 { 488 if (WARN_ON_ONCE(is_td_vcpu(vcpu))) 489 return; 490 491 vmx_cache_reg(vcpu, reg); 492 } 493 494 static unsigned long vt_get_rflags(struct kvm_vcpu *vcpu) 495 { 496 if (is_td_vcpu(vcpu)) 497 return 0; 498 499 return vmx_get_rflags(vcpu); 500 } 501 502 static void vt_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 503 { 504 if (is_td_vcpu(vcpu)) 505 return; 506 507 vmx_set_rflags(vcpu, rflags); 508 } 509 510 static bool vt_get_if_flag(struct kvm_vcpu *vcpu) 511 { 512 if (is_td_vcpu(vcpu)) 513 return false; 514 515 return vmx_get_if_flag(vcpu); 516 } 517 518 static void vt_flush_tlb_all(struct kvm_vcpu *vcpu) 519 { 520 if (is_td_vcpu(vcpu)) { 521 tdx_flush_tlb_all(vcpu); 522 return; 523 } 524 525 vmx_flush_tlb_all(vcpu); 526 } 527 528 static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) 529 { 530 if (is_td_vcpu(vcpu)) { 531 tdx_flush_tlb_current(vcpu); 532 return; 533 } 534 535 vmx_flush_tlb_current(vcpu); 536 } 537 538 static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 539 { 540 if (is_td_vcpu(vcpu)) 541 return; 542 543 vmx_flush_tlb_gva(vcpu, addr); 544 } 545 546 static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) 547 { 548 if (is_td_vcpu(vcpu)) 549 return; 550 551 vmx_flush_tlb_guest(vcpu); 552 } 553 554 static void vt_inject_nmi(struct kvm_vcpu *vcpu) 555 { 556 if (is_td_vcpu(vcpu)) { 557 tdx_inject_nmi(vcpu); 558 return; 559 } 560 561 vmx_inject_nmi(vcpu); 562 } 563 564 static int vt_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 565 { 566 /* 567 * The TDX module manages NMI windows and NMI reinjection, and hides NMI 568 * blocking, all KVM can do is throw an NMI over the wall. 569 */ 570 if (is_td_vcpu(vcpu)) 571 return true; 572 573 return vmx_nmi_allowed(vcpu, for_injection); 574 } 575 576 static bool vt_get_nmi_mask(struct kvm_vcpu *vcpu) 577 { 578 /* 579 * KVM can't get NMI blocking status for TDX guest, assume NMIs are 580 * always unmasked. 581 */ 582 if (is_td_vcpu(vcpu)) 583 return false; 584 585 return vmx_get_nmi_mask(vcpu); 586 } 587 588 static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 589 { 590 if (is_td_vcpu(vcpu)) 591 return; 592 593 vmx_set_nmi_mask(vcpu, masked); 594 } 595 596 static void vt_enable_nmi_window(struct kvm_vcpu *vcpu) 597 { 598 /* Refer to the comments in tdx_inject_nmi(). */ 599 if (is_td_vcpu(vcpu)) 600 return; 601 602 vmx_enable_nmi_window(vcpu); 603 } 604 605 static void vt_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 606 int pgd_level) 607 { 608 if (is_td_vcpu(vcpu)) { 609 tdx_load_mmu_pgd(vcpu, root_hpa, pgd_level); 610 return; 611 } 612 613 vmx_load_mmu_pgd(vcpu, root_hpa, pgd_level); 614 } 615 616 static void vt_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 617 { 618 if (is_td_vcpu(vcpu)) 619 return; 620 621 vmx_set_interrupt_shadow(vcpu, mask); 622 } 623 624 static u32 vt_get_interrupt_shadow(struct kvm_vcpu *vcpu) 625 { 626 if (is_td_vcpu(vcpu)) 627 return 0; 628 629 return vmx_get_interrupt_shadow(vcpu); 630 } 631 632 static void vt_patch_hypercall(struct kvm_vcpu *vcpu, 633 unsigned char *hypercall) 634 { 635 /* 636 * Because guest memory is protected, guest can't be patched. TD kernel 637 * is modified to use TDG.VP.VMCALL for hypercall. 638 */ 639 if (is_td_vcpu(vcpu)) 640 return; 641 642 vmx_patch_hypercall(vcpu, hypercall); 643 } 644 645 static void vt_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 646 { 647 if (is_td_vcpu(vcpu)) 648 return; 649 650 vmx_inject_irq(vcpu, reinjected); 651 } 652 653 static void vt_inject_exception(struct kvm_vcpu *vcpu) 654 { 655 if (is_td_vcpu(vcpu)) 656 return; 657 658 vmx_inject_exception(vcpu); 659 } 660 661 static void vt_cancel_injection(struct kvm_vcpu *vcpu) 662 { 663 if (is_td_vcpu(vcpu)) 664 return; 665 666 vmx_cancel_injection(vcpu); 667 } 668 669 static int vt_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 670 { 671 if (is_td_vcpu(vcpu)) 672 return tdx_interrupt_allowed(vcpu); 673 674 return vmx_interrupt_allowed(vcpu, for_injection); 675 } 676 677 static void vt_enable_irq_window(struct kvm_vcpu *vcpu) 678 { 679 if (is_td_vcpu(vcpu)) 680 return; 681 682 vmx_enable_irq_window(vcpu); 683 } 684 685 static void vt_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) 686 { 687 *intr_info = 0; 688 *error_code = 0; 689 690 if (is_td_vcpu(vcpu)) 691 return; 692 693 vmx_get_entry_info(vcpu, intr_info, error_code); 694 } 695 696 static void vt_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 697 u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) 698 { 699 if (is_td_vcpu(vcpu)) { 700 tdx_get_exit_info(vcpu, reason, info1, info2, intr_info, 701 error_code); 702 return; 703 } 704 705 vmx_get_exit_info(vcpu, reason, info1, info2, intr_info, error_code); 706 } 707 708 static void vt_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 709 { 710 if (is_td_vcpu(vcpu)) 711 return; 712 713 vmx_update_cr8_intercept(vcpu, tpr, irr); 714 } 715 716 static void vt_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 717 { 718 if (is_td_vcpu(vcpu)) 719 return; 720 721 vmx_set_apic_access_page_addr(vcpu); 722 } 723 724 static void vt_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 725 { 726 if (is_td_vcpu(vcpu)) { 727 KVM_BUG_ON(!kvm_vcpu_apicv_active(vcpu), vcpu->kvm); 728 return; 729 } 730 731 vmx_refresh_apicv_exec_ctrl(vcpu); 732 } 733 734 static void vt_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 735 { 736 if (is_td_vcpu(vcpu)) 737 return; 738 739 vmx_load_eoi_exitmap(vcpu, eoi_exit_bitmap); 740 } 741 742 static int vt_set_tss_addr(struct kvm *kvm, unsigned int addr) 743 { 744 if (is_td(kvm)) 745 return 0; 746 747 return vmx_set_tss_addr(kvm, addr); 748 } 749 750 static int vt_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 751 { 752 if (is_td(kvm)) 753 return 0; 754 755 return vmx_set_identity_map_addr(kvm, ident_addr); 756 } 757 758 static u64 vt_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 759 { 760 /* TDX doesn't support L2 guest at the moment. */ 761 if (is_td_vcpu(vcpu)) 762 return 0; 763 764 return vmx_get_l2_tsc_offset(vcpu); 765 } 766 767 static u64 vt_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 768 { 769 /* TDX doesn't support L2 guest at the moment. */ 770 if (is_td_vcpu(vcpu)) 771 return 0; 772 773 return vmx_get_l2_tsc_multiplier(vcpu); 774 } 775 776 static void vt_write_tsc_offset(struct kvm_vcpu *vcpu) 777 { 778 /* In TDX, tsc offset can't be changed. */ 779 if (is_td_vcpu(vcpu)) 780 return; 781 782 vmx_write_tsc_offset(vcpu); 783 } 784 785 static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu) 786 { 787 /* In TDX, tsc multiplier can't be changed. */ 788 if (is_td_vcpu(vcpu)) 789 return; 790 791 vmx_write_tsc_multiplier(vcpu); 792 } 793 794 #ifdef CONFIG_X86_64 795 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 796 bool *expired) 797 { 798 /* VMX-preemption timer isn't available for TDX. */ 799 if (is_td_vcpu(vcpu)) 800 return -EINVAL; 801 802 return vmx_set_hv_timer(vcpu, guest_deadline_tsc, expired); 803 } 804 805 static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu) 806 { 807 /* VMX-preemption timer can't be set. See vt_set_hv_timer(). */ 808 if (is_td_vcpu(vcpu)) 809 return; 810 811 vmx_cancel_hv_timer(vcpu); 812 } 813 #endif 814 815 static void vt_setup_mce(struct kvm_vcpu *vcpu) 816 { 817 if (is_td_vcpu(vcpu)) 818 return; 819 820 vmx_setup_mce(vcpu); 821 } 822 823 static int vt_mem_enc_ioctl(struct kvm *kvm, void __user *argp) 824 { 825 if (!is_td(kvm)) 826 return -ENOTTY; 827 828 return tdx_vm_ioctl(kvm, argp); 829 } 830 831 static int vt_vcpu_mem_enc_ioctl(struct kvm_vcpu *vcpu, void __user *argp) 832 { 833 if (!is_td_vcpu(vcpu)) 834 return -EINVAL; 835 836 return tdx_vcpu_ioctl(vcpu, argp); 837 } 838 839 static int vt_vcpu_mem_enc_unlocked_ioctl(struct kvm_vcpu *vcpu, void __user *argp) 840 { 841 if (!is_td_vcpu(vcpu)) 842 return -EINVAL; 843 844 return tdx_vcpu_unlocked_ioctl(vcpu, argp); 845 } 846 847 static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, 848 bool is_private) 849 { 850 if (is_td(kvm)) 851 return tdx_gmem_max_mapping_level(kvm, pfn, is_private); 852 853 return 0; 854 } 855 856 #define vt_op(name) vt_##name 857 #define vt_op_tdx_only(name) vt_##name 858 #else /* CONFIG_KVM_INTEL_TDX */ 859 #define vt_op(name) vmx_##name 860 #define vt_op_tdx_only(name) NULL 861 #endif /* CONFIG_KVM_INTEL_TDX */ 862 863 #define VMX_REQUIRED_APICV_INHIBITS \ 864 (BIT(APICV_INHIBIT_REASON_DISABLED) | \ 865 BIT(APICV_INHIBIT_REASON_ABSENT) | \ 866 BIT(APICV_INHIBIT_REASON_HYPERV) | \ 867 BIT(APICV_INHIBIT_REASON_BLOCKIRQ) | \ 868 BIT(APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED) | \ 869 BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) | \ 870 BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED)) 871 872 struct kvm_x86_ops vt_x86_ops __initdata = { 873 .name = KBUILD_MODNAME, 874 875 .check_processor_compatibility = vmx_check_processor_compat, 876 877 .hardware_unsetup = vt_op(hardware_unsetup), 878 879 .enable_virtualization_cpu = vmx_enable_virtualization_cpu, 880 .disable_virtualization_cpu = vt_op(disable_virtualization_cpu), 881 .emergency_disable_virtualization_cpu = vmx_emergency_disable_virtualization_cpu, 882 883 .has_emulated_msr = vt_op(has_emulated_msr), 884 885 .vm_size = sizeof(struct kvm_vmx), 886 887 .vm_init = vt_op(vm_init), 888 .vm_destroy = vt_op(vm_destroy), 889 .vm_pre_destroy = vt_op_tdx_only(vm_pre_destroy), 890 891 .vcpu_precreate = vt_op(vcpu_precreate), 892 .vcpu_create = vt_op(vcpu_create), 893 .vcpu_free = vt_op(vcpu_free), 894 .vcpu_reset = vt_op(vcpu_reset), 895 896 .prepare_switch_to_guest = vt_op(prepare_switch_to_guest), 897 .vcpu_load = vt_op(vcpu_load), 898 .vcpu_put = vt_op(vcpu_put), 899 900 .HOST_OWNED_DEBUGCTL = VMX_HOST_OWNED_DEBUGCTL_BITS, 901 902 .update_exception_bitmap = vt_op(update_exception_bitmap), 903 .get_feature_msr = vmx_get_feature_msr, 904 .get_msr = vt_op(get_msr), 905 .set_msr = vt_op(set_msr), 906 907 .get_segment_base = vt_op(get_segment_base), 908 .get_segment = vt_op(get_segment), 909 .set_segment = vt_op(set_segment), 910 .get_cpl = vt_op(get_cpl), 911 .get_cpl_no_cache = vt_op(get_cpl_no_cache), 912 .get_cs_db_l_bits = vt_op(get_cs_db_l_bits), 913 .is_valid_cr0 = vt_op(is_valid_cr0), 914 .set_cr0 = vt_op(set_cr0), 915 .is_valid_cr4 = vt_op(is_valid_cr4), 916 .set_cr4 = vt_op(set_cr4), 917 .set_efer = vt_op(set_efer), 918 .get_idt = vt_op(get_idt), 919 .set_idt = vt_op(set_idt), 920 .get_gdt = vt_op(get_gdt), 921 .set_gdt = vt_op(set_gdt), 922 .set_dr7 = vt_op(set_dr7), 923 .sync_dirty_debug_regs = vt_op(sync_dirty_debug_regs), 924 .cache_reg = vt_op(cache_reg), 925 .get_rflags = vt_op(get_rflags), 926 .set_rflags = vt_op(set_rflags), 927 .get_if_flag = vt_op(get_if_flag), 928 929 .flush_tlb_all = vt_op(flush_tlb_all), 930 .flush_tlb_current = vt_op(flush_tlb_current), 931 .flush_tlb_gva = vt_op(flush_tlb_gva), 932 .flush_tlb_guest = vt_op(flush_tlb_guest), 933 934 .vcpu_pre_run = vt_op(vcpu_pre_run), 935 .vcpu_run = vt_op(vcpu_run), 936 .handle_exit = vt_op(handle_exit), 937 .skip_emulated_instruction = vmx_skip_emulated_instruction, 938 .update_emulated_instruction = vmx_update_emulated_instruction, 939 .set_interrupt_shadow = vt_op(set_interrupt_shadow), 940 .get_interrupt_shadow = vt_op(get_interrupt_shadow), 941 .patch_hypercall = vt_op(patch_hypercall), 942 .inject_irq = vt_op(inject_irq), 943 .inject_nmi = vt_op(inject_nmi), 944 .inject_exception = vt_op(inject_exception), 945 .cancel_injection = vt_op(cancel_injection), 946 .interrupt_allowed = vt_op(interrupt_allowed), 947 .nmi_allowed = vt_op(nmi_allowed), 948 .get_nmi_mask = vt_op(get_nmi_mask), 949 .set_nmi_mask = vt_op(set_nmi_mask), 950 .enable_nmi_window = vt_op(enable_nmi_window), 951 .enable_irq_window = vt_op(enable_irq_window), 952 .update_cr8_intercept = vt_op(update_cr8_intercept), 953 954 .x2apic_icr_is_split = false, 955 .set_virtual_apic_mode = vt_op(set_virtual_apic_mode), 956 .set_apic_access_page_addr = vt_op(set_apic_access_page_addr), 957 .refresh_apicv_exec_ctrl = vt_op(refresh_apicv_exec_ctrl), 958 .load_eoi_exitmap = vt_op(load_eoi_exitmap), 959 .apicv_pre_state_restore = pi_apicv_pre_state_restore, 960 .required_apicv_inhibits = VMX_REQUIRED_APICV_INHIBITS, 961 .hwapic_isr_update = vt_op(hwapic_isr_update), 962 .sync_pir_to_irr = vt_op(sync_pir_to_irr), 963 .deliver_interrupt = vt_op(deliver_interrupt), 964 .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, 965 966 .set_tss_addr = vt_op(set_tss_addr), 967 .set_identity_map_addr = vt_op(set_identity_map_addr), 968 .get_mt_mask = vmx_get_mt_mask, 969 970 .get_exit_info = vt_op(get_exit_info), 971 .get_entry_info = vt_op(get_entry_info), 972 973 .vcpu_after_set_cpuid = vt_op(vcpu_after_set_cpuid), 974 975 .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, 976 977 .get_l2_tsc_offset = vt_op(get_l2_tsc_offset), 978 .get_l2_tsc_multiplier = vt_op(get_l2_tsc_multiplier), 979 .write_tsc_offset = vt_op(write_tsc_offset), 980 .write_tsc_multiplier = vt_op(write_tsc_multiplier), 981 982 .load_mmu_pgd = vt_op(load_mmu_pgd), 983 984 .check_intercept = vmx_check_intercept, 985 .handle_exit_irqoff = vmx_handle_exit_irqoff, 986 987 .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), 988 989 .nested_ops = &vmx_nested_ops, 990 991 .pi_update_irte = vmx_pi_update_irte, 992 .pi_start_bypass = vmx_pi_start_bypass, 993 994 #ifdef CONFIG_X86_64 995 .set_hv_timer = vt_op(set_hv_timer), 996 .cancel_hv_timer = vt_op(cancel_hv_timer), 997 #endif 998 999 .setup_mce = vt_op(setup_mce), 1000 1001 #ifdef CONFIG_KVM_SMM 1002 .smi_allowed = vt_op(smi_allowed), 1003 .enter_smm = vt_op(enter_smm), 1004 .leave_smm = vt_op(leave_smm), 1005 .enable_smi_window = vt_op(enable_smi_window), 1006 #endif 1007 1008 .check_emulate_instruction = vt_op(check_emulate_instruction), 1009 .apic_init_signal_blocked = vt_op(apic_init_signal_blocked), 1010 .migrate_timers = vmx_migrate_timers, 1011 1012 .recalc_intercepts = vt_op(recalc_intercepts), 1013 .complete_emulated_msr = vt_op(complete_emulated_msr), 1014 1015 .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, 1016 1017 .get_untagged_addr = vmx_get_untagged_addr, 1018 1019 .mem_enc_ioctl = vt_op_tdx_only(mem_enc_ioctl), 1020 .vcpu_mem_enc_ioctl = vt_op_tdx_only(vcpu_mem_enc_ioctl), 1021 .vcpu_mem_enc_unlocked_ioctl = vt_op_tdx_only(vcpu_mem_enc_unlocked_ioctl), 1022 1023 .gmem_max_mapping_level = vt_op_tdx_only(gmem_max_mapping_level) 1024 }; 1025 1026 struct kvm_x86_init_ops vt_init_ops __initdata = { 1027 .hardware_setup = vt_op(hardware_setup), 1028 .handle_intel_pt_intr = NULL, 1029 1030 .runtime_ops = &vt_x86_ops, 1031 .pmu_ops = &intel_pmu_ops, 1032 }; 1033 1034 static void __exit vt_exit(void) 1035 { 1036 kvm_exit(); 1037 vmx_exit(); 1038 } 1039 module_exit(vt_exit); 1040 1041 static int __init vt_init(void) 1042 { 1043 unsigned vcpu_size, vcpu_align; 1044 int r; 1045 1046 r = vmx_init(); 1047 if (r) 1048 return r; 1049 1050 /* 1051 * TDX and VMX have different vCPU structures. Calculate the 1052 * maximum size/align so that kvm_init() can use the larger 1053 * values to create the kmem_vcpu_cache. 1054 */ 1055 vcpu_size = sizeof(struct vcpu_vmx); 1056 vcpu_align = __alignof__(struct vcpu_vmx); 1057 if (enable_tdx) { 1058 vcpu_size = max_t(unsigned, vcpu_size, 1059 sizeof(struct vcpu_tdx)); 1060 vcpu_align = max_t(unsigned, vcpu_align, 1061 __alignof__(struct vcpu_tdx)); 1062 kvm_caps.supported_vm_types |= BIT(KVM_X86_TDX_VM); 1063 } 1064 1065 /* 1066 * Common KVM initialization _must_ come last, after this, /dev/kvm is 1067 * exposed to userspace! 1068 */ 1069 r = kvm_init(vcpu_size, vcpu_align, THIS_MODULE); 1070 if (r) 1071 goto err_kvm_init; 1072 1073 return 0; 1074 1075 err_kvm_init: 1076 vmx_exit(); 1077 return r; 1078 } 1079 module_init(vt_init); 1080