1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/moduleparam.h> 3 4 #include "x86_ops.h" 5 #include "vmx.h" 6 #include "mmu.h" 7 #include "nested.h" 8 #include "pmu.h" 9 #include "posted_intr.h" 10 #include "tdx.h" 11 #include "tdx_arch.h" 12 13 #ifdef CONFIG_KVM_INTEL_TDX 14 static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); 15 16 static void vt_disable_virtualization_cpu(void) 17 { 18 /* Note, TDX *and* VMX need to be disabled if TDX is enabled. */ 19 if (enable_tdx) 20 tdx_disable_virtualization_cpu(); 21 vmx_disable_virtualization_cpu(); 22 } 23 24 static __init int vt_hardware_setup(void) 25 { 26 int ret; 27 28 ret = vmx_hardware_setup(); 29 if (ret) 30 return ret; 31 32 return enable_tdx ? tdx_hardware_setup() : 0; 33 } 34 35 static void vt_hardware_unsetup(void) 36 { 37 if (enable_tdx) 38 tdx_hardware_unsetup(); 39 40 vmx_hardware_unsetup(); 41 } 42 43 static int vt_vm_init(struct kvm *kvm) 44 { 45 if (is_td(kvm)) 46 return tdx_vm_init(kvm); 47 48 return vmx_vm_init(kvm); 49 } 50 51 static void vt_vm_pre_destroy(struct kvm *kvm) 52 { 53 if (is_td(kvm)) 54 return tdx_mmu_release_hkid(kvm); 55 } 56 57 static void vt_vm_destroy(struct kvm *kvm) 58 { 59 if (is_td(kvm)) 60 return tdx_vm_destroy(kvm); 61 62 vmx_vm_destroy(kvm); 63 } 64 65 static int vt_vcpu_precreate(struct kvm *kvm) 66 { 67 if (is_td(kvm)) 68 return 0; 69 70 return vmx_vcpu_precreate(kvm); 71 } 72 73 static int vt_vcpu_create(struct kvm_vcpu *vcpu) 74 { 75 if (is_td_vcpu(vcpu)) 76 return tdx_vcpu_create(vcpu); 77 78 return vmx_vcpu_create(vcpu); 79 } 80 81 static void vt_vcpu_free(struct kvm_vcpu *vcpu) 82 { 83 if (is_td_vcpu(vcpu)) { 84 tdx_vcpu_free(vcpu); 85 return; 86 } 87 88 vmx_vcpu_free(vcpu); 89 } 90 91 static void vt_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 92 { 93 if (is_td_vcpu(vcpu)) { 94 tdx_vcpu_reset(vcpu, init_event); 95 return; 96 } 97 98 vmx_vcpu_reset(vcpu, init_event); 99 } 100 101 static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 102 { 103 if (is_td_vcpu(vcpu)) { 104 tdx_vcpu_load(vcpu, cpu); 105 return; 106 } 107 108 vmx_vcpu_load(vcpu, cpu); 109 } 110 111 static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 112 { 113 /* 114 * Basic TDX does not support feature PML. KVM does not enable PML in 115 * TD's VMCS, nor does it allocate or flush PML buffer for TDX. 116 */ 117 if (WARN_ON_ONCE(is_td_vcpu(vcpu))) 118 return; 119 120 vmx_update_cpu_dirty_logging(vcpu); 121 } 122 123 static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 124 { 125 if (is_td_vcpu(vcpu)) { 126 tdx_prepare_switch_to_guest(vcpu); 127 return; 128 } 129 130 vmx_prepare_switch_to_guest(vcpu); 131 } 132 133 static void vt_vcpu_put(struct kvm_vcpu *vcpu) 134 { 135 if (is_td_vcpu(vcpu)) { 136 tdx_vcpu_put(vcpu); 137 return; 138 } 139 140 vmx_vcpu_put(vcpu); 141 } 142 143 static int vt_vcpu_pre_run(struct kvm_vcpu *vcpu) 144 { 145 if (is_td_vcpu(vcpu)) 146 return tdx_vcpu_pre_run(vcpu); 147 148 return vmx_vcpu_pre_run(vcpu); 149 } 150 151 static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) 152 { 153 if (is_td_vcpu(vcpu)) 154 return tdx_vcpu_run(vcpu, run_flags); 155 156 return vmx_vcpu_run(vcpu, run_flags); 157 } 158 159 static int vt_handle_exit(struct kvm_vcpu *vcpu, 160 enum exit_fastpath_completion fastpath) 161 { 162 if (is_td_vcpu(vcpu)) 163 return tdx_handle_exit(vcpu, fastpath); 164 165 return vmx_handle_exit(vcpu, fastpath); 166 } 167 168 static int vt_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 169 { 170 if (unlikely(is_td_vcpu(vcpu))) 171 return tdx_set_msr(vcpu, msr_info); 172 173 return vmx_set_msr(vcpu, msr_info); 174 } 175 176 /* 177 * The kvm parameter can be NULL (module initialization, or invocation before 178 * VM creation). Be sure to check the kvm parameter before using it. 179 */ 180 static bool vt_has_emulated_msr(struct kvm *kvm, u32 index) 181 { 182 if (kvm && is_td(kvm)) 183 return tdx_has_emulated_msr(index); 184 185 return vmx_has_emulated_msr(kvm, index); 186 } 187 188 static int vt_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 189 { 190 if (unlikely(is_td_vcpu(vcpu))) 191 return tdx_get_msr(vcpu, msr_info); 192 193 return vmx_get_msr(vcpu, msr_info); 194 } 195 196 static void vt_recalc_intercepts(struct kvm_vcpu *vcpu) 197 { 198 /* 199 * TDX doesn't allow VMM to configure interception of instructions or 200 * MSR accesses. TDX guest requests MSR accesses by calling TDVMCALL. 201 * The MSR filters will be applied when handling the TDVMCALL for 202 * RDMSR/WRMSR if the userspace has set any. 203 */ 204 if (is_td_vcpu(vcpu)) 205 return; 206 207 vmx_recalc_intercepts(vcpu); 208 } 209 210 static int vt_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) 211 { 212 if (is_td_vcpu(vcpu)) 213 return tdx_complete_emulated_msr(vcpu, err); 214 215 return vmx_complete_emulated_msr(vcpu, err); 216 } 217 218 #ifdef CONFIG_KVM_SMM 219 static int vt_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 220 { 221 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 222 return 0; 223 224 return vmx_smi_allowed(vcpu, for_injection); 225 } 226 227 static int vt_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 228 { 229 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 230 return 0; 231 232 return vmx_enter_smm(vcpu, smram); 233 } 234 235 static int vt_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 236 { 237 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 238 return 0; 239 240 return vmx_leave_smm(vcpu, smram); 241 } 242 243 static void vt_enable_smi_window(struct kvm_vcpu *vcpu) 244 { 245 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 246 return; 247 248 /* RSM will cause a vmexit anyway. */ 249 vmx_enable_smi_window(vcpu); 250 } 251 #endif 252 253 static int vt_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 254 void *insn, int insn_len) 255 { 256 /* 257 * For TDX, this can only be triggered for MMIO emulation. Let the 258 * guest retry after installing the SPTE with suppress #VE bit cleared, 259 * so that the guest will receive #VE when retry. The guest is expected 260 * to call TDG.VP.VMCALL<MMIO> to request VMM to do MMIO emulation on 261 * #VE. 262 */ 263 if (is_td_vcpu(vcpu)) 264 return X86EMUL_RETRY_INSTR; 265 266 return vmx_check_emulate_instruction(vcpu, emul_type, insn, insn_len); 267 } 268 269 static bool vt_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 270 { 271 /* 272 * INIT and SIPI are always blocked for TDX, i.e., INIT handling and 273 * the OP vcpu_deliver_sipi_vector() won't be called. 274 */ 275 if (is_td_vcpu(vcpu)) 276 return true; 277 278 return vmx_apic_init_signal_blocked(vcpu); 279 } 280 281 static void vt_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 282 { 283 /* Only x2APIC mode is supported for TD. */ 284 if (is_td_vcpu(vcpu)) 285 return; 286 287 return vmx_set_virtual_apic_mode(vcpu); 288 } 289 290 static void vt_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 291 { 292 if (is_td_vcpu(vcpu)) 293 return; 294 295 return vmx_hwapic_isr_update(vcpu, max_isr); 296 } 297 298 static int vt_sync_pir_to_irr(struct kvm_vcpu *vcpu) 299 { 300 if (is_td_vcpu(vcpu)) 301 return -1; 302 303 return vmx_sync_pir_to_irr(vcpu); 304 } 305 306 static void vt_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 307 int trig_mode, int vector) 308 { 309 if (is_td_vcpu(apic->vcpu)) { 310 tdx_deliver_interrupt(apic, delivery_mode, trig_mode, 311 vector); 312 return; 313 } 314 315 vmx_deliver_interrupt(apic, delivery_mode, trig_mode, vector); 316 } 317 318 static void vt_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 319 { 320 if (is_td_vcpu(vcpu)) 321 return; 322 323 vmx_vcpu_after_set_cpuid(vcpu); 324 } 325 326 static void vt_update_exception_bitmap(struct kvm_vcpu *vcpu) 327 { 328 if (is_td_vcpu(vcpu)) 329 return; 330 331 vmx_update_exception_bitmap(vcpu); 332 } 333 334 static u64 vt_get_segment_base(struct kvm_vcpu *vcpu, int seg) 335 { 336 if (is_td_vcpu(vcpu)) 337 return 0; 338 339 return vmx_get_segment_base(vcpu, seg); 340 } 341 342 static void vt_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, 343 int seg) 344 { 345 if (is_td_vcpu(vcpu)) { 346 memset(var, 0, sizeof(*var)); 347 return; 348 } 349 350 vmx_get_segment(vcpu, var, seg); 351 } 352 353 static void vt_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, 354 int seg) 355 { 356 if (is_td_vcpu(vcpu)) 357 return; 358 359 vmx_set_segment(vcpu, var, seg); 360 } 361 362 static int vt_get_cpl(struct kvm_vcpu *vcpu) 363 { 364 if (is_td_vcpu(vcpu)) 365 return 0; 366 367 return vmx_get_cpl(vcpu); 368 } 369 370 static int vt_get_cpl_no_cache(struct kvm_vcpu *vcpu) 371 { 372 if (is_td_vcpu(vcpu)) 373 return 0; 374 375 return vmx_get_cpl_no_cache(vcpu); 376 } 377 378 static void vt_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 379 { 380 if (is_td_vcpu(vcpu)) { 381 *db = 0; 382 *l = 0; 383 return; 384 } 385 386 vmx_get_cs_db_l_bits(vcpu, db, l); 387 } 388 389 static bool vt_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 390 { 391 if (is_td_vcpu(vcpu)) 392 return true; 393 394 return vmx_is_valid_cr0(vcpu, cr0); 395 } 396 397 static void vt_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 398 { 399 if (is_td_vcpu(vcpu)) 400 return; 401 402 vmx_set_cr0(vcpu, cr0); 403 } 404 405 static bool vt_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 406 { 407 if (is_td_vcpu(vcpu)) 408 return true; 409 410 return vmx_is_valid_cr4(vcpu, cr4); 411 } 412 413 static void vt_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 414 { 415 if (is_td_vcpu(vcpu)) 416 return; 417 418 vmx_set_cr4(vcpu, cr4); 419 } 420 421 static int vt_set_efer(struct kvm_vcpu *vcpu, u64 efer) 422 { 423 if (is_td_vcpu(vcpu)) 424 return 0; 425 426 return vmx_set_efer(vcpu, efer); 427 } 428 429 static void vt_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 430 { 431 if (is_td_vcpu(vcpu)) { 432 memset(dt, 0, sizeof(*dt)); 433 return; 434 } 435 436 vmx_get_idt(vcpu, dt); 437 } 438 439 static void vt_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 440 { 441 if (is_td_vcpu(vcpu)) 442 return; 443 444 vmx_set_idt(vcpu, dt); 445 } 446 447 static void vt_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 448 { 449 if (is_td_vcpu(vcpu)) { 450 memset(dt, 0, sizeof(*dt)); 451 return; 452 } 453 454 vmx_get_gdt(vcpu, dt); 455 } 456 457 static void vt_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 458 { 459 if (is_td_vcpu(vcpu)) 460 return; 461 462 vmx_set_gdt(vcpu, dt); 463 } 464 465 static void vt_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 466 { 467 if (is_td_vcpu(vcpu)) 468 return; 469 470 vmx_set_dr7(vcpu, val); 471 } 472 473 static void vt_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 474 { 475 /* 476 * MOV-DR exiting is always cleared for TD guest, even in debug mode. 477 * Thus KVM_DEBUGREG_WONT_EXIT can never be set and it should never 478 * reach here for TD vcpu. 479 */ 480 if (is_td_vcpu(vcpu)) 481 return; 482 483 vmx_sync_dirty_debug_regs(vcpu); 484 } 485 486 static void vt_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 487 { 488 if (WARN_ON_ONCE(is_td_vcpu(vcpu))) 489 return; 490 491 vmx_cache_reg(vcpu, reg); 492 } 493 494 static unsigned long vt_get_rflags(struct kvm_vcpu *vcpu) 495 { 496 if (is_td_vcpu(vcpu)) 497 return 0; 498 499 return vmx_get_rflags(vcpu); 500 } 501 502 static void vt_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 503 { 504 if (is_td_vcpu(vcpu)) 505 return; 506 507 vmx_set_rflags(vcpu, rflags); 508 } 509 510 static bool vt_get_if_flag(struct kvm_vcpu *vcpu) 511 { 512 if (is_td_vcpu(vcpu)) 513 return false; 514 515 return vmx_get_if_flag(vcpu); 516 } 517 518 static void vt_flush_tlb_all(struct kvm_vcpu *vcpu) 519 { 520 if (is_td_vcpu(vcpu)) { 521 tdx_flush_tlb_all(vcpu); 522 return; 523 } 524 525 vmx_flush_tlb_all(vcpu); 526 } 527 528 static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) 529 { 530 if (is_td_vcpu(vcpu)) { 531 tdx_flush_tlb_current(vcpu); 532 return; 533 } 534 535 vmx_flush_tlb_current(vcpu); 536 } 537 538 static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 539 { 540 if (is_td_vcpu(vcpu)) 541 return; 542 543 vmx_flush_tlb_gva(vcpu, addr); 544 } 545 546 static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) 547 { 548 if (is_td_vcpu(vcpu)) 549 return; 550 551 vmx_flush_tlb_guest(vcpu); 552 } 553 554 static void vt_inject_nmi(struct kvm_vcpu *vcpu) 555 { 556 if (is_td_vcpu(vcpu)) { 557 tdx_inject_nmi(vcpu); 558 return; 559 } 560 561 vmx_inject_nmi(vcpu); 562 } 563 564 static int vt_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 565 { 566 /* 567 * The TDX module manages NMI windows and NMI reinjection, and hides NMI 568 * blocking, all KVM can do is throw an NMI over the wall. 569 */ 570 if (is_td_vcpu(vcpu)) 571 return true; 572 573 return vmx_nmi_allowed(vcpu, for_injection); 574 } 575 576 static bool vt_get_nmi_mask(struct kvm_vcpu *vcpu) 577 { 578 /* 579 * KVM can't get NMI blocking status for TDX guest, assume NMIs are 580 * always unmasked. 581 */ 582 if (is_td_vcpu(vcpu)) 583 return false; 584 585 return vmx_get_nmi_mask(vcpu); 586 } 587 588 static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 589 { 590 if (is_td_vcpu(vcpu)) 591 return; 592 593 vmx_set_nmi_mask(vcpu, masked); 594 } 595 596 static void vt_enable_nmi_window(struct kvm_vcpu *vcpu) 597 { 598 /* Refer to the comments in tdx_inject_nmi(). */ 599 if (is_td_vcpu(vcpu)) 600 return; 601 602 vmx_enable_nmi_window(vcpu); 603 } 604 605 static void vt_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 606 int pgd_level) 607 { 608 if (is_td_vcpu(vcpu)) { 609 tdx_load_mmu_pgd(vcpu, root_hpa, pgd_level); 610 return; 611 } 612 613 vmx_load_mmu_pgd(vcpu, root_hpa, pgd_level); 614 } 615 616 static void vt_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 617 { 618 if (is_td_vcpu(vcpu)) 619 return; 620 621 vmx_set_interrupt_shadow(vcpu, mask); 622 } 623 624 static u32 vt_get_interrupt_shadow(struct kvm_vcpu *vcpu) 625 { 626 if (is_td_vcpu(vcpu)) 627 return 0; 628 629 return vmx_get_interrupt_shadow(vcpu); 630 } 631 632 static void vt_patch_hypercall(struct kvm_vcpu *vcpu, 633 unsigned char *hypercall) 634 { 635 /* 636 * Because guest memory is protected, guest can't be patched. TD kernel 637 * is modified to use TDG.VP.VMCALL for hypercall. 638 */ 639 if (is_td_vcpu(vcpu)) 640 return; 641 642 vmx_patch_hypercall(vcpu, hypercall); 643 } 644 645 static void vt_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 646 { 647 if (is_td_vcpu(vcpu)) 648 return; 649 650 vmx_inject_irq(vcpu, reinjected); 651 } 652 653 static void vt_inject_exception(struct kvm_vcpu *vcpu) 654 { 655 if (is_td_vcpu(vcpu)) 656 return; 657 658 vmx_inject_exception(vcpu); 659 } 660 661 static void vt_cancel_injection(struct kvm_vcpu *vcpu) 662 { 663 if (is_td_vcpu(vcpu)) 664 return; 665 666 vmx_cancel_injection(vcpu); 667 } 668 669 static int vt_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 670 { 671 if (is_td_vcpu(vcpu)) 672 return tdx_interrupt_allowed(vcpu); 673 674 return vmx_interrupt_allowed(vcpu, for_injection); 675 } 676 677 static void vt_enable_irq_window(struct kvm_vcpu *vcpu) 678 { 679 if (is_td_vcpu(vcpu)) 680 return; 681 682 vmx_enable_irq_window(vcpu); 683 } 684 685 static void vt_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) 686 { 687 *intr_info = 0; 688 *error_code = 0; 689 690 if (is_td_vcpu(vcpu)) 691 return; 692 693 vmx_get_entry_info(vcpu, intr_info, error_code); 694 } 695 696 static void vt_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 697 u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) 698 { 699 if (is_td_vcpu(vcpu)) { 700 tdx_get_exit_info(vcpu, reason, info1, info2, intr_info, 701 error_code); 702 return; 703 } 704 705 vmx_get_exit_info(vcpu, reason, info1, info2, intr_info, error_code); 706 } 707 708 static void vt_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 709 { 710 if (is_td_vcpu(vcpu)) 711 return; 712 713 vmx_update_cr8_intercept(vcpu, tpr, irr); 714 } 715 716 static void vt_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 717 { 718 if (is_td_vcpu(vcpu)) 719 return; 720 721 vmx_set_apic_access_page_addr(vcpu); 722 } 723 724 static void vt_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 725 { 726 if (is_td_vcpu(vcpu)) { 727 KVM_BUG_ON(!kvm_vcpu_apicv_active(vcpu), vcpu->kvm); 728 return; 729 } 730 731 vmx_refresh_apicv_exec_ctrl(vcpu); 732 } 733 734 static void vt_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 735 { 736 if (is_td_vcpu(vcpu)) 737 return; 738 739 vmx_load_eoi_exitmap(vcpu, eoi_exit_bitmap); 740 } 741 742 static int vt_set_tss_addr(struct kvm *kvm, unsigned int addr) 743 { 744 if (is_td(kvm)) 745 return 0; 746 747 return vmx_set_tss_addr(kvm, addr); 748 } 749 750 static int vt_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 751 { 752 if (is_td(kvm)) 753 return 0; 754 755 return vmx_set_identity_map_addr(kvm, ident_addr); 756 } 757 758 static bool vt_tdp_has_smep(struct kvm *kvm) 759 { 760 if (is_td(kvm)) 761 return false; 762 763 return vmx_tdp_has_smep(kvm); 764 } 765 766 static u64 vt_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 767 { 768 /* TDX doesn't support L2 guest at the moment. */ 769 if (is_td_vcpu(vcpu)) 770 return 0; 771 772 return vmx_get_l2_tsc_offset(vcpu); 773 } 774 775 static u64 vt_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 776 { 777 /* TDX doesn't support L2 guest at the moment. */ 778 if (is_td_vcpu(vcpu)) 779 return 0; 780 781 return vmx_get_l2_tsc_multiplier(vcpu); 782 } 783 784 static void vt_write_tsc_offset(struct kvm_vcpu *vcpu) 785 { 786 /* In TDX, tsc offset can't be changed. */ 787 if (is_td_vcpu(vcpu)) 788 return; 789 790 vmx_write_tsc_offset(vcpu); 791 } 792 793 static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu) 794 { 795 /* In TDX, tsc multiplier can't be changed. */ 796 if (is_td_vcpu(vcpu)) 797 return; 798 799 vmx_write_tsc_multiplier(vcpu); 800 } 801 802 #ifdef CONFIG_X86_64 803 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 804 bool *expired) 805 { 806 /* VMX-preemption timer isn't available for TDX. */ 807 if (is_td_vcpu(vcpu)) 808 return -EINVAL; 809 810 return vmx_set_hv_timer(vcpu, guest_deadline_tsc, expired); 811 } 812 813 static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu) 814 { 815 /* VMX-preemption timer can't be set. See vt_set_hv_timer(). */ 816 if (is_td_vcpu(vcpu)) 817 return; 818 819 vmx_cancel_hv_timer(vcpu); 820 } 821 #endif 822 823 static void vt_setup_mce(struct kvm_vcpu *vcpu) 824 { 825 if (is_td_vcpu(vcpu)) 826 return; 827 828 vmx_setup_mce(vcpu); 829 } 830 831 static int vt_mem_enc_ioctl(struct kvm *kvm, void __user *argp) 832 { 833 if (!is_td(kvm)) 834 return -ENOTTY; 835 836 return tdx_vm_ioctl(kvm, argp); 837 } 838 839 static int vt_vcpu_mem_enc_ioctl(struct kvm_vcpu *vcpu, void __user *argp) 840 { 841 if (!is_td_vcpu(vcpu)) 842 return -EINVAL; 843 844 return tdx_vcpu_ioctl(vcpu, argp); 845 } 846 847 static int vt_vcpu_mem_enc_unlocked_ioctl(struct kvm_vcpu *vcpu, void __user *argp) 848 { 849 if (!is_td_vcpu(vcpu)) 850 return -EINVAL; 851 852 return tdx_vcpu_unlocked_ioctl(vcpu, argp); 853 } 854 855 static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, 856 bool is_private) 857 { 858 if (is_td(kvm)) 859 return tdx_gmem_max_mapping_level(kvm, pfn, is_private); 860 861 return 0; 862 } 863 864 #define vt_op(name) vt_##name 865 #define vt_op_tdx_only(name) vt_##name 866 #else /* CONFIG_KVM_INTEL_TDX */ 867 #define vt_op(name) vmx_##name 868 #define vt_op_tdx_only(name) NULL 869 #endif /* CONFIG_KVM_INTEL_TDX */ 870 871 #define VMX_REQUIRED_APICV_INHIBITS \ 872 (BIT(APICV_INHIBIT_REASON_DISABLED) | \ 873 BIT(APICV_INHIBIT_REASON_ABSENT) | \ 874 BIT(APICV_INHIBIT_REASON_HYPERV) | \ 875 BIT(APICV_INHIBIT_REASON_BLOCKIRQ) | \ 876 BIT(APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED) | \ 877 BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) | \ 878 BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED)) 879 880 struct kvm_x86_ops vt_x86_ops __initdata = { 881 .name = KBUILD_MODNAME, 882 883 .check_processor_compatibility = vmx_check_processor_compat, 884 885 .hardware_unsetup = vt_op(hardware_unsetup), 886 887 .enable_virtualization_cpu = vmx_enable_virtualization_cpu, 888 .disable_virtualization_cpu = vt_op(disable_virtualization_cpu), 889 .emergency_disable_virtualization_cpu = vmx_emergency_disable_virtualization_cpu, 890 891 .has_emulated_msr = vt_op(has_emulated_msr), 892 893 .vm_size = sizeof(struct kvm_vmx), 894 895 .vm_init = vt_op(vm_init), 896 .vm_destroy = vt_op(vm_destroy), 897 .vm_pre_destroy = vt_op_tdx_only(vm_pre_destroy), 898 899 .vcpu_precreate = vt_op(vcpu_precreate), 900 .vcpu_create = vt_op(vcpu_create), 901 .vcpu_free = vt_op(vcpu_free), 902 .vcpu_reset = vt_op(vcpu_reset), 903 904 .prepare_switch_to_guest = vt_op(prepare_switch_to_guest), 905 .vcpu_load = vt_op(vcpu_load), 906 .vcpu_put = vt_op(vcpu_put), 907 908 .HOST_OWNED_DEBUGCTL = VMX_HOST_OWNED_DEBUGCTL_BITS, 909 910 .update_exception_bitmap = vt_op(update_exception_bitmap), 911 .get_feature_msr = vmx_get_feature_msr, 912 .get_msr = vt_op(get_msr), 913 .set_msr = vt_op(set_msr), 914 915 .get_segment_base = vt_op(get_segment_base), 916 .get_segment = vt_op(get_segment), 917 .set_segment = vt_op(set_segment), 918 .get_cpl = vt_op(get_cpl), 919 .get_cpl_no_cache = vt_op(get_cpl_no_cache), 920 .get_cs_db_l_bits = vt_op(get_cs_db_l_bits), 921 .is_valid_cr0 = vt_op(is_valid_cr0), 922 .set_cr0 = vt_op(set_cr0), 923 .is_valid_cr4 = vt_op(is_valid_cr4), 924 .set_cr4 = vt_op(set_cr4), 925 .set_efer = vt_op(set_efer), 926 .get_idt = vt_op(get_idt), 927 .set_idt = vt_op(set_idt), 928 .get_gdt = vt_op(get_gdt), 929 .set_gdt = vt_op(set_gdt), 930 .set_dr7 = vt_op(set_dr7), 931 .sync_dirty_debug_regs = vt_op(sync_dirty_debug_regs), 932 .cache_reg = vt_op(cache_reg), 933 .get_rflags = vt_op(get_rflags), 934 .set_rflags = vt_op(set_rflags), 935 .get_if_flag = vt_op(get_if_flag), 936 937 .flush_tlb_all = vt_op(flush_tlb_all), 938 .flush_tlb_current = vt_op(flush_tlb_current), 939 .flush_tlb_gva = vt_op(flush_tlb_gva), 940 .flush_tlb_guest = vt_op(flush_tlb_guest), 941 942 .vcpu_pre_run = vt_op(vcpu_pre_run), 943 .vcpu_run = vt_op(vcpu_run), 944 .handle_exit = vt_op(handle_exit), 945 .skip_emulated_instruction = vmx_skip_emulated_instruction, 946 .update_emulated_instruction = vmx_update_emulated_instruction, 947 .set_interrupt_shadow = vt_op(set_interrupt_shadow), 948 .get_interrupt_shadow = vt_op(get_interrupt_shadow), 949 .patch_hypercall = vt_op(patch_hypercall), 950 .inject_irq = vt_op(inject_irq), 951 .inject_nmi = vt_op(inject_nmi), 952 .inject_exception = vt_op(inject_exception), 953 .cancel_injection = vt_op(cancel_injection), 954 .interrupt_allowed = vt_op(interrupt_allowed), 955 .nmi_allowed = vt_op(nmi_allowed), 956 .get_nmi_mask = vt_op(get_nmi_mask), 957 .set_nmi_mask = vt_op(set_nmi_mask), 958 .enable_nmi_window = vt_op(enable_nmi_window), 959 .enable_irq_window = vt_op(enable_irq_window), 960 .update_cr8_intercept = vt_op(update_cr8_intercept), 961 962 .x2apic_icr_is_split = false, 963 .set_virtual_apic_mode = vt_op(set_virtual_apic_mode), 964 .set_apic_access_page_addr = vt_op(set_apic_access_page_addr), 965 .refresh_apicv_exec_ctrl = vt_op(refresh_apicv_exec_ctrl), 966 .load_eoi_exitmap = vt_op(load_eoi_exitmap), 967 .apicv_pre_state_restore = pi_apicv_pre_state_restore, 968 .required_apicv_inhibits = VMX_REQUIRED_APICV_INHIBITS, 969 .hwapic_isr_update = vt_op(hwapic_isr_update), 970 .sync_pir_to_irr = vt_op(sync_pir_to_irr), 971 .deliver_interrupt = vt_op(deliver_interrupt), 972 .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, 973 974 .set_tss_addr = vt_op(set_tss_addr), 975 .set_identity_map_addr = vt_op(set_identity_map_addr), 976 .get_mt_mask = vmx_get_mt_mask, 977 .tdp_has_smep = vt_op(tdp_has_smep), 978 979 .get_exit_info = vt_op(get_exit_info), 980 .get_entry_info = vt_op(get_entry_info), 981 982 .vcpu_after_set_cpuid = vt_op(vcpu_after_set_cpuid), 983 984 .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, 985 986 .get_l2_tsc_offset = vt_op(get_l2_tsc_offset), 987 .get_l2_tsc_multiplier = vt_op(get_l2_tsc_multiplier), 988 .write_tsc_offset = vt_op(write_tsc_offset), 989 .write_tsc_multiplier = vt_op(write_tsc_multiplier), 990 991 .load_mmu_pgd = vt_op(load_mmu_pgd), 992 993 .check_intercept = vmx_check_intercept, 994 .handle_exit_irqoff = vmx_handle_exit_irqoff, 995 996 .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), 997 998 .nested_ops = &vmx_nested_ops, 999 1000 .pi_update_irte = vmx_pi_update_irte, 1001 .pi_start_bypass = vmx_pi_start_bypass, 1002 1003 #ifdef CONFIG_X86_64 1004 .set_hv_timer = vt_op(set_hv_timer), 1005 .cancel_hv_timer = vt_op(cancel_hv_timer), 1006 #endif 1007 1008 .setup_mce = vt_op(setup_mce), 1009 1010 #ifdef CONFIG_KVM_SMM 1011 .smi_allowed = vt_op(smi_allowed), 1012 .enter_smm = vt_op(enter_smm), 1013 .leave_smm = vt_op(leave_smm), 1014 .enable_smi_window = vt_op(enable_smi_window), 1015 #endif 1016 1017 .check_emulate_instruction = vt_op(check_emulate_instruction), 1018 .apic_init_signal_blocked = vt_op(apic_init_signal_blocked), 1019 .migrate_timers = vmx_migrate_timers, 1020 1021 .recalc_intercepts = vt_op(recalc_intercepts), 1022 .complete_emulated_msr = vt_op(complete_emulated_msr), 1023 1024 .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, 1025 1026 .get_untagged_addr = vmx_get_untagged_addr, 1027 1028 .mem_enc_ioctl = vt_op_tdx_only(mem_enc_ioctl), 1029 .vcpu_mem_enc_ioctl = vt_op_tdx_only(vcpu_mem_enc_ioctl), 1030 .vcpu_mem_enc_unlocked_ioctl = vt_op_tdx_only(vcpu_mem_enc_unlocked_ioctl), 1031 1032 .gmem_max_mapping_level = vt_op_tdx_only(gmem_max_mapping_level) 1033 }; 1034 1035 struct kvm_x86_init_ops vt_init_ops __initdata = { 1036 .hardware_setup = vt_op(hardware_setup), 1037 .handle_intel_pt_intr = NULL, 1038 1039 .runtime_ops = &vt_x86_ops, 1040 .pmu_ops = &intel_pmu_ops, 1041 }; 1042 1043 static void __exit vt_exit(void) 1044 { 1045 kvm_exit(); 1046 vmx_exit(); 1047 } 1048 module_exit(vt_exit); 1049 1050 static int __init vt_init(void) 1051 { 1052 unsigned vcpu_size, vcpu_align; 1053 int r; 1054 1055 r = vmx_init(); 1056 if (r) 1057 return r; 1058 1059 /* 1060 * TDX and VMX have different vCPU structures. Calculate the 1061 * maximum size/align so that kvm_init() can use the larger 1062 * values to create the kmem_vcpu_cache. 1063 */ 1064 vcpu_size = sizeof(struct vcpu_vmx); 1065 vcpu_align = __alignof__(struct vcpu_vmx); 1066 if (enable_tdx) { 1067 vcpu_size = max_t(unsigned, vcpu_size, 1068 sizeof(struct vcpu_tdx)); 1069 vcpu_align = max_t(unsigned, vcpu_align, 1070 __alignof__(struct vcpu_tdx)); 1071 kvm_caps.supported_vm_types |= BIT(KVM_X86_TDX_VM); 1072 } 1073 1074 /* 1075 * Common KVM initialization _must_ come last, after this, /dev/kvm is 1076 * exposed to userspace! 1077 */ 1078 r = kvm_init(vcpu_size, vcpu_align, THIS_MODULE); 1079 if (r) 1080 goto err_kvm_init; 1081 1082 return 0; 1083 1084 err_kvm_init: 1085 vmx_exit(); 1086 return r; 1087 } 1088 module_init(vt_init); 1089