1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/moduleparam.h> 3 4 #include "x86_ops.h" 5 #include "vmx.h" 6 #include "mmu.h" 7 #include "nested.h" 8 #include "pmu.h" 9 #include "posted_intr.h" 10 #include "tdx.h" 11 #include "tdx_arch.h" 12 13 #ifdef CONFIG_KVM_INTEL_TDX 14 static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); 15 16 static void vt_disable_virtualization_cpu(void) 17 { 18 /* Note, TDX *and* VMX need to be disabled if TDX is enabled. */ 19 if (enable_tdx) 20 tdx_disable_virtualization_cpu(); 21 vmx_disable_virtualization_cpu(); 22 } 23 24 static __init int vt_hardware_setup(void) 25 { 26 int ret; 27 28 ret = vmx_hardware_setup(); 29 if (ret) 30 return ret; 31 32 if (enable_tdx) 33 tdx_hardware_setup(); 34 35 return 0; 36 } 37 38 static int vt_vm_init(struct kvm *kvm) 39 { 40 if (is_td(kvm)) 41 return tdx_vm_init(kvm); 42 43 return vmx_vm_init(kvm); 44 } 45 46 static void vt_vm_pre_destroy(struct kvm *kvm) 47 { 48 if (is_td(kvm)) 49 return tdx_mmu_release_hkid(kvm); 50 } 51 52 static void vt_vm_destroy(struct kvm *kvm) 53 { 54 if (is_td(kvm)) 55 return tdx_vm_destroy(kvm); 56 57 vmx_vm_destroy(kvm); 58 } 59 60 static int vt_vcpu_precreate(struct kvm *kvm) 61 { 62 if (is_td(kvm)) 63 return 0; 64 65 return vmx_vcpu_precreate(kvm); 66 } 67 68 static int vt_vcpu_create(struct kvm_vcpu *vcpu) 69 { 70 if (is_td_vcpu(vcpu)) 71 return tdx_vcpu_create(vcpu); 72 73 return vmx_vcpu_create(vcpu); 74 } 75 76 static void vt_vcpu_free(struct kvm_vcpu *vcpu) 77 { 78 if (is_td_vcpu(vcpu)) { 79 tdx_vcpu_free(vcpu); 80 return; 81 } 82 83 vmx_vcpu_free(vcpu); 84 } 85 86 static void vt_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 87 { 88 if (is_td_vcpu(vcpu)) { 89 tdx_vcpu_reset(vcpu, init_event); 90 return; 91 } 92 93 vmx_vcpu_reset(vcpu, init_event); 94 } 95 96 static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 97 { 98 if (is_td_vcpu(vcpu)) { 99 tdx_vcpu_load(vcpu, cpu); 100 return; 101 } 102 103 vmx_vcpu_load(vcpu, cpu); 104 } 105 106 static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 107 { 108 /* 109 * Basic TDX does not support feature PML. KVM does not enable PML in 110 * TD's VMCS, nor does it allocate or flush PML buffer for TDX. 111 */ 112 if (WARN_ON_ONCE(is_td_vcpu(vcpu))) 113 return; 114 115 vmx_update_cpu_dirty_logging(vcpu); 116 } 117 118 static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 119 { 120 if (is_td_vcpu(vcpu)) { 121 tdx_prepare_switch_to_guest(vcpu); 122 return; 123 } 124 125 vmx_prepare_switch_to_guest(vcpu); 126 } 127 128 static void vt_vcpu_put(struct kvm_vcpu *vcpu) 129 { 130 if (is_td_vcpu(vcpu)) { 131 tdx_vcpu_put(vcpu); 132 return; 133 } 134 135 vmx_vcpu_put(vcpu); 136 } 137 138 static int vt_vcpu_pre_run(struct kvm_vcpu *vcpu) 139 { 140 if (is_td_vcpu(vcpu)) 141 return tdx_vcpu_pre_run(vcpu); 142 143 return vmx_vcpu_pre_run(vcpu); 144 } 145 146 static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) 147 { 148 if (is_td_vcpu(vcpu)) 149 return tdx_vcpu_run(vcpu, run_flags); 150 151 return vmx_vcpu_run(vcpu, run_flags); 152 } 153 154 static int vt_handle_exit(struct kvm_vcpu *vcpu, 155 enum exit_fastpath_completion fastpath) 156 { 157 if (is_td_vcpu(vcpu)) 158 return tdx_handle_exit(vcpu, fastpath); 159 160 return vmx_handle_exit(vcpu, fastpath); 161 } 162 163 static int vt_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 164 { 165 if (unlikely(is_td_vcpu(vcpu))) 166 return tdx_set_msr(vcpu, msr_info); 167 168 return vmx_set_msr(vcpu, msr_info); 169 } 170 171 /* 172 * The kvm parameter can be NULL (module initialization, or invocation before 173 * VM creation). Be sure to check the kvm parameter before using it. 174 */ 175 static bool vt_has_emulated_msr(struct kvm *kvm, u32 index) 176 { 177 if (kvm && is_td(kvm)) 178 return tdx_has_emulated_msr(index); 179 180 return vmx_has_emulated_msr(kvm, index); 181 } 182 183 static int vt_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 184 { 185 if (unlikely(is_td_vcpu(vcpu))) 186 return tdx_get_msr(vcpu, msr_info); 187 188 return vmx_get_msr(vcpu, msr_info); 189 } 190 191 static void vt_recalc_msr_intercepts(struct kvm_vcpu *vcpu) 192 { 193 /* 194 * TDX doesn't allow VMM to configure interception of MSR accesses. 195 * TDX guest requests MSR accesses by calling TDVMCALL. The MSR 196 * filters will be applied when handling the TDVMCALL for RDMSR/WRMSR 197 * if the userspace has set any. 198 */ 199 if (is_td_vcpu(vcpu)) 200 return; 201 202 vmx_recalc_msr_intercepts(vcpu); 203 } 204 205 static int vt_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) 206 { 207 if (is_td_vcpu(vcpu)) 208 return tdx_complete_emulated_msr(vcpu, err); 209 210 return vmx_complete_emulated_msr(vcpu, err); 211 } 212 213 #ifdef CONFIG_KVM_SMM 214 static int vt_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 215 { 216 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 217 return 0; 218 219 return vmx_smi_allowed(vcpu, for_injection); 220 } 221 222 static int vt_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 223 { 224 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 225 return 0; 226 227 return vmx_enter_smm(vcpu, smram); 228 } 229 230 static int vt_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 231 { 232 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 233 return 0; 234 235 return vmx_leave_smm(vcpu, smram); 236 } 237 238 static void vt_enable_smi_window(struct kvm_vcpu *vcpu) 239 { 240 if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm)) 241 return; 242 243 /* RSM will cause a vmexit anyway. */ 244 vmx_enable_smi_window(vcpu); 245 } 246 #endif 247 248 static int vt_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 249 void *insn, int insn_len) 250 { 251 /* 252 * For TDX, this can only be triggered for MMIO emulation. Let the 253 * guest retry after installing the SPTE with suppress #VE bit cleared, 254 * so that the guest will receive #VE when retry. The guest is expected 255 * to call TDG.VP.VMCALL<MMIO> to request VMM to do MMIO emulation on 256 * #VE. 257 */ 258 if (is_td_vcpu(vcpu)) 259 return X86EMUL_RETRY_INSTR; 260 261 return vmx_check_emulate_instruction(vcpu, emul_type, insn, insn_len); 262 } 263 264 static bool vt_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 265 { 266 /* 267 * INIT and SIPI are always blocked for TDX, i.e., INIT handling and 268 * the OP vcpu_deliver_sipi_vector() won't be called. 269 */ 270 if (is_td_vcpu(vcpu)) 271 return true; 272 273 return vmx_apic_init_signal_blocked(vcpu); 274 } 275 276 static void vt_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 277 { 278 /* Only x2APIC mode is supported for TD. */ 279 if (is_td_vcpu(vcpu)) 280 return; 281 282 return vmx_set_virtual_apic_mode(vcpu); 283 } 284 285 static void vt_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 286 { 287 if (is_td_vcpu(vcpu)) 288 return; 289 290 return vmx_hwapic_isr_update(vcpu, max_isr); 291 } 292 293 static int vt_sync_pir_to_irr(struct kvm_vcpu *vcpu) 294 { 295 if (is_td_vcpu(vcpu)) 296 return -1; 297 298 return vmx_sync_pir_to_irr(vcpu); 299 } 300 301 static void vt_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 302 int trig_mode, int vector) 303 { 304 if (is_td_vcpu(apic->vcpu)) { 305 tdx_deliver_interrupt(apic, delivery_mode, trig_mode, 306 vector); 307 return; 308 } 309 310 vmx_deliver_interrupt(apic, delivery_mode, trig_mode, vector); 311 } 312 313 static void vt_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 314 { 315 if (is_td_vcpu(vcpu)) 316 return; 317 318 vmx_vcpu_after_set_cpuid(vcpu); 319 } 320 321 static void vt_update_exception_bitmap(struct kvm_vcpu *vcpu) 322 { 323 if (is_td_vcpu(vcpu)) 324 return; 325 326 vmx_update_exception_bitmap(vcpu); 327 } 328 329 static u64 vt_get_segment_base(struct kvm_vcpu *vcpu, int seg) 330 { 331 if (is_td_vcpu(vcpu)) 332 return 0; 333 334 return vmx_get_segment_base(vcpu, seg); 335 } 336 337 static void vt_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, 338 int seg) 339 { 340 if (is_td_vcpu(vcpu)) { 341 memset(var, 0, sizeof(*var)); 342 return; 343 } 344 345 vmx_get_segment(vcpu, var, seg); 346 } 347 348 static void vt_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, 349 int seg) 350 { 351 if (is_td_vcpu(vcpu)) 352 return; 353 354 vmx_set_segment(vcpu, var, seg); 355 } 356 357 static int vt_get_cpl(struct kvm_vcpu *vcpu) 358 { 359 if (is_td_vcpu(vcpu)) 360 return 0; 361 362 return vmx_get_cpl(vcpu); 363 } 364 365 static int vt_get_cpl_no_cache(struct kvm_vcpu *vcpu) 366 { 367 if (is_td_vcpu(vcpu)) 368 return 0; 369 370 return vmx_get_cpl_no_cache(vcpu); 371 } 372 373 static void vt_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 374 { 375 if (is_td_vcpu(vcpu)) { 376 *db = 0; 377 *l = 0; 378 return; 379 } 380 381 vmx_get_cs_db_l_bits(vcpu, db, l); 382 } 383 384 static bool vt_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 385 { 386 if (is_td_vcpu(vcpu)) 387 return true; 388 389 return vmx_is_valid_cr0(vcpu, cr0); 390 } 391 392 static void vt_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 393 { 394 if (is_td_vcpu(vcpu)) 395 return; 396 397 vmx_set_cr0(vcpu, cr0); 398 } 399 400 static bool vt_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 401 { 402 if (is_td_vcpu(vcpu)) 403 return true; 404 405 return vmx_is_valid_cr4(vcpu, cr4); 406 } 407 408 static void vt_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 409 { 410 if (is_td_vcpu(vcpu)) 411 return; 412 413 vmx_set_cr4(vcpu, cr4); 414 } 415 416 static int vt_set_efer(struct kvm_vcpu *vcpu, u64 efer) 417 { 418 if (is_td_vcpu(vcpu)) 419 return 0; 420 421 return vmx_set_efer(vcpu, efer); 422 } 423 424 static void vt_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 425 { 426 if (is_td_vcpu(vcpu)) { 427 memset(dt, 0, sizeof(*dt)); 428 return; 429 } 430 431 vmx_get_idt(vcpu, dt); 432 } 433 434 static void vt_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 435 { 436 if (is_td_vcpu(vcpu)) 437 return; 438 439 vmx_set_idt(vcpu, dt); 440 } 441 442 static void vt_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 443 { 444 if (is_td_vcpu(vcpu)) { 445 memset(dt, 0, sizeof(*dt)); 446 return; 447 } 448 449 vmx_get_gdt(vcpu, dt); 450 } 451 452 static void vt_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 453 { 454 if (is_td_vcpu(vcpu)) 455 return; 456 457 vmx_set_gdt(vcpu, dt); 458 } 459 460 static void vt_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 461 { 462 if (is_td_vcpu(vcpu)) 463 return; 464 465 vmx_set_dr7(vcpu, val); 466 } 467 468 static void vt_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 469 { 470 /* 471 * MOV-DR exiting is always cleared for TD guest, even in debug mode. 472 * Thus KVM_DEBUGREG_WONT_EXIT can never be set and it should never 473 * reach here for TD vcpu. 474 */ 475 if (is_td_vcpu(vcpu)) 476 return; 477 478 vmx_sync_dirty_debug_regs(vcpu); 479 } 480 481 static void vt_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 482 { 483 if (WARN_ON_ONCE(is_td_vcpu(vcpu))) 484 return; 485 486 vmx_cache_reg(vcpu, reg); 487 } 488 489 static unsigned long vt_get_rflags(struct kvm_vcpu *vcpu) 490 { 491 if (is_td_vcpu(vcpu)) 492 return 0; 493 494 return vmx_get_rflags(vcpu); 495 } 496 497 static void vt_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 498 { 499 if (is_td_vcpu(vcpu)) 500 return; 501 502 vmx_set_rflags(vcpu, rflags); 503 } 504 505 static bool vt_get_if_flag(struct kvm_vcpu *vcpu) 506 { 507 if (is_td_vcpu(vcpu)) 508 return false; 509 510 return vmx_get_if_flag(vcpu); 511 } 512 513 static void vt_flush_tlb_all(struct kvm_vcpu *vcpu) 514 { 515 if (is_td_vcpu(vcpu)) { 516 tdx_flush_tlb_all(vcpu); 517 return; 518 } 519 520 vmx_flush_tlb_all(vcpu); 521 } 522 523 static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) 524 { 525 if (is_td_vcpu(vcpu)) { 526 tdx_flush_tlb_current(vcpu); 527 return; 528 } 529 530 vmx_flush_tlb_current(vcpu); 531 } 532 533 static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 534 { 535 if (is_td_vcpu(vcpu)) 536 return; 537 538 vmx_flush_tlb_gva(vcpu, addr); 539 } 540 541 static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) 542 { 543 if (is_td_vcpu(vcpu)) 544 return; 545 546 vmx_flush_tlb_guest(vcpu); 547 } 548 549 static void vt_inject_nmi(struct kvm_vcpu *vcpu) 550 { 551 if (is_td_vcpu(vcpu)) { 552 tdx_inject_nmi(vcpu); 553 return; 554 } 555 556 vmx_inject_nmi(vcpu); 557 } 558 559 static int vt_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 560 { 561 /* 562 * The TDX module manages NMI windows and NMI reinjection, and hides NMI 563 * blocking, all KVM can do is throw an NMI over the wall. 564 */ 565 if (is_td_vcpu(vcpu)) 566 return true; 567 568 return vmx_nmi_allowed(vcpu, for_injection); 569 } 570 571 static bool vt_get_nmi_mask(struct kvm_vcpu *vcpu) 572 { 573 /* 574 * KVM can't get NMI blocking status for TDX guest, assume NMIs are 575 * always unmasked. 576 */ 577 if (is_td_vcpu(vcpu)) 578 return false; 579 580 return vmx_get_nmi_mask(vcpu); 581 } 582 583 static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 584 { 585 if (is_td_vcpu(vcpu)) 586 return; 587 588 vmx_set_nmi_mask(vcpu, masked); 589 } 590 591 static void vt_enable_nmi_window(struct kvm_vcpu *vcpu) 592 { 593 /* Refer to the comments in tdx_inject_nmi(). */ 594 if (is_td_vcpu(vcpu)) 595 return; 596 597 vmx_enable_nmi_window(vcpu); 598 } 599 600 static void vt_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 601 int pgd_level) 602 { 603 if (is_td_vcpu(vcpu)) { 604 tdx_load_mmu_pgd(vcpu, root_hpa, pgd_level); 605 return; 606 } 607 608 vmx_load_mmu_pgd(vcpu, root_hpa, pgd_level); 609 } 610 611 static void vt_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 612 { 613 if (is_td_vcpu(vcpu)) 614 return; 615 616 vmx_set_interrupt_shadow(vcpu, mask); 617 } 618 619 static u32 vt_get_interrupt_shadow(struct kvm_vcpu *vcpu) 620 { 621 if (is_td_vcpu(vcpu)) 622 return 0; 623 624 return vmx_get_interrupt_shadow(vcpu); 625 } 626 627 static void vt_patch_hypercall(struct kvm_vcpu *vcpu, 628 unsigned char *hypercall) 629 { 630 /* 631 * Because guest memory is protected, guest can't be patched. TD kernel 632 * is modified to use TDG.VP.VMCALL for hypercall. 633 */ 634 if (is_td_vcpu(vcpu)) 635 return; 636 637 vmx_patch_hypercall(vcpu, hypercall); 638 } 639 640 static void vt_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 641 { 642 if (is_td_vcpu(vcpu)) 643 return; 644 645 vmx_inject_irq(vcpu, reinjected); 646 } 647 648 static void vt_inject_exception(struct kvm_vcpu *vcpu) 649 { 650 if (is_td_vcpu(vcpu)) 651 return; 652 653 vmx_inject_exception(vcpu); 654 } 655 656 static void vt_cancel_injection(struct kvm_vcpu *vcpu) 657 { 658 if (is_td_vcpu(vcpu)) 659 return; 660 661 vmx_cancel_injection(vcpu); 662 } 663 664 static int vt_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 665 { 666 if (is_td_vcpu(vcpu)) 667 return tdx_interrupt_allowed(vcpu); 668 669 return vmx_interrupt_allowed(vcpu, for_injection); 670 } 671 672 static void vt_enable_irq_window(struct kvm_vcpu *vcpu) 673 { 674 if (is_td_vcpu(vcpu)) 675 return; 676 677 vmx_enable_irq_window(vcpu); 678 } 679 680 static void vt_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) 681 { 682 *intr_info = 0; 683 *error_code = 0; 684 685 if (is_td_vcpu(vcpu)) 686 return; 687 688 vmx_get_entry_info(vcpu, intr_info, error_code); 689 } 690 691 static void vt_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 692 u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) 693 { 694 if (is_td_vcpu(vcpu)) { 695 tdx_get_exit_info(vcpu, reason, info1, info2, intr_info, 696 error_code); 697 return; 698 } 699 700 vmx_get_exit_info(vcpu, reason, info1, info2, intr_info, error_code); 701 } 702 703 static void vt_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 704 { 705 if (is_td_vcpu(vcpu)) 706 return; 707 708 vmx_update_cr8_intercept(vcpu, tpr, irr); 709 } 710 711 static void vt_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 712 { 713 if (is_td_vcpu(vcpu)) 714 return; 715 716 vmx_set_apic_access_page_addr(vcpu); 717 } 718 719 static void vt_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 720 { 721 if (is_td_vcpu(vcpu)) { 722 KVM_BUG_ON(!kvm_vcpu_apicv_active(vcpu), vcpu->kvm); 723 return; 724 } 725 726 vmx_refresh_apicv_exec_ctrl(vcpu); 727 } 728 729 static void vt_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 730 { 731 if (is_td_vcpu(vcpu)) 732 return; 733 734 vmx_load_eoi_exitmap(vcpu, eoi_exit_bitmap); 735 } 736 737 static int vt_set_tss_addr(struct kvm *kvm, unsigned int addr) 738 { 739 if (is_td(kvm)) 740 return 0; 741 742 return vmx_set_tss_addr(kvm, addr); 743 } 744 745 static int vt_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 746 { 747 if (is_td(kvm)) 748 return 0; 749 750 return vmx_set_identity_map_addr(kvm, ident_addr); 751 } 752 753 static u64 vt_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 754 { 755 /* TDX doesn't support L2 guest at the moment. */ 756 if (is_td_vcpu(vcpu)) 757 return 0; 758 759 return vmx_get_l2_tsc_offset(vcpu); 760 } 761 762 static u64 vt_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 763 { 764 /* TDX doesn't support L2 guest at the moment. */ 765 if (is_td_vcpu(vcpu)) 766 return 0; 767 768 return vmx_get_l2_tsc_multiplier(vcpu); 769 } 770 771 static void vt_write_tsc_offset(struct kvm_vcpu *vcpu) 772 { 773 /* In TDX, tsc offset can't be changed. */ 774 if (is_td_vcpu(vcpu)) 775 return; 776 777 vmx_write_tsc_offset(vcpu); 778 } 779 780 static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu) 781 { 782 /* In TDX, tsc multiplier can't be changed. */ 783 if (is_td_vcpu(vcpu)) 784 return; 785 786 vmx_write_tsc_multiplier(vcpu); 787 } 788 789 #ifdef CONFIG_X86_64 790 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 791 bool *expired) 792 { 793 /* VMX-preemption timer isn't available for TDX. */ 794 if (is_td_vcpu(vcpu)) 795 return -EINVAL; 796 797 return vmx_set_hv_timer(vcpu, guest_deadline_tsc, expired); 798 } 799 800 static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu) 801 { 802 /* VMX-preemption timer can't be set. See vt_set_hv_timer(). */ 803 if (is_td_vcpu(vcpu)) 804 return; 805 806 vmx_cancel_hv_timer(vcpu); 807 } 808 #endif 809 810 static void vt_setup_mce(struct kvm_vcpu *vcpu) 811 { 812 if (is_td_vcpu(vcpu)) 813 return; 814 815 vmx_setup_mce(vcpu); 816 } 817 818 static int vt_mem_enc_ioctl(struct kvm *kvm, void __user *argp) 819 { 820 if (!is_td(kvm)) 821 return -ENOTTY; 822 823 return tdx_vm_ioctl(kvm, argp); 824 } 825 826 static int vt_vcpu_mem_enc_ioctl(struct kvm_vcpu *vcpu, void __user *argp) 827 { 828 if (!is_td_vcpu(vcpu)) 829 return -EINVAL; 830 831 return tdx_vcpu_ioctl(vcpu, argp); 832 } 833 834 static int vt_gmem_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn) 835 { 836 if (is_td(kvm)) 837 return tdx_gmem_private_max_mapping_level(kvm, pfn); 838 839 return 0; 840 } 841 842 #define vt_op(name) vt_##name 843 #define vt_op_tdx_only(name) vt_##name 844 #else /* CONFIG_KVM_INTEL_TDX */ 845 #define vt_op(name) vmx_##name 846 #define vt_op_tdx_only(name) NULL 847 #endif /* CONFIG_KVM_INTEL_TDX */ 848 849 #define VMX_REQUIRED_APICV_INHIBITS \ 850 (BIT(APICV_INHIBIT_REASON_DISABLED) | \ 851 BIT(APICV_INHIBIT_REASON_ABSENT) | \ 852 BIT(APICV_INHIBIT_REASON_HYPERV) | \ 853 BIT(APICV_INHIBIT_REASON_BLOCKIRQ) | \ 854 BIT(APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED) | \ 855 BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) | \ 856 BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED)) 857 858 struct kvm_x86_ops vt_x86_ops __initdata = { 859 .name = KBUILD_MODNAME, 860 861 .check_processor_compatibility = vmx_check_processor_compat, 862 863 .hardware_unsetup = vmx_hardware_unsetup, 864 865 .enable_virtualization_cpu = vmx_enable_virtualization_cpu, 866 .disable_virtualization_cpu = vt_op(disable_virtualization_cpu), 867 .emergency_disable_virtualization_cpu = vmx_emergency_disable_virtualization_cpu, 868 869 .has_emulated_msr = vt_op(has_emulated_msr), 870 871 .vm_size = sizeof(struct kvm_vmx), 872 873 .vm_init = vt_op(vm_init), 874 .vm_destroy = vt_op(vm_destroy), 875 .vm_pre_destroy = vt_op_tdx_only(vm_pre_destroy), 876 877 .vcpu_precreate = vt_op(vcpu_precreate), 878 .vcpu_create = vt_op(vcpu_create), 879 .vcpu_free = vt_op(vcpu_free), 880 .vcpu_reset = vt_op(vcpu_reset), 881 882 .prepare_switch_to_guest = vt_op(prepare_switch_to_guest), 883 .vcpu_load = vt_op(vcpu_load), 884 .vcpu_put = vt_op(vcpu_put), 885 886 .HOST_OWNED_DEBUGCTL = VMX_HOST_OWNED_DEBUGCTL_BITS, 887 888 .update_exception_bitmap = vt_op(update_exception_bitmap), 889 .get_feature_msr = vmx_get_feature_msr, 890 .get_msr = vt_op(get_msr), 891 .set_msr = vt_op(set_msr), 892 893 .get_segment_base = vt_op(get_segment_base), 894 .get_segment = vt_op(get_segment), 895 .set_segment = vt_op(set_segment), 896 .get_cpl = vt_op(get_cpl), 897 .get_cpl_no_cache = vt_op(get_cpl_no_cache), 898 .get_cs_db_l_bits = vt_op(get_cs_db_l_bits), 899 .is_valid_cr0 = vt_op(is_valid_cr0), 900 .set_cr0 = vt_op(set_cr0), 901 .is_valid_cr4 = vt_op(is_valid_cr4), 902 .set_cr4 = vt_op(set_cr4), 903 .set_efer = vt_op(set_efer), 904 .get_idt = vt_op(get_idt), 905 .set_idt = vt_op(set_idt), 906 .get_gdt = vt_op(get_gdt), 907 .set_gdt = vt_op(set_gdt), 908 .set_dr7 = vt_op(set_dr7), 909 .sync_dirty_debug_regs = vt_op(sync_dirty_debug_regs), 910 .cache_reg = vt_op(cache_reg), 911 .get_rflags = vt_op(get_rflags), 912 .set_rflags = vt_op(set_rflags), 913 .get_if_flag = vt_op(get_if_flag), 914 915 .flush_tlb_all = vt_op(flush_tlb_all), 916 .flush_tlb_current = vt_op(flush_tlb_current), 917 .flush_tlb_gva = vt_op(flush_tlb_gva), 918 .flush_tlb_guest = vt_op(flush_tlb_guest), 919 920 .vcpu_pre_run = vt_op(vcpu_pre_run), 921 .vcpu_run = vt_op(vcpu_run), 922 .handle_exit = vt_op(handle_exit), 923 .skip_emulated_instruction = vmx_skip_emulated_instruction, 924 .update_emulated_instruction = vmx_update_emulated_instruction, 925 .set_interrupt_shadow = vt_op(set_interrupt_shadow), 926 .get_interrupt_shadow = vt_op(get_interrupt_shadow), 927 .patch_hypercall = vt_op(patch_hypercall), 928 .inject_irq = vt_op(inject_irq), 929 .inject_nmi = vt_op(inject_nmi), 930 .inject_exception = vt_op(inject_exception), 931 .cancel_injection = vt_op(cancel_injection), 932 .interrupt_allowed = vt_op(interrupt_allowed), 933 .nmi_allowed = vt_op(nmi_allowed), 934 .get_nmi_mask = vt_op(get_nmi_mask), 935 .set_nmi_mask = vt_op(set_nmi_mask), 936 .enable_nmi_window = vt_op(enable_nmi_window), 937 .enable_irq_window = vt_op(enable_irq_window), 938 .update_cr8_intercept = vt_op(update_cr8_intercept), 939 940 .x2apic_icr_is_split = false, 941 .set_virtual_apic_mode = vt_op(set_virtual_apic_mode), 942 .set_apic_access_page_addr = vt_op(set_apic_access_page_addr), 943 .refresh_apicv_exec_ctrl = vt_op(refresh_apicv_exec_ctrl), 944 .load_eoi_exitmap = vt_op(load_eoi_exitmap), 945 .apicv_pre_state_restore = pi_apicv_pre_state_restore, 946 .required_apicv_inhibits = VMX_REQUIRED_APICV_INHIBITS, 947 .hwapic_isr_update = vt_op(hwapic_isr_update), 948 .sync_pir_to_irr = vt_op(sync_pir_to_irr), 949 .deliver_interrupt = vt_op(deliver_interrupt), 950 .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, 951 952 .set_tss_addr = vt_op(set_tss_addr), 953 .set_identity_map_addr = vt_op(set_identity_map_addr), 954 .get_mt_mask = vmx_get_mt_mask, 955 956 .get_exit_info = vt_op(get_exit_info), 957 .get_entry_info = vt_op(get_entry_info), 958 959 .vcpu_after_set_cpuid = vt_op(vcpu_after_set_cpuid), 960 961 .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, 962 963 .get_l2_tsc_offset = vt_op(get_l2_tsc_offset), 964 .get_l2_tsc_multiplier = vt_op(get_l2_tsc_multiplier), 965 .write_tsc_offset = vt_op(write_tsc_offset), 966 .write_tsc_multiplier = vt_op(write_tsc_multiplier), 967 968 .load_mmu_pgd = vt_op(load_mmu_pgd), 969 970 .check_intercept = vmx_check_intercept, 971 .handle_exit_irqoff = vmx_handle_exit_irqoff, 972 973 .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), 974 975 .nested_ops = &vmx_nested_ops, 976 977 .pi_update_irte = vmx_pi_update_irte, 978 .pi_start_bypass = vmx_pi_start_bypass, 979 980 #ifdef CONFIG_X86_64 981 .set_hv_timer = vt_op(set_hv_timer), 982 .cancel_hv_timer = vt_op(cancel_hv_timer), 983 #endif 984 985 .setup_mce = vt_op(setup_mce), 986 987 #ifdef CONFIG_KVM_SMM 988 .smi_allowed = vt_op(smi_allowed), 989 .enter_smm = vt_op(enter_smm), 990 .leave_smm = vt_op(leave_smm), 991 .enable_smi_window = vt_op(enable_smi_window), 992 #endif 993 994 .check_emulate_instruction = vt_op(check_emulate_instruction), 995 .apic_init_signal_blocked = vt_op(apic_init_signal_blocked), 996 .migrate_timers = vmx_migrate_timers, 997 998 .recalc_msr_intercepts = vt_op(recalc_msr_intercepts), 999 .complete_emulated_msr = vt_op(complete_emulated_msr), 1000 1001 .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, 1002 1003 .get_untagged_addr = vmx_get_untagged_addr, 1004 1005 .mem_enc_ioctl = vt_op_tdx_only(mem_enc_ioctl), 1006 .vcpu_mem_enc_ioctl = vt_op_tdx_only(vcpu_mem_enc_ioctl), 1007 1008 .private_max_mapping_level = vt_op_tdx_only(gmem_private_max_mapping_level) 1009 }; 1010 1011 struct kvm_x86_init_ops vt_init_ops __initdata = { 1012 .hardware_setup = vt_op(hardware_setup), 1013 .handle_intel_pt_intr = NULL, 1014 1015 .runtime_ops = &vt_x86_ops, 1016 .pmu_ops = &intel_pmu_ops, 1017 }; 1018 1019 static void __exit vt_exit(void) 1020 { 1021 kvm_exit(); 1022 tdx_cleanup(); 1023 vmx_exit(); 1024 } 1025 module_exit(vt_exit); 1026 1027 static int __init vt_init(void) 1028 { 1029 unsigned vcpu_size, vcpu_align; 1030 int r; 1031 1032 r = vmx_init(); 1033 if (r) 1034 return r; 1035 1036 /* tdx_init() has been taken */ 1037 r = tdx_bringup(); 1038 if (r) 1039 goto err_tdx_bringup; 1040 1041 /* 1042 * TDX and VMX have different vCPU structures. Calculate the 1043 * maximum size/align so that kvm_init() can use the larger 1044 * values to create the kmem_vcpu_cache. 1045 */ 1046 vcpu_size = sizeof(struct vcpu_vmx); 1047 vcpu_align = __alignof__(struct vcpu_vmx); 1048 if (enable_tdx) { 1049 vcpu_size = max_t(unsigned, vcpu_size, 1050 sizeof(struct vcpu_tdx)); 1051 vcpu_align = max_t(unsigned, vcpu_align, 1052 __alignof__(struct vcpu_tdx)); 1053 kvm_caps.supported_vm_types |= BIT(KVM_X86_TDX_VM); 1054 } 1055 1056 /* 1057 * Common KVM initialization _must_ come last, after this, /dev/kvm is 1058 * exposed to userspace! 1059 */ 1060 r = kvm_init(vcpu_size, vcpu_align, THIS_MODULE); 1061 if (r) 1062 goto err_kvm_init; 1063 1064 return 0; 1065 1066 err_kvm_init: 1067 tdx_cleanup(); 1068 err_tdx_bringup: 1069 vmx_exit(); 1070 return r; 1071 } 1072 module_init(vt_init); 1073