1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This header defines architecture specific interfaces, x86 version 6 */ 7 8 #ifndef _ASM_X86_KVM_HOST_H 9 #define _ASM_X86_KVM_HOST_H 10 11 #include <linux/types.h> 12 #include <linux/mm.h> 13 #include <linux/mmu_notifier.h> 14 #include <linux/tracepoint.h> 15 #include <linux/cpumask.h> 16 #include <linux/irq_work.h> 17 #include <linux/irq.h> 18 #include <linux/workqueue.h> 19 #include <linux/xarray.h> 20 21 #include <linux/kvm.h> 22 #include <linux/kvm_para.h> 23 #include <linux/kvm_types.h> 24 #include <linux/perf_event.h> 25 #include <linux/pvclock_gtod.h> 26 #include <linux/clocksource.h> 27 #include <linux/irqbypass.h> 28 #include <linux/kfifo.h> 29 #include <linux/sched/vhost_task.h> 30 #include <linux/call_once.h> 31 #include <linux/atomic.h> 32 33 #include <asm/apic.h> 34 #include <asm/pvclock-abi.h> 35 #include <asm/debugreg.h> 36 #include <asm/desc.h> 37 #include <asm/mtrr.h> 38 #include <asm/msr-index.h> 39 #include <asm/msr.h> 40 #include <asm/asm.h> 41 #include <asm/irq_remapping.h> 42 #include <asm/kvm_page_track.h> 43 #include <asm/kvm_vcpu_regs.h> 44 #include <asm/virt.h> 45 46 #include <hyperv/hvhdk.h> 47 48 #define kvm_file_to_kvm_arch x86 49 50 #define __KVM_HAVE_ARCH_VCPU_DEBUGFS 51 #define __KVM_HAVE_MSI 52 #define __KVM_HAVE_NO_POLL 53 #define __KVM_HAVE_READONLY_MEM 54 55 /* 56 * CONFIG_KVM_MAX_NR_VCPUS is defined iff CONFIG_KVM!=n, provide a dummy max if 57 * KVM is disabled (arbitrarily use the default from CONFIG_KVM_MAX_NR_VCPUS). 58 */ 59 #ifdef CONFIG_KVM_MAX_NR_VCPUS 60 #define KVM_MAX_VCPUS CONFIG_KVM_MAX_NR_VCPUS 61 #else 62 #define KVM_MAX_VCPUS 1024 63 #endif 64 65 /* 66 * In x86, the VCPU ID corresponds to the APIC ID, and APIC IDs 67 * might be larger than the actual number of VCPUs because the 68 * APIC ID encodes CPU topology information. 69 * 70 * In the worst case, we'll need less than one extra bit for the 71 * Core ID, and less than one extra bit for the Package (Die) ID, 72 * so ratio of 4 should be enough. 73 */ 74 #define KVM_VCPU_ID_RATIO 4 75 #define KVM_MAX_VCPU_IDS (KVM_MAX_VCPUS * KVM_VCPU_ID_RATIO) 76 77 /* memory slots that are not exposed to userspace */ 78 #define KVM_INTERNAL_MEM_SLOTS 3 79 80 #define KVM_HALT_POLL_NS_DEFAULT 200000 81 82 #define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS 83 84 #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ 85 KVM_DIRTY_LOG_INITIALLY_SET) 86 87 /* x86-specific vcpu->requests bit members */ 88 #define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0) 89 #define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1) 90 #define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2) 91 #define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3) 92 #define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4) 93 #define KVM_REQ_LOAD_MMU_PGD KVM_ARCH_REQ(5) 94 #define KVM_REQ_EVENT KVM_ARCH_REQ(6) 95 #define KVM_REQ_APF_HALT KVM_ARCH_REQ(7) 96 #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8) 97 #define KVM_REQ_NMI KVM_ARCH_REQ(9) 98 #define KVM_REQ_PMU KVM_ARCH_REQ(10) 99 #define KVM_REQ_PMI KVM_ARCH_REQ(11) 100 #ifdef CONFIG_KVM_SMM 101 #define KVM_REQ_SMI KVM_ARCH_REQ(12) 102 #endif 103 #define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13) 104 #define KVM_REQ_MCLOCK_INPROGRESS \ 105 KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 106 #define KVM_REQ_SCAN_IOAPIC \ 107 KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 108 #define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16) 109 #define KVM_REQ_APIC_PAGE_RELOAD \ 110 KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 111 #define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18) 112 #define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19) 113 #define KVM_REQ_HV_RESET KVM_ARCH_REQ(20) 114 #define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21) 115 #define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22) 116 #define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23) 117 #define KVM_REQ_GET_NESTED_STATE_PAGES KVM_ARCH_REQ(24) 118 #define KVM_REQ_APICV_UPDATE \ 119 KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 120 #define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26) 121 #define KVM_REQ_TLB_FLUSH_GUEST \ 122 KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 123 #define KVM_REQ_APF_READY KVM_ARCH_REQ(28) 124 #define KVM_REQ_RECALC_INTERCEPTS KVM_ARCH_REQ(29) 125 #define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \ 126 KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 127 #define KVM_REQ_MMU_FREE_OBSOLETE_ROOTS \ 128 KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 129 #define KVM_REQ_HV_TLB_FLUSH \ 130 KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 131 #define KVM_REQ_VMSA_PAGE_RELOAD \ 132 KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 133 #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \ 134 KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT) 135 136 #define INVALID_PAGE (~(hpa_t)0) 137 #define VALID_PAGE(x) ((x) != INVALID_PAGE) 138 139 /* KVM Hugepage definitions for x86 */ 140 #define KVM_MAX_HUGEPAGE_LEVEL PG_LEVEL_1G 141 #define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - PG_LEVEL_4K + 1) 142 #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) 143 #define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x)) 144 #define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x)) 145 #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) 146 #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) 147 148 #define KVM_MAX_CPUID_ENTRIES 256 149 #define KVM_NR_VAR_MTRR 8 150 151 #define ASYNC_PF_PER_VCPU 64 152 153 enum kvm_reg { 154 VCPU_REGS_RAX = __VCPU_REGS_RAX, 155 VCPU_REGS_RCX = __VCPU_REGS_RCX, 156 VCPU_REGS_RDX = __VCPU_REGS_RDX, 157 VCPU_REGS_RBX = __VCPU_REGS_RBX, 158 VCPU_REGS_RSP = __VCPU_REGS_RSP, 159 VCPU_REGS_RBP = __VCPU_REGS_RBP, 160 VCPU_REGS_RSI = __VCPU_REGS_RSI, 161 VCPU_REGS_RDI = __VCPU_REGS_RDI, 162 #ifdef CONFIG_X86_64 163 VCPU_REGS_R8 = 8, 164 VCPU_REGS_R9, 165 VCPU_REGS_R10, 166 VCPU_REGS_R11, 167 VCPU_REGS_R12, 168 VCPU_REGS_R13, 169 VCPU_REGS_R14, 170 VCPU_REGS_R15, 171 #endif 172 NR_VCPU_GENERAL_PURPOSE_REGS, 173 174 VCPU_REG_RIP = NR_VCPU_GENERAL_PURPOSE_REGS, 175 176 VCPU_REG_PDPTR, 177 VCPU_REG_CR0, 178 /* 179 * Alias AMD's ERAPS (not a real register) to CR3 so that common code 180 * can trigger emulation of the RAP (Return Address Predictor) with 181 * minimal support required in common code. Piggyback CR3 as the RAP 182 * is cleared on writes to CR3, i.e. marking CR3 dirty will naturally 183 * mark ERAPS dirty as well. 184 */ 185 VCPU_REG_CR3, 186 VCPU_REG_ERAPS = VCPU_REG_CR3, 187 VCPU_REG_CR4, 188 VCPU_REG_RFLAGS, 189 VCPU_REG_SEGMENTS, 190 VCPU_REG_EXIT_INFO_1, 191 VCPU_REG_EXIT_INFO_2, 192 193 NR_VCPU_TOTAL_REGS, 194 }; 195 196 enum { 197 VCPU_SREG_ES, 198 VCPU_SREG_CS, 199 VCPU_SREG_SS, 200 VCPU_SREG_DS, 201 VCPU_SREG_FS, 202 VCPU_SREG_GS, 203 VCPU_SREG_TR, 204 VCPU_SREG_LDTR, 205 }; 206 207 enum exit_fastpath_completion { 208 EXIT_FASTPATH_NONE, 209 EXIT_FASTPATH_REENTER_GUEST, 210 EXIT_FASTPATH_EXIT_HANDLED, 211 EXIT_FASTPATH_EXIT_USERSPACE, 212 }; 213 typedef enum exit_fastpath_completion fastpath_t; 214 215 struct x86_emulate_ctxt; 216 struct x86_exception; 217 union kvm_smram; 218 enum x86_intercept; 219 enum x86_intercept_stage; 220 221 #define KVM_NR_DB_REGS 4 222 223 #define PFERR_PRESENT_MASK BIT(0) 224 #define PFERR_WRITE_MASK BIT(1) 225 #define PFERR_USER_MASK BIT(2) 226 #define PFERR_RSVD_MASK BIT(3) 227 #define PFERR_FETCH_MASK BIT(4) 228 #define PFERR_PK_MASK BIT(5) 229 #define PFERR_SS_MASK BIT(6) 230 #define PFERR_SGX_MASK BIT(15) 231 #define PFERR_GUEST_RMP_MASK BIT_ULL(31) 232 #define PFERR_GUEST_FINAL_MASK BIT_ULL(32) 233 #define PFERR_GUEST_PAGE_MASK BIT_ULL(33) 234 #define PFERR_GUEST_FAULT_STAGE_MASK \ 235 (PFERR_GUEST_FINAL_MASK | PFERR_GUEST_PAGE_MASK) 236 #define PFERR_GUEST_ENC_MASK BIT_ULL(34) 237 #define PFERR_GUEST_SIZEM_MASK BIT_ULL(35) 238 #define PFERR_GUEST_VMPL_MASK BIT_ULL(36) 239 240 /* 241 * IMPLICIT_ACCESS is a KVM-defined flag used to correctly perform SMAP checks 242 * when emulating instructions that triggers implicit access. 243 */ 244 #define PFERR_IMPLICIT_ACCESS BIT_ULL(48) 245 /* 246 * PRIVATE_ACCESS is a KVM-defined flag us to indicate that a fault occurred 247 * when the guest was accessing private memory. 248 */ 249 #define PFERR_PRIVATE_ACCESS BIT_ULL(49) 250 #define PFERR_SYNTHETIC_MASK (PFERR_IMPLICIT_ACCESS | PFERR_PRIVATE_ACCESS) 251 252 struct kvm_kernel_irqfd; 253 struct kvm_kernel_irq_routing_entry; 254 255 struct kvm_apic_map; 256 257 struct kvm_x86_msr_filter; 258 struct kvm_x86_pmu_event_filter; 259 260 struct kvm_caps { 261 /* control of guest tsc rate supported? */ 262 bool has_tsc_control; 263 /* maximum supported tsc_khz for guests */ 264 u32 max_guest_tsc_khz; 265 /* number of bits of the fractional part of the TSC scaling ratio */ 266 u8 tsc_scaling_ratio_frac_bits; 267 /* maximum allowed value of TSC scaling ratio */ 268 u64 max_tsc_scaling_ratio; 269 /* 1ull << kvm_caps.tsc_scaling_ratio_frac_bits */ 270 u64 default_tsc_scaling_ratio; 271 /* bus lock detection supported? */ 272 bool has_bus_lock_exit; 273 /* notify VM exit supported? */ 274 bool has_notify_vmexit; 275 /* bit mask of VM types */ 276 u32 supported_vm_types; 277 278 u64 supported_mce_cap; 279 u64 supported_xcr0; 280 u64 supported_xss; 281 u64 supported_perf_cap; 282 283 u64 supported_efer_bits; 284 285 u64 supported_quirks; 286 u64 inapplicable_quirks; 287 }; 288 extern struct kvm_caps kvm_caps; 289 290 struct kvm_host_values { 291 /* 292 * The host's raw MAXPHYADDR, i.e. the number of non-reserved physical 293 * address bits irrespective of features that repurpose legal bits, 294 * e.g. MKTME. 295 */ 296 u8 maxphyaddr; 297 298 u64 efer; 299 u64 xcr0; 300 u64 xss; 301 u64 s_cet; 302 u64 arch_capabilities; 303 }; 304 extern struct kvm_host_values kvm_host; 305 306 /* 307 * kvm_mmu_page_role tracks the properties of a shadow page (where shadow page 308 * also includes TDP pages) to determine whether or not a page can be used in 309 * the given MMU context. This is a subset of the overall kvm_cpu_role to 310 * minimize the size of kvm_memory_slot.arch.gfn_write_track, i.e. allows 311 * allocating 2 bytes per gfn instead of 4 bytes per gfn. 312 * 313 * Upper-level shadow pages having gptes are tracked for write-protection via 314 * gfn_write_track. As above, gfn_write_track is a 16 bit counter, so KVM must 315 * not create more than 2^16-1 upper-level shadow pages at a single gfn, 316 * otherwise gfn_write_track will overflow and explosions will ensue. 317 * 318 * A unique shadow page (SP) for a gfn is created if and only if an existing SP 319 * cannot be reused. The ability to reuse a SP is tracked by its role, which 320 * incorporates various mode bits and properties of the SP. Roughly speaking, 321 * the number of unique SPs that can theoretically be created is 2^n, where n 322 * is the number of bits that are used to compute the role. 323 * 324 * But, even though there are 21 bits in the mask below, not all combinations 325 * of modes and flags are possible: 326 * 327 * - invalid shadow pages are not accounted, mirror pages are not shadowed, 328 * so the bits are effectively 19. 329 * 330 * - quadrant will only be used if has_4_byte_gpte=1 (non-PAE paging); 331 * execonly and ad_disabled are only used for nested EPT which has 332 * has_4_byte_gpte=0. Therefore, 2 bits are always unused. 333 * 334 * - the 4 bits of level are effectively limited to the values 2/3/4/5, 335 * as 4k SPs are not tracked (allowed to go unsync). In addition non-PAE 336 * paging has exactly one upper level, making level completely redundant 337 * when has_4_byte_gpte=1. 338 * 339 * - on top of this, smap_andnot_wp is only set if cr0_wp=0, 340 * therefore these two bits only give rise to 3 possibilities. 341 * 342 * Therefore, the maximum number of possible upper-level shadow pages for a 343 * single gfn is a bit less than 2^14. 344 */ 345 union kvm_mmu_page_role { 346 u32 word; 347 struct { 348 unsigned level:4; 349 unsigned has_4_byte_gpte:1; 350 unsigned quadrant:2; 351 unsigned direct:1; 352 unsigned access:4; 353 unsigned invalid:1; 354 unsigned efer_nx:1; 355 unsigned cr0_wp:1; 356 unsigned smap_andnot_wp:1; 357 unsigned ad_disabled:1; 358 unsigned guest_mode:1; 359 unsigned passthrough:1; 360 unsigned is_mirror:1; 361 362 /* 363 * cr4_smep is also set for EPT MBEC. Because it affects 364 * which pages are considered non-present (bit 10 additionally 365 * must be zero if MBEC is on) it has to be in the base role. 366 * It also has to be in the base role for AMD GMET because 367 * kernel-executable pages need to have U=0 with GMET enabled. 368 */ 369 unsigned cr4_smep:1; 370 371 unsigned:3; 372 373 /* 374 * This is left at the top of the word so that 375 * kvm_memslots_for_spte_role can extract it with a 376 * simple shift. While there is room, give it a whole 377 * byte so it is also faster to load it from memory. 378 */ 379 unsigned smm:8; 380 }; 381 }; 382 383 /* 384 * kvm_mmu_extended_role complements kvm_mmu_page_role, tracking properties 385 * relevant to the current MMU configuration. When loading CR0, CR4, or EFER, 386 * including on nested transitions, if nothing in the full role changes then 387 * MMU re-configuration can be skipped. @valid bit is set on first usage so we 388 * don't treat all-zero structure as valid data. 389 * 390 * The properties that are tracked in the extended role but not the page role 391 * are for things that either (a) do not affect the validity of the shadow page 392 * or (b) are indirectly reflected in the shadow page's role. For example, 393 * CR4.PKE only affects permission checks for software walks of the guest page 394 * tables (because KVM doesn't support Protection Keys with shadow paging), and 395 * CR0.PG, CR4.PAE, and CR4.PSE are indirectly reflected in role.level. 396 * 397 * Note, SMAP is not redundant with smap_andnot_wp in the page role. If 398 * CR0.WP=1, KVM can reuse shadow pages for the guest regardless of SMAP, 399 * but the MMU's permission checks for software walks need to be SMAP 400 * aware regardless of CR0.WP. 401 */ 402 union kvm_mmu_extended_role { 403 u32 word; 404 struct { 405 unsigned int valid:1; 406 unsigned int execonly:1; 407 unsigned int cr4_pse:1; 408 unsigned int cr4_pke:1; 409 unsigned int cr4_smap:1; 410 unsigned int cr4_la57:1; 411 unsigned int efer_lma:1; 412 413 /* 414 * True if either CR4.SMEP or EFER.NXE are set. For AMD NPT 415 * this is the "real" host CR4.SMEP whereas cr4_smep is 416 * actually GMET. 417 */ 418 unsigned int has_pferr_fetch:1; 419 }; 420 }; 421 422 union kvm_cpu_role { 423 u64 as_u64; 424 struct { 425 union kvm_mmu_page_role base; 426 union kvm_mmu_extended_role ext; 427 }; 428 }; 429 430 struct kvm_rmap_head { 431 atomic_long_t val; 432 }; 433 434 struct kvm_pio_request { 435 unsigned long count; 436 int in; 437 int port; 438 int size; 439 }; 440 441 #define PT64_ROOT_MAX_LEVEL 5 442 443 struct kvm_page_format { 444 u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL]; 445 u64 bad_mt_xwr; 446 447 /* 448 * The pkru_mask indicates if protection key checks are needed. It 449 * consists of 16 domains indexed by page fault error code bits [4:1], 450 * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables. 451 * Each domain has 2 bits which are ANDed with AD and WD from PKRU. 452 */ 453 u32 pkru_mask; 454 455 /* 456 * Bitmap; bit set = permission fault 457 * Array index: page fault error code [4:1] 458 * Bit index: pte permissions in ACC_* format 459 */ 460 u16 permissions[16]; 461 }; 462 463 struct kvm_mmu_root_info { 464 gpa_t pgd; 465 hpa_t hpa; 466 }; 467 468 #define KVM_MMU_ROOT_INFO_INVALID \ 469 ((struct kvm_mmu_root_info) { .pgd = INVALID_PAGE, .hpa = INVALID_PAGE }) 470 471 #define KVM_MMU_NUM_PREV_ROOTS 3 472 473 #define KVM_MMU_ROOT_CURRENT BIT(0) 474 #define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i) 475 #define KVM_MMU_ROOTS_ALL (BIT(1 + KVM_MMU_NUM_PREV_ROOTS) - 1) 476 477 #define KVM_HAVE_MMU_RWLOCK 478 479 struct kvm_mmu_page; 480 struct kvm_page_fault; 481 482 /* 483 * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit, 484 * and 2-level 32-bit). The kvm_pagewalk structure abstracts the details of the 485 * current mmu mode. 486 */ 487 struct kvm_pagewalk { 488 unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu); 489 u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); 490 void (*inject_page_fault)(struct kvm_vcpu *vcpu, 491 struct x86_exception *fault, 492 bool from_hardware); 493 gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w, 494 gpa_t gva_or_gpa, u64 access, 495 struct x86_exception *exception); 496 497 union kvm_cpu_role cpu_role; 498 struct kvm_page_format fmt; 499 }; 500 501 struct kvm_mmu { 502 int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); 503 int (*sync_spte)(struct kvm_vcpu *vcpu, 504 struct kvm_mmu_page *sp, int i); 505 struct kvm_pagewalk *w; 506 507 struct kvm_mmu_root_info root; 508 hpa_t mirror_root_hpa; 509 union kvm_mmu_page_role root_role; 510 511 struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS]; 512 513 u64 *pae_root; 514 u64 *pml4_root; 515 u64 *pml5_root; 516 517 /* 518 * check zero bits on shadow page table entries, these 519 * bits include not only hardware reserved bits but also 520 * the bits spte never used. 521 */ 522 struct kvm_page_format fmt; 523 }; 524 525 enum pmc_type { 526 KVM_PMC_GP = 0, 527 KVM_PMC_FIXED, 528 }; 529 530 struct kvm_pmc { 531 enum pmc_type type; 532 u8 idx; 533 bool is_paused; 534 bool intr; 535 /* 536 * Base value of the PMC counter, relative to the *consumed* count in 537 * the associated perf_event. This value includes counter updates from 538 * the perf_event and emulated_count since the last time the counter 539 * was reprogrammed, but it is *not* the current value as seen by the 540 * guest or userspace. 541 * 542 * The count is relative to the associated perf_event so that KVM 543 * doesn't need to reprogram the perf_event every time the guest writes 544 * to the counter. 545 */ 546 u64 counter; 547 /* 548 * PMC events triggered by KVM emulation that haven't been fully 549 * processed, i.e. haven't undergone overflow detection. 550 */ 551 u64 emulated_counter; 552 u64 eventsel; 553 u64 eventsel_hw; 554 struct perf_event *perf_event; 555 struct kvm_vcpu *vcpu; 556 /* 557 * only for creating or reusing perf_event, 558 * eventsel value for general purpose counters, 559 * ctrl value for fixed counters. 560 */ 561 u64 current_config; 562 }; 563 564 /* More counters may conflict with other existing Architectural MSRs */ 565 #define KVM_MAX(a, b) ((a) >= (b) ? (a) : (b)) 566 #define KVM_MAX_NR_INTEL_GP_COUNTERS 8 567 #define KVM_MAX_NR_AMD_GP_COUNTERS 6 568 #define KVM_MAX_NR_GP_COUNTERS KVM_MAX(KVM_MAX_NR_INTEL_GP_COUNTERS, \ 569 KVM_MAX_NR_AMD_GP_COUNTERS) 570 571 #define KVM_MAX_NR_INTEL_FIXED_COUNTERS 3 572 #define KVM_MAX_NR_AMD_FIXED_COUNTERS 0 573 #define KVM_MAX_NR_FIXED_COUNTERS KVM_MAX(KVM_MAX_NR_INTEL_FIXED_COUNTERS, \ 574 KVM_MAX_NR_AMD_FIXED_COUNTERS) 575 576 struct kvm_pmu { 577 u8 version; 578 unsigned nr_arch_gp_counters; 579 unsigned nr_arch_fixed_counters; 580 unsigned available_event_types; 581 u64 fixed_ctr_ctrl; 582 u64 fixed_ctr_ctrl_hw; 583 u64 fixed_ctr_ctrl_rsvd; 584 u64 global_ctrl; 585 u64 global_status; 586 u64 counter_bitmask[2]; 587 u64 global_ctrl_rsvd; 588 u64 global_status_rsvd; 589 u64 reserved_bits; 590 u64 raw_event_mask; 591 struct kvm_pmc gp_counters[KVM_MAX_NR_GP_COUNTERS]; 592 struct kvm_pmc fixed_counters[KVM_MAX_NR_FIXED_COUNTERS]; 593 594 /* 595 * Overlay the bitmap with a 64-bit atomic so that all bits can be 596 * set in a single access, e.g. to reprogram all counters when the PMU 597 * filter changes. 598 */ 599 union { 600 DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX); 601 atomic64_t __reprogram_pmi; 602 }; 603 DECLARE_BITMAP(all_valid_pmc_idx, X86_PMC_IDX_MAX); 604 DECLARE_BITMAP(pmc_in_use, X86_PMC_IDX_MAX); 605 606 DECLARE_BITMAP(pmc_counting_instructions, X86_PMC_IDX_MAX); 607 DECLARE_BITMAP(pmc_counting_branches, X86_PMC_IDX_MAX); 608 609 DECLARE_BITMAP(pmc_has_mode_specific_enables, X86_PMC_IDX_MAX); 610 611 u64 ds_area; 612 u64 pebs_enable; 613 u64 pebs_enable_rsvd; 614 u64 pebs_data_cfg; 615 u64 pebs_data_cfg_rsvd; 616 617 /* 618 * The gate to release perf_events not marked in 619 * pmc_in_use only once in a vcpu time slice. 620 */ 621 bool need_cleanup; 622 623 /* 624 * The total number of programmed perf_events and it helps to avoid 625 * redundant check before cleanup if guest don't use vPMU at all. 626 */ 627 u8 event_count; 628 }; 629 630 struct kvm_pmu_ops; 631 632 enum { 633 KVM_DEBUGREG_BP_ENABLED = BIT(0), 634 KVM_DEBUGREG_WONT_EXIT = BIT(1), 635 /* 636 * Guest debug registers (DR0-3, DR6 and DR7) are saved/restored by 637 * hardware on exit from or enter to guest. KVM needn't switch them. 638 * DR0-3, DR6 and DR7 are set to their architectural INIT value on VM 639 * exit, host values need to be restored. 640 */ 641 KVM_DEBUGREG_AUTO_SWITCH = BIT(2), 642 }; 643 644 struct kvm_mtrr { 645 u64 var[KVM_NR_VAR_MTRR * 2]; 646 u64 fixed_64k; 647 u64 fixed_16k[2]; 648 u64 fixed_4k[8]; 649 u64 deftype; 650 }; 651 652 struct kvm_hypervisor_cpuid { 653 u32 base; 654 u32 limit; 655 }; 656 657 #ifdef CONFIG_KVM_XEN 658 /* Xen HVM per vcpu emulation context */ 659 struct kvm_vcpu_xen { 660 u64 hypercall_rip; 661 u32 current_runstate; 662 u8 upcall_vector; 663 struct gfn_to_pfn_cache vcpu_info_cache; 664 struct gfn_to_pfn_cache vcpu_time_info_cache; 665 struct gfn_to_pfn_cache runstate_cache; 666 struct gfn_to_pfn_cache runstate2_cache; 667 u64 last_steal; 668 u64 runstate_entry_time; 669 u64 runstate_times[4]; 670 unsigned long evtchn_pending_sel; 671 u32 vcpu_id; /* The Xen / ACPI vCPU ID */ 672 u32 timer_virq; 673 u64 timer_expires; /* In guest epoch */ 674 atomic_t timer_pending; 675 struct hrtimer timer; 676 int poll_evtchn; 677 struct timer_list poll_timer; 678 struct kvm_hypervisor_cpuid cpuid; 679 }; 680 #endif 681 682 struct kvm_vcpu_hv; 683 684 struct kvm_queued_exception { 685 bool pending; 686 bool injected; 687 bool has_error_code; 688 u8 vector; 689 u32 error_code; 690 unsigned long payload; 691 bool has_payload; 692 }; 693 694 /* 695 * Hardware-defined CPUID leafs that are either scattered by the kernel or are 696 * unknown to the kernel, but need to be directly used by KVM. Note, these 697 * word values conflict with the kernel's "bug" caps, but KVM doesn't use those. 698 */ 699 enum kvm_only_cpuid_leafs { 700 CPUID_12_EAX = NCAPINTS, 701 CPUID_7_1_EDX, 702 CPUID_8000_0007_EDX, 703 CPUID_8000_0022_EAX, 704 CPUID_7_2_EDX, 705 CPUID_24_0_EBX, 706 CPUID_8000_0021_ECX, 707 CPUID_7_1_ECX, 708 CPUID_1E_1_EAX, 709 CPUID_24_1_ECX, 710 NR_KVM_CPU_CAPS, 711 712 NKVMCAPINTS = NR_KVM_CPU_CAPS - NCAPINTS, 713 }; 714 715 struct kvm_vcpu_arch { 716 /* 717 * rip and regs accesses must go through 718 * kvm_{register,rip}_{read,write} functions. 719 */ 720 unsigned long regs[NR_VCPU_GENERAL_PURPOSE_REGS]; 721 unsigned long rip; 722 DECLARE_BITMAP(regs_avail, NR_VCPU_TOTAL_REGS); 723 DECLARE_BITMAP(regs_dirty, NR_VCPU_TOTAL_REGS); 724 725 unsigned long cr0; 726 unsigned long cr0_guest_owned_bits; 727 unsigned long cr2; 728 unsigned long cr3; 729 unsigned long cr4; 730 unsigned long cr4_guest_owned_bits; 731 unsigned long cr4_guest_rsvd_bits; 732 unsigned long cr8; 733 u32 host_pkru; 734 u32 pkru; 735 u32 hflags; 736 u64 efer; 737 u64 host_debugctl; 738 u64 apic_base; 739 struct kvm_lapic *apic; /* kernel irqchip context */ 740 bool load_eoi_exitmap_pending; 741 DECLARE_BITMAP(ioapic_handled_vectors, 256); 742 unsigned long apic_attention; 743 int32_t apic_arb_prio; 744 int mp_state; 745 u64 ia32_misc_enable_msr; 746 u64 smbase; 747 u64 smi_count; 748 bool at_instruction_boundary; 749 bool tpr_access_reporting; 750 bool xfd_no_write_intercept; 751 u64 microcode_version; 752 u64 arch_capabilities; 753 u64 perf_capabilities; 754 755 /* 756 * Paging state of the vcpu 757 * 758 * If the vcpu runs in guest mode with two level paging this still saves 759 * the paging mode of the l1 guest. This context is always used to 760 * handle faults. 761 */ 762 struct kvm_mmu *mmu; 763 764 /* Non-nested MMU for L1 */ 765 struct kvm_mmu root_mmu; 766 767 /* L1 TDP when running nested */ 768 struct kvm_mmu guest_mmu; 769 struct kvm_pagewalk ngpa_walk; 770 771 /* 772 * Pagewalk context used for gva_to_gpa translations. 773 */ 774 struct kvm_pagewalk gva_walk; 775 776 u64 pdptrs[4]; /* pae */ 777 778 struct kvm_mmu_memory_cache mmu_pte_list_desc_cache; 779 struct kvm_mmu_memory_cache mmu_shadow_page_cache; 780 struct kvm_mmu_memory_cache mmu_shadowed_info_cache; 781 struct kvm_mmu_memory_cache mmu_page_header_cache; 782 /* 783 * This cache is to allocate external page table. E.g. private EPT used 784 * by the TDX module. 785 */ 786 struct kvm_mmu_memory_cache mmu_external_spt_cache; 787 788 struct page *pml_page; 789 790 /* 791 * QEMU userspace and the guest each have their own FPU state. 792 * In vcpu_run, we switch between the user and guest FPU contexts. 793 * While running a VCPU, the VCPU thread will have the guest FPU 794 * context. 795 * 796 * Note that while the PKRU state lives inside the fpu registers, 797 * it is switched out separately at VMENTER and VMEXIT time. The 798 * "guest_fpstate" state here contains the guest FPU context, with the 799 * host PRKU bits. 800 */ 801 struct fpu_guest guest_fpu; 802 803 u64 xcr0; 804 u64 guest_supported_xcr0; 805 u64 ia32_xss; 806 u64 guest_supported_xss; 807 808 struct kvm_pio_request pio; 809 void *pio_data; 810 void *sev_pio_data; 811 unsigned sev_pio_count; 812 813 u8 event_exit_inst_len; 814 815 bool exception_from_userspace; 816 817 /* Exceptions to be injected to the guest. */ 818 struct kvm_queued_exception exception; 819 /* Exception VM-Exits to be synthesized to L1. */ 820 struct kvm_queued_exception exception_vmexit; 821 822 struct kvm_queued_interrupt { 823 bool injected; 824 bool soft; 825 u8 nr; 826 } interrupt; 827 828 int halt_request; /* real mode on Intel only */ 829 830 int cpuid_nent; 831 struct kvm_cpuid_entry2 *cpuid_entries; 832 bool cpuid_dynamic_bits_dirty; 833 bool is_amd_compatible; 834 835 /* 836 * cpu_caps holds the effective guest capabilities, i.e. the features 837 * the vCPU is allowed to use. Typically, but not always, features can 838 * be used by the guest if and only if both KVM and userspace want to 839 * expose the feature to the guest. 840 * 841 * A common exception is for virtualization holes, i.e. when KVM can't 842 * prevent the guest from using a feature, in which case the vCPU "has" 843 * the feature regardless of what KVM or userspace desires. 844 * 845 * Note, features that don't require KVM involvement in any way are 846 * NOT enforced/sanitized by KVM, i.e. are taken verbatim from the 847 * guest CPUID provided by userspace. 848 */ 849 u32 cpu_caps[NR_KVM_CPU_CAPS]; 850 851 u64 reserved_gpa_bits; 852 int maxphyaddr; 853 854 /* emulate context */ 855 856 struct x86_emulate_ctxt *emulate_ctxt; 857 bool emulate_regs_need_sync_to_vcpu; 858 bool emulate_regs_need_sync_from_vcpu; 859 int (*complete_userspace_io)(struct kvm_vcpu *vcpu); 860 unsigned long cui_linear_rip; 861 int cui_rdmsr_imm_reg; 862 863 gpa_t time; 864 s8 pvclock_tsc_shift; 865 u32 pvclock_tsc_mul; 866 u64 hw_tsc_hz; 867 struct gfn_to_pfn_cache pv_time; 868 /* set guest stopped flag in pvclock flags field */ 869 bool pvclock_set_guest_stopped_request; 870 871 struct { 872 u8 preempted; 873 u64 msr_val; 874 u64 last_steal; 875 struct gfn_to_hva_cache cache; 876 } st; 877 878 u64 l1_tsc_offset; 879 u64 tsc_offset; /* current tsc offset */ 880 u64 last_guest_tsc; 881 u64 last_host_tsc; 882 u64 tsc_offset_adjustment; 883 u64 this_tsc_nsec; 884 u64 this_tsc_write; 885 u64 this_tsc_generation; 886 bool tsc_catchup; 887 bool tsc_always_catchup; 888 s8 virtual_tsc_shift; 889 u32 virtual_tsc_mult; 890 u32 virtual_tsc_khz; 891 s64 ia32_tsc_adjust_msr; 892 u64 msr_ia32_power_ctl; 893 u64 l1_tsc_scaling_ratio; 894 u64 tsc_scaling_ratio; /* current scaling ratio */ 895 896 atomic_t nmi_queued; /* unprocessed asynchronous NMIs */ 897 /* Number of NMIs pending injection, not including hardware vNMIs. */ 898 unsigned int nmi_pending; 899 bool nmi_injected; /* Trying to inject an NMI this entry */ 900 bool smi_pending; /* SMI queued after currently running handler */ 901 u8 handling_intr_from_guest; 902 903 struct kvm_mtrr mtrr_state; 904 u64 pat; 905 906 unsigned switch_db_regs; 907 unsigned long db[KVM_NR_DB_REGS]; 908 unsigned long dr6; 909 unsigned long dr7; 910 unsigned long eff_db[KVM_NR_DB_REGS]; 911 unsigned long guest_debug_dr7; 912 u64 msr_platform_info; 913 u64 msr_misc_features_enables; 914 915 u64 mcg_cap; 916 u64 mcg_status; 917 u64 mcg_ctl; 918 u64 mcg_ext_ctl; 919 u64 *mce_banks; 920 u64 *mci_ctl2_banks; 921 922 /* Cache MMIO info */ 923 u64 mmio_gva; 924 unsigned mmio_access; 925 gfn_t mmio_gfn; 926 u64 mmio_gen; 927 928 struct kvm_pmu pmu; 929 930 /* used for guest single stepping over the given code position */ 931 unsigned long singlestep_rip; 932 933 #ifdef CONFIG_KVM_HYPERV 934 bool hyperv_enabled; 935 struct kvm_vcpu_hv *hyperv; 936 #endif 937 #ifdef CONFIG_KVM_XEN 938 struct kvm_vcpu_xen xen; 939 #endif 940 cpumask_var_t wbinvd_dirty_mask; 941 942 unsigned long last_retry_eip; 943 unsigned long last_retry_addr; 944 945 struct { 946 bool halted; 947 gfn_t gfns[ASYNC_PF_PER_VCPU]; 948 struct gfn_to_hva_cache data; 949 u64 msr_en_val; /* MSR_KVM_ASYNC_PF_EN */ 950 u64 msr_int_val; /* MSR_KVM_ASYNC_PF_INT */ 951 u16 vec; 952 u32 id; 953 u32 host_apf_flags; 954 bool pageready_pending; 955 } apf; 956 957 /* OSVW MSRs (AMD only) */ 958 struct { 959 u64 length; 960 u64 status; 961 } osvw; 962 963 struct { 964 u64 msr_val; 965 struct gfn_to_hva_cache data; 966 } pv_eoi; 967 968 u64 msr_kvm_poll_control; 969 970 /* pv related host specific info */ 971 struct { 972 bool pv_unhalted; 973 } pv; 974 975 u8 pending_ioapic_eoi; 976 int pending_external_vector; 977 int highest_stale_pending_ioapic_eoi; 978 979 /* be preempted when it's in kernel-mode(cpl=0) */ 980 bool preempted_in_kernel; 981 982 /* Host CPU on which VM-entry was most recently attempted */ 983 int last_vmentry_cpu; 984 985 /* AMD MSRC001_0015 Hardware Configuration */ 986 u64 msr_hwcr; 987 988 /* pv related cpuid info */ 989 struct { 990 /* 991 * value of the eax register in the KVM_CPUID_FEATURES CPUID 992 * leaf. 993 */ 994 u32 features; 995 996 /* 997 * indicates whether pv emulation should be disabled if features 998 * are not present in the guest's cpuid 999 */ 1000 bool enforce; 1001 } pv_cpuid; 1002 1003 /* Protected Guests */ 1004 bool guest_state_protected; 1005 bool guest_tsc_protected; 1006 1007 /* 1008 * Set when PDPTS were loaded directly by the userspace without 1009 * reading the guest memory 1010 */ 1011 bool pdptrs_from_userspace; 1012 1013 /* 1014 * Set if an emulated nested VM-Enter to L2 is pending completion. KVM 1015 * must not synthesize a VM-Exit to L1 before entering L2, as VM-Exits 1016 * can only occur at instruction boundaries. The only exception is 1017 * VMX's "notify" exits, which exist in large part to break the CPU out 1018 * of infinite ucode loops, but can corrupt vCPU state in the process! 1019 * 1020 * For all intents and purposes, this is a boolean, but it's tracked as 1021 * a u8 so that KVM can detect when userspace may have stuffed vCPU 1022 * state and generated an architecturally-impossible VM-Exit. 1023 */ 1024 #define KVM_NESTED_RUN_PENDING 1 1025 #define KVM_NESTED_RUN_PENDING_UNTRUSTED 2 1026 u8 nested_run_pending; 1027 1028 #if IS_ENABLED(CONFIG_HYPERV) 1029 hpa_t hv_root_tdp; 1030 #endif 1031 }; 1032 1033 struct kvm_lpage_info { 1034 int disallow_lpage; 1035 }; 1036 1037 struct kvm_arch_memory_slot { 1038 struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES]; 1039 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; 1040 unsigned short *gfn_write_track; 1041 }; 1042 1043 /* Hyper-V synthetic debugger (SynDbg)*/ 1044 struct kvm_hv_syndbg { 1045 struct { 1046 u64 control; 1047 u64 status; 1048 u64 send_page; 1049 u64 recv_page; 1050 u64 pending_page; 1051 } control; 1052 u64 options; 1053 }; 1054 1055 /* Current state of Hyper-V TSC page clocksource */ 1056 enum hv_tsc_page_status { 1057 /* TSC page was not set up or disabled */ 1058 HV_TSC_PAGE_UNSET = 0, 1059 /* TSC page MSR was written by the guest, update pending */ 1060 HV_TSC_PAGE_GUEST_CHANGED, 1061 /* TSC page update was triggered from the host side */ 1062 HV_TSC_PAGE_HOST_CHANGED, 1063 /* TSC page was properly set up and is currently active */ 1064 HV_TSC_PAGE_SET, 1065 /* TSC page was set up with an inaccessible GPA */ 1066 HV_TSC_PAGE_BROKEN, 1067 }; 1068 1069 #ifdef CONFIG_KVM_HYPERV 1070 /* Hyper-V emulation context */ 1071 struct kvm_hv { 1072 struct mutex hv_lock; 1073 u64 hv_guest_os_id; 1074 u64 hv_hypercall; 1075 u64 hv_tsc_page; 1076 enum hv_tsc_page_status hv_tsc_page_status; 1077 1078 /* Hyper-v based guest crash (NT kernel bugcheck) parameters */ 1079 u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS]; 1080 u64 hv_crash_ctl; 1081 1082 struct ms_hyperv_tsc_page tsc_ref; 1083 1084 struct idr conn_to_evt; 1085 1086 u64 hv_reenlightenment_control; 1087 u64 hv_tsc_emulation_control; 1088 u64 hv_tsc_emulation_status; 1089 u64 hv_invtsc_control; 1090 1091 /* How many vCPUs have VP index != vCPU index */ 1092 atomic_t num_mismatched_vp_indexes; 1093 1094 /* 1095 * How many SynICs use 'AutoEOI' feature 1096 * (protected by arch.apicv_update_lock) 1097 */ 1098 unsigned int synic_auto_eoi_used; 1099 1100 struct kvm_hv_syndbg hv_syndbg; 1101 1102 bool xsaves_xsavec_checked; 1103 }; 1104 #endif 1105 1106 #ifdef CONFIG_KVM_XEN 1107 /* Xen emulation context */ 1108 struct kvm_xen { 1109 struct mutex xen_lock; 1110 u32 xen_version; 1111 bool long_mode; 1112 bool runstate_update_flag; 1113 u8 upcall_vector; 1114 struct gfn_to_pfn_cache shinfo_cache; 1115 struct xarray evtchn_ports; 1116 unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 1117 1118 struct kvm_xen_hvm_config hvm_config; 1119 }; 1120 #endif 1121 1122 enum kvm_irqchip_mode { 1123 KVM_IRQCHIP_NONE, 1124 #ifdef CONFIG_KVM_IOAPIC 1125 KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */ 1126 #endif 1127 KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */ 1128 }; 1129 1130 enum kvm_suppress_eoi_broadcast_mode { 1131 KVM_SUPPRESS_EOI_BROADCAST_QUIRKED, /* Legacy behavior */ 1132 KVM_SUPPRESS_EOI_BROADCAST_ENABLED, /* Enable Suppress EOI broadcast */ 1133 KVM_SUPPRESS_EOI_BROADCAST_DISABLED /* Disable Suppress EOI broadcast */ 1134 }; 1135 1136 struct kvm_possible_nx_huge_pages { 1137 /* 1138 * A list of kvm_mmu_page structs that, if zapped, could possibly be 1139 * replaced by an NX huge page. A shadow page is on this list if its 1140 * existence disallows an NX huge page (nx_huge_page_disallowed is set) 1141 * and there are no other conditions that prevent a huge page, e.g. 1142 * the backing host page is huge, dirtly logging is not enabled for its 1143 * memslot, etc... Note, zapping shadow pages on this list doesn't 1144 * guarantee an NX huge page will be created in its stead, e.g. if the 1145 * guest attempts to execute from the region then KVM obviously can't 1146 * create an NX huge page (without hanging the guest). 1147 */ 1148 struct list_head pages; 1149 u64 nr_pages; 1150 }; 1151 1152 enum kvm_mmu_type { 1153 KVM_SHADOW_MMU, 1154 #ifdef CONFIG_X86_64 1155 KVM_TDP_MMU, 1156 #endif 1157 KVM_NR_MMU_TYPES, 1158 }; 1159 1160 struct kvm_arch { 1161 unsigned long n_requested_mmu_pages; 1162 unsigned long n_max_mmu_pages; 1163 unsigned int indirect_shadow_pages; 1164 u8 mmu_valid_gen; 1165 u8 vm_type; 1166 bool has_private_mem; 1167 bool has_protected_page_tables; 1168 bool has_protected_state; 1169 bool has_protected_eoi; 1170 bool has_protected_pmu; 1171 bool pre_fault_allowed; 1172 struct hlist_head *mmu_page_hash; 1173 struct list_head active_mmu_pages; 1174 struct kvm_possible_nx_huge_pages possible_nx_huge_pages[KVM_NR_MMU_TYPES]; 1175 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING 1176 struct kvm_page_track_notifier_head track_notifier_head; 1177 #endif 1178 /* 1179 * Protects marking pages unsync during page faults, as TDP MMU page 1180 * faults only take mmu_lock for read. For simplicity, the unsync 1181 * pages lock is always taken when marking pages unsync regardless of 1182 * whether mmu_lock is held for read or write. 1183 */ 1184 spinlock_t mmu_unsync_pages_lock; 1185 1186 u64 shadow_mmio_value; 1187 1188 #define __KVM_HAVE_ARCH_NONCOHERENT_DMA 1189 atomic_t noncoherent_dma_count; 1190 unsigned long nr_possible_bypass_irqs; 1191 1192 #ifdef CONFIG_KVM_IOAPIC 1193 struct kvm_pic *vpic; 1194 struct kvm_ioapic *vioapic; 1195 struct kvm_pit *vpit; 1196 #endif 1197 atomic_t vapics_in_nmi_mode; 1198 1199 struct mutex apic_map_lock; 1200 struct kvm_apic_map __rcu *apic_map; 1201 atomic_t apic_map_dirty; 1202 1203 bool apic_access_memslot_enabled; 1204 bool apic_access_memslot_inhibited; 1205 1206 /* 1207 * Force apicv_update_lock and apicv_nr_irq_window_req to reside in a 1208 * dedicated cacheline. They are write-mostly, whereas most everything 1209 * else in kvm_arch is read-mostly. Note that apicv_inhibit_reasons is 1210 * read-mostly: toggling VM-wide inhibits is rare; _checking_ for 1211 * inhibits is common. 1212 */ 1213 ____cacheline_aligned 1214 /* 1215 * Protects apicv_inhibit_reasons and apicv_nr_irq_window_req (with an 1216 * asterisk, see kvm_inc_or_dec_irq_window_inhibit() for details). 1217 */ 1218 struct rw_semaphore apicv_update_lock; 1219 atomic_t apicv_nr_irq_window_req; 1220 ____cacheline_aligned 1221 1222 unsigned long apicv_inhibit_reasons; 1223 1224 gpa_t wall_clock; 1225 1226 u64 disabled_exits; 1227 1228 s64 kvmclock_offset; 1229 1230 /* 1231 * This also protects nr_vcpus_matched_tsc which is read from a 1232 * preemption-disabled region, so it must be a raw spinlock. 1233 */ 1234 raw_spinlock_t tsc_write_lock; 1235 u64 last_tsc_nsec; 1236 u64 last_tsc_write; 1237 u32 last_tsc_khz; 1238 u64 last_tsc_offset; 1239 u64 last_tsc_scaling_ratio; 1240 u64 cur_tsc_nsec; 1241 u64 cur_tsc_write; 1242 u64 cur_tsc_offset; 1243 u64 cur_tsc_generation; 1244 int nr_vcpus_matched_tsc; 1245 1246 u32 default_tsc_khz; 1247 bool user_set_tsc; 1248 u64 apic_bus_cycle_ns; 1249 1250 seqcount_raw_spinlock_t pvclock_sc; 1251 bool use_master_clock; 1252 u64 master_kernel_ns; 1253 u64 master_cycle_now; 1254 struct ratelimit_state kvmclock_update_rs; 1255 u64 master_tsc_scaling_ratio; 1256 s8 master_tsc_shift; 1257 u32 master_tsc_mul; 1258 1259 #ifdef CONFIG_KVM_HYPERV 1260 struct kvm_hv hyperv; 1261 #endif 1262 1263 #ifdef CONFIG_KVM_XEN 1264 struct kvm_xen xen; 1265 #endif 1266 1267 bool backwards_tsc_observed; 1268 bool boot_vcpu_runs_old_kvmclock; 1269 u32 bsp_vcpu_id; 1270 1271 u64 disabled_quirks; 1272 1273 enum kvm_irqchip_mode irqchip_mode; 1274 u8 nr_reserved_ioapic_pins; 1275 1276 bool disabled_lapic_found; 1277 1278 bool x2apic_format; 1279 bool x2apic_broadcast_quirk_disabled; 1280 enum kvm_suppress_eoi_broadcast_mode suppress_eoi_broadcast_mode; 1281 1282 bool has_mapped_host_mmio; 1283 bool guest_can_read_msr_platform_info; 1284 bool exception_payload_enabled; 1285 1286 bool triple_fault_event; 1287 1288 bool bus_lock_detection_enabled; 1289 bool enable_pmu; 1290 bool created_mediated_pmu; 1291 1292 u32 notify_window; 1293 u32 notify_vmexit_flags; 1294 /* 1295 * If exit_on_emulation_error is set, and the in-kernel instruction 1296 * emulator fails to emulate an instruction, allow userspace 1297 * the opportunity to look at it. 1298 */ 1299 bool exit_on_emulation_error; 1300 1301 /* Deflect RDMSR and WRMSR to user space when they trigger a #GP */ 1302 u32 user_space_msr_mask; 1303 struct kvm_x86_msr_filter __rcu *msr_filter; 1304 1305 u32 hypercall_exit_enabled; 1306 1307 /* Guest can access the SGX PROVISIONKEY. */ 1308 bool sgx_provisioning_allowed; 1309 1310 struct kvm_x86_pmu_event_filter __rcu *pmu_event_filter; 1311 struct vhost_task *nx_huge_page_recovery_thread; 1312 u64 nx_huge_page_last; 1313 struct once nx_once; 1314 1315 #ifdef CONFIG_X86_64 1316 #ifdef CONFIG_KVM_PROVE_MMU 1317 /* 1318 * The number of TDP MMU pages across all roots. Used only to sanity 1319 * check that KVM isn't leaking TDP MMU pages. 1320 */ 1321 atomic64_t tdp_mmu_pages; 1322 #endif 1323 1324 /* 1325 * List of struct kvm_mmu_pages being used as roots. 1326 * All struct kvm_mmu_pages in the list should have 1327 * tdp_mmu_page set. 1328 * 1329 * For reads, this list is protected by: 1330 * RCU alone or 1331 * the MMU lock in read mode + RCU or 1332 * the MMU lock in write mode 1333 * 1334 * For writes, this list is protected by tdp_mmu_pages_lock; see 1335 * below for the details. 1336 * 1337 * Roots will remain in the list until their tdp_mmu_root_count 1338 * drops to zero, at which point the thread that decremented the 1339 * count to zero should removed the root from the list and clean 1340 * it up, freeing the root after an RCU grace period. 1341 */ 1342 struct list_head tdp_mmu_roots; 1343 1344 /* 1345 * Protects accesses to the following fields when the MMU lock 1346 * is held in read mode: 1347 * - tdp_mmu_roots (above) 1348 * - the link field of kvm_mmu_page structs used by the TDP MMU 1349 * - possible_nx_huge_pages[KVM_TDP_MMU]; 1350 * - the possible_nx_huge_page_link field of kvm_mmu_page structs used 1351 * by the TDP MMU 1352 * Because the lock is only taken within the MMU lock, strictly 1353 * speaking it is redundant to acquire this lock when the thread 1354 * holds the MMU lock in write mode. However it often simplifies 1355 * the code to do so. 1356 */ 1357 spinlock_t tdp_mmu_pages_lock; 1358 #endif /* CONFIG_X86_64 */ 1359 1360 /* 1361 * If set, at least one shadow root has been allocated. This flag 1362 * is used as one input when determining whether certain memslot 1363 * related allocations are necessary. 1364 */ 1365 bool shadow_root_allocated; 1366 1367 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING 1368 /* 1369 * If set, the VM has (or had) an external write tracking user, and 1370 * thus all write tracking metadata has been allocated, even if KVM 1371 * itself isn't using write tracking. 1372 */ 1373 bool external_write_tracking_enabled; 1374 #endif 1375 1376 #if IS_ENABLED(CONFIG_HYPERV) 1377 hpa_t hv_root_tdp; 1378 spinlock_t hv_root_tdp_lock; 1379 struct hv_partition_assist_pg *hv_pa_pg; 1380 #endif 1381 /* 1382 * VM-scope maximum vCPU ID. Used to determine the size of structures 1383 * that increase along with the maximum vCPU ID, in which case, using 1384 * the global KVM_MAX_VCPU_IDS may lead to significant memory waste. 1385 */ 1386 u32 max_vcpu_ids; 1387 1388 bool disable_nx_huge_pages; 1389 1390 /* 1391 * Memory caches used to allocate shadow pages when performing eager 1392 * page splitting. No need for a shadowed_info_cache since eager page 1393 * splitting only allocates direct shadow pages. 1394 * 1395 * Protected by kvm->slots_lock. 1396 */ 1397 struct kvm_mmu_memory_cache split_shadow_page_cache; 1398 struct kvm_mmu_memory_cache split_page_header_cache; 1399 1400 /* 1401 * Memory cache used to allocate pte_list_desc structs while splitting 1402 * huge pages. In the worst case, to split one huge page, 512 1403 * pte_list_desc structs are needed to add each lower level leaf sptep 1404 * to the rmap plus 1 to extend the parent_ptes rmap of the lower level 1405 * page table. 1406 * 1407 * Protected by kvm->slots_lock. 1408 */ 1409 #define SPLIT_DESC_CACHE_MIN_NR_OBJECTS (SPTE_ENT_PER_PAGE + 1) 1410 struct kvm_mmu_memory_cache split_desc_cache; 1411 1412 gfn_t gfn_direct_bits; 1413 int mirror_root_level; 1414 1415 /* 1416 * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero 1417 * value indicates CPU dirty logging is unsupported or disabled in 1418 * current VM. 1419 */ 1420 int cpu_dirty_log_size; 1421 }; 1422 1423 struct kvm_vm_stat { 1424 struct kvm_vm_stat_generic generic; 1425 u64 mmu_shadow_zapped; 1426 u64 mmu_pte_write; 1427 u64 mmu_pde_zapped; 1428 u64 mmu_flooded; 1429 u64 mmu_recycled; 1430 u64 mmu_cache_miss; 1431 u64 mmu_unsync; 1432 u64 mmu_shadow_pages; 1433 union { 1434 struct { 1435 atomic64_t pages_4k; 1436 atomic64_t pages_2m; 1437 atomic64_t pages_1g; 1438 }; 1439 atomic64_t pages[KVM_NR_PAGE_SIZES]; 1440 }; 1441 u64 nx_lpage_splits; 1442 u64 max_mmu_page_hash_collisions; 1443 u64 max_mmu_rmap_size; 1444 }; 1445 1446 struct kvm_vcpu_stat { 1447 struct kvm_vcpu_stat_generic generic; 1448 u64 pf_taken; 1449 u64 pf_fixed; 1450 u64 pf_emulate; 1451 u64 pf_spurious; 1452 u64 pf_fast; 1453 u64 pf_mmio_spte_created; 1454 u64 pf_guest; 1455 u64 tlb_flush; 1456 u64 invlpg; 1457 1458 u64 exits; 1459 u64 io_exits; 1460 u64 mmio_exits; 1461 u64 signal_exits; 1462 u64 irq_window_exits; 1463 u64 nmi_window_exits; 1464 u64 l1d_flush; 1465 u64 halt_exits; 1466 u64 request_irq_exits; 1467 u64 irq_exits; 1468 u64 host_state_reload; 1469 u64 fpu_reload; 1470 u64 insn_emulation; 1471 u64 insn_emulation_fail; 1472 u64 hypercalls; 1473 u64 irq_injections; 1474 u64 nmi_injections; 1475 u64 req_event; 1476 u64 nested_run; 1477 u64 directed_yield_attempted; 1478 u64 directed_yield_successful; 1479 u64 preemption_reported; 1480 u64 preemption_other; 1481 u64 guest_mode; 1482 u64 notify_window_exits; 1483 }; 1484 1485 struct x86_instruction_info; 1486 1487 struct msr_data { 1488 bool host_initiated; 1489 u32 index; 1490 u64 data; 1491 }; 1492 1493 struct kvm_lapic_irq { 1494 u32 vector; 1495 u16 delivery_mode; 1496 u16 dest_mode; 1497 bool level; 1498 u16 trig_mode; 1499 u32 shorthand; 1500 u32 dest_id; 1501 bool msi_redir_hint; 1502 }; 1503 1504 enum kvm_x86_run_flags { 1505 KVM_RUN_FORCE_IMMEDIATE_EXIT = BIT(0), 1506 KVM_RUN_LOAD_GUEST_DR6 = BIT(1), 1507 KVM_RUN_LOAD_DEBUGCTL = BIT(2), 1508 }; 1509 1510 struct kvm_x86_ops { 1511 const char *name; 1512 1513 int (*check_processor_compatibility)(void); 1514 1515 int (*enable_virtualization_cpu)(void); 1516 void (*disable_virtualization_cpu)(void); 1517 cpu_emergency_virt_cb *emergency_disable_virtualization_cpu; 1518 1519 void (*hardware_unsetup)(void); 1520 bool (*has_emulated_msr)(struct kvm *kvm, u32 index); 1521 void (*vcpu_after_set_cpuid)(struct kvm_vcpu *vcpu); 1522 1523 unsigned int vm_size; 1524 int (*vm_init)(struct kvm *kvm); 1525 void (*vm_destroy)(struct kvm *kvm); 1526 void (*vm_pre_destroy)(struct kvm *kvm); 1527 1528 /* Create, but do not attach this VCPU */ 1529 int (*vcpu_precreate)(struct kvm *kvm); 1530 int (*vcpu_create)(struct kvm_vcpu *vcpu); 1531 void (*vcpu_free)(struct kvm_vcpu *vcpu); 1532 void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event); 1533 1534 void (*prepare_switch_to_guest)(struct kvm_vcpu *vcpu); 1535 void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); 1536 void (*vcpu_put)(struct kvm_vcpu *vcpu); 1537 1538 /* 1539 * Mask of DEBUGCTL bits that are owned by the host, i.e. that need to 1540 * match the host's value even while the guest is active. 1541 */ 1542 const u64 HOST_OWNED_DEBUGCTL; 1543 1544 void (*update_exception_bitmap)(struct kvm_vcpu *vcpu); 1545 int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); 1546 int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); 1547 u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); 1548 void (*get_segment)(struct kvm_vcpu *vcpu, 1549 struct kvm_segment *var, int seg); 1550 int (*get_cpl)(struct kvm_vcpu *vcpu); 1551 int (*get_cpl_no_cache)(struct kvm_vcpu *vcpu); 1552 void (*set_segment)(struct kvm_vcpu *vcpu, 1553 struct kvm_segment *var, int seg); 1554 void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); 1555 bool (*is_valid_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); 1556 void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); 1557 void (*post_set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); 1558 bool (*is_valid_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); 1559 void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); 1560 int (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); 1561 void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1562 void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1563 void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1564 void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1565 void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu); 1566 void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value); 1567 void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg); 1568 unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); 1569 void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); 1570 bool (*get_if_flag)(struct kvm_vcpu *vcpu); 1571 1572 void (*flush_tlb_all)(struct kvm_vcpu *vcpu); 1573 void (*flush_tlb_current)(struct kvm_vcpu *vcpu); 1574 #if IS_ENABLED(CONFIG_HYPERV) 1575 int (*flush_remote_tlbs)(struct kvm *kvm); 1576 int (*flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn, 1577 gfn_t nr_pages); 1578 #endif 1579 1580 /* 1581 * Flush any TLB entries associated with the given GVA. 1582 * Does not need to flush GPA->HPA mappings. 1583 * Can potentially get non-canonical addresses through INVLPGs, which 1584 * the implementation may choose to ignore if appropriate. 1585 */ 1586 void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr); 1587 1588 /* 1589 * Flush any TLB entries created by the guest. Like tlb_flush_gva(), 1590 * does not need to flush GPA->HPA mappings. 1591 */ 1592 void (*flush_tlb_guest)(struct kvm_vcpu *vcpu); 1593 1594 bool (*vcpu_needs_initialization)(struct kvm_vcpu *vcpu); 1595 enum exit_fastpath_completion (*vcpu_run)(struct kvm_vcpu *vcpu, 1596 u64 run_flags); 1597 int (*handle_exit)(struct kvm_vcpu *vcpu, 1598 enum exit_fastpath_completion exit_fastpath); 1599 int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); 1600 void (*update_emulated_instruction)(struct kvm_vcpu *vcpu); 1601 bool (*unhandleable_emulation_required)(struct kvm_vcpu *vcpu); 1602 1603 void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask); 1604 u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu); 1605 void (*patch_hypercall)(struct kvm_vcpu *vcpu, 1606 unsigned char *hypercall_addr); 1607 void (*inject_irq)(struct kvm_vcpu *vcpu, bool reinjected); 1608 void (*inject_nmi)(struct kvm_vcpu *vcpu); 1609 void (*inject_exception)(struct kvm_vcpu *vcpu); 1610 void (*cancel_injection)(struct kvm_vcpu *vcpu); 1611 int (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection); 1612 int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); 1613 bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); 1614 void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); 1615 /* 1616 * Force the vCPU out of any hardware-tracked halted/inactive state so 1617 * that it will fetch and execute instructions on the next VM-Enter. 1618 * Only needed by VMX, where VMCS.GUEST_ACTIVITY_STATE persists across 1619 * VM-Exit/VM-Enter; SVM has no equivalent VMCB field. 1620 */ 1621 void (*clear_hlt)(struct kvm_vcpu *vcpu); 1622 /* Whether or not a virtual NMI is pending in hardware. */ 1623 bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu); 1624 /* 1625 * Attempt to pend a virtual NMI in hardware. Returns %true on success 1626 * to allow using static_call_ret0 as the fallback. 1627 */ 1628 bool (*set_vnmi_pending)(struct kvm_vcpu *vcpu); 1629 void (*enable_nmi_window)(struct kvm_vcpu *vcpu); 1630 void (*enable_irq_window)(struct kvm_vcpu *vcpu); 1631 void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr); 1632 1633 const bool x2apic_icr_is_split; 1634 const unsigned long required_apicv_inhibits; 1635 bool allow_apicv_in_x2apic_without_x2apic_virtualization; 1636 void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu); 1637 void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr); 1638 void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); 1639 void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); 1640 void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu); 1641 void (*deliver_interrupt)(struct kvm_lapic *apic, int delivery_mode, 1642 int trig_mode, int vector); 1643 int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu); 1644 int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); 1645 int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr); 1646 u8 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio); 1647 bool (*tdp_has_smep)(struct kvm *kvm); 1648 1649 void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, hpa_t root_hpa, 1650 int root_level); 1651 1652 /* Update the external page table from spte getting set. */ 1653 int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, u64 old_spte, 1654 u64 new_spte, enum pg_level level); 1655 1656 /* Update external page tables for page table about to be freed. */ 1657 void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp); 1658 1659 int (*topup_external_cache)(struct kvm_vcpu *vcpu, int min_nr_spts); 1660 1661 bool (*has_wbinvd_exit)(void); 1662 1663 u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu); 1664 u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu); 1665 void (*write_tsc_offset)(struct kvm_vcpu *vcpu); 1666 void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu); 1667 1668 /* 1669 * Retrieve somewhat arbitrary exit/entry information. Intended to 1670 * be used only from within tracepoints or error paths. 1671 */ 1672 void (*get_exit_info)(struct kvm_vcpu *vcpu, u32 *reason, 1673 u64 *info1, u64 *info2, 1674 u32 *intr_info, u32 *error_code); 1675 1676 void (*get_entry_info)(struct kvm_vcpu *vcpu, 1677 u32 *intr_info, u32 *error_code); 1678 1679 int (*check_intercept)(struct kvm_vcpu *vcpu, 1680 struct x86_instruction_info *info, 1681 enum x86_intercept_stage stage, 1682 struct x86_exception *exception); 1683 void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); 1684 1685 void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable); 1686 1687 void (*vcpu_blocking)(struct kvm_vcpu *vcpu); 1688 void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); 1689 1690 int (*pi_update_irte)(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, 1691 unsigned int host_irq, uint32_t guest_irq, 1692 struct kvm_vcpu *vcpu, u32 vector); 1693 void (*pi_start_bypass)(struct kvm *kvm); 1694 void (*apicv_pre_state_restore)(struct kvm_vcpu *vcpu); 1695 void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu); 1696 bool (*dy_apicv_has_pending_interrupt)(struct kvm_vcpu *vcpu); 1697 bool (*protected_apic_has_interrupt)(struct kvm_vcpu *vcpu); 1698 1699 int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 1700 bool *expired); 1701 void (*cancel_hv_timer)(struct kvm_vcpu *vcpu); 1702 1703 void (*setup_mce)(struct kvm_vcpu *vcpu); 1704 1705 #ifdef CONFIG_KVM_SMM 1706 int (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); 1707 int (*enter_smm)(struct kvm_vcpu *vcpu, union kvm_smram *smram); 1708 int (*leave_smm)(struct kvm_vcpu *vcpu, const union kvm_smram *smram); 1709 void (*enable_smi_window)(struct kvm_vcpu *vcpu); 1710 #endif 1711 1712 int (*dev_get_attr)(u32 group, u64 attr, u64 *val); 1713 int (*mem_enc_ioctl)(struct kvm *kvm, void __user *argp); 1714 int (*vcpu_mem_enc_ioctl)(struct kvm_vcpu *vcpu, void __user *argp); 1715 int (*vcpu_mem_enc_unlocked_ioctl)(struct kvm_vcpu *vcpu, void __user *argp); 1716 int (*mem_enc_register_region)(struct kvm *kvm, struct kvm_enc_region *argp); 1717 int (*mem_enc_unregister_region)(struct kvm *kvm, struct kvm_enc_region *argp); 1718 int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); 1719 int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); 1720 void (*guest_memory_reclaimed)(struct kvm *kvm); 1721 void (*reload_vmsa)(struct kvm_vcpu *vcpu); 1722 1723 int (*get_feature_msr)(u32 msr, u64 *data); 1724 1725 int (*check_emulate_instruction)(struct kvm_vcpu *vcpu, int emul_type, 1726 void *insn, int insn_len); 1727 1728 bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu); 1729 int (*enable_l2_tlb_flush)(struct kvm_vcpu *vcpu); 1730 1731 void (*migrate_timers)(struct kvm_vcpu *vcpu); 1732 void (*recalc_intercepts)(struct kvm_vcpu *vcpu); 1733 int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err); 1734 1735 void (*vcpu_deliver_sipi_vector)(struct kvm_vcpu *vcpu, u8 vector); 1736 1737 /* 1738 * Returns vCPU specific APICv inhibit reasons 1739 */ 1740 unsigned long (*vcpu_get_apicv_inhibit_reasons)(struct kvm_vcpu *vcpu); 1741 1742 gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); 1743 void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); 1744 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT 1745 int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, 1746 kvm_pfn_t nr_pages); 1747 #endif 1748 #if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM) 1749 void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); 1750 #endif 1751 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE 1752 void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); 1753 #endif 1754 int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); 1755 }; 1756 1757 struct kvm_x86_nested_ops { 1758 bool enabled; 1759 1760 void (*leave_nested)(struct kvm_vcpu *vcpu); 1761 bool (*is_exception_vmexit)(struct kvm_vcpu *vcpu, u8 vector, 1762 u32 error_code); 1763 int (*check_events)(struct kvm_vcpu *vcpu); 1764 bool (*has_events)(struct kvm_vcpu *vcpu, bool for_injection); 1765 void (*triple_fault)(struct kvm_vcpu *vcpu); 1766 int (*get_state)(struct kvm_vcpu *vcpu, 1767 struct kvm_nested_state __user *user_kvm_nested_state, 1768 unsigned user_data_size); 1769 int (*set_state)(struct kvm_vcpu *vcpu, 1770 struct kvm_nested_state __user *user_kvm_nested_state, 1771 struct kvm_nested_state *kvm_state); 1772 bool (*get_nested_state_pages)(struct kvm_vcpu *vcpu); 1773 int (*write_log_dirty)(struct kvm_vcpu *vcpu, gpa_t l2_gpa); 1774 gpa_t (*translate_nested_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, 1775 u64 access, 1776 struct x86_exception *exception, 1777 u64 pte_access); 1778 1779 int (*enable_evmcs)(struct kvm_vcpu *vcpu, 1780 uint16_t *vmcs_version); 1781 uint16_t (*get_evmcs_version)(struct kvm_vcpu *vcpu); 1782 void (*hv_inject_synthetic_vmexit_post_tlb_flush)(struct kvm_vcpu *vcpu); 1783 }; 1784 1785 struct kvm_x86_init_ops { 1786 int (*hardware_setup)(void); 1787 unsigned int (*handle_intel_pt_intr)(void); 1788 1789 struct kvm_x86_ops *runtime_ops; 1790 struct kvm_pmu_ops *pmu_ops; 1791 struct kvm_x86_nested_ops *nested_ops; 1792 }; 1793 1794 struct kvm_arch_async_pf { 1795 u32 token; 1796 gfn_t gfn; 1797 unsigned long cr3; 1798 bool direct_map; 1799 u64 error_code; 1800 }; 1801 1802 extern bool __read_mostly allow_smaller_maxphyaddr; 1803 extern bool __read_mostly enable_apicv; 1804 extern bool __read_mostly enable_ipiv; 1805 extern bool __read_mostly enable_device_posted_irqs; 1806 extern struct kvm_x86_ops kvm_x86_ops; 1807 extern struct kvm_x86_nested_ops kvm_nested_ops __read_mostly; 1808 1809 #define kvm_x86_call(func) static_call(kvm_x86_##func) 1810 1811 #define KVM_X86_OP(func) \ 1812 DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func)); 1813 #define KVM_X86_OP_OPTIONAL KVM_X86_OP 1814 #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP 1815 #include <asm/kvm-x86-ops.h> 1816 1817 #define kvm_nested_call(func) static_call(kvm_x86_nested_##func) 1818 1819 #define KVM_X86_NESTED_OP(func) \ 1820 DECLARE_STATIC_CALL(kvm_x86_nested_##func, *(((struct kvm_x86_nested_ops *)0)->func)); 1821 #define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP 1822 #define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP 1823 #include <asm/kvm-x86-nested-ops.h> 1824 1825 #define __KVM_HAVE_ARCH_VM_ALLOC 1826 static inline struct kvm *kvm_arch_alloc_vm(void) 1827 { 1828 return kvzalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT); 1829 } 1830 1831 #define __KVM_HAVE_ARCH_VM_FREE 1832 void kvm_arch_free_vm(struct kvm *kvm); 1833 1834 #if IS_ENABLED(CONFIG_HYPERV) 1835 #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS 1836 static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm) 1837 { 1838 if (kvm_x86_ops.flush_remote_tlbs && 1839 !kvm_x86_call(flush_remote_tlbs)(kvm)) 1840 return 0; 1841 else 1842 return -ENOTSUPP; 1843 } 1844 1845 #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE 1846 static inline int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, 1847 u64 nr_pages) 1848 { 1849 if (!kvm_x86_ops.flush_remote_tlbs_range) 1850 return -EOPNOTSUPP; 1851 1852 return kvm_x86_call(flush_remote_tlbs_range)(kvm, gfn, nr_pages); 1853 } 1854 #endif /* CONFIG_HYPERV */ 1855 1856 enum kvm_intr_type { 1857 /* Values are arbitrary, but must be non-zero. */ 1858 KVM_HANDLING_IRQ = 1, 1859 KVM_HANDLING_NMI, 1860 }; 1861 1862 /* Enable perf NMI and timer modes to work, and minimise false positives. */ 1863 #define kvm_arch_pmi_in_guest(vcpu) \ 1864 ((vcpu) && (vcpu)->arch.handling_intr_from_guest && \ 1865 (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI))) 1866 1867 #if defined(CONFIG_KVM_SW_PROTECTED_VM) || \ 1868 defined(CONFIG_KVM_INTEL_TDX) || \ 1869 defined(CONFIG_KVM_AMD_SEV) 1870 #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) 1871 #endif 1872 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT 1873 #define kvm_arch_has_gmem_convert() (!!kvm_x86_ops.gmem_make_private) 1874 #endif 1875 1876 #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state) 1877 1878 #ifdef CONFIG_X86_64 1879 static inline unsigned long read_msr(unsigned long msr) 1880 { 1881 u64 value; 1882 1883 rdmsrq(msr, value); 1884 return value; 1885 } 1886 #endif 1887 1888 #define HF_GUEST_MASK (1 << 0) /* VCPU is in guest-mode */ 1889 1890 #ifdef CONFIG_KVM_SMM 1891 #define HF_SMM_MASK (1 << 1) 1892 #define HF_SMM_INSIDE_NMI_MASK (1 << 2) 1893 1894 # define KVM_MAX_NR_ADDRESS_SPACES 2 1895 /* SMM is currently unsupported for guests with private memory. */ 1896 # define kvm_arch_nr_memslot_as_ids(kvm) (kvm_arch_has_private_mem(kvm) ? 1 : 2) 1897 # define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0) 1898 # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm) 1899 #else 1900 # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0) 1901 #endif 1902 1903 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 1904 struct kvm_async_pf *work); 1905 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 1906 struct kvm_async_pf *work); 1907 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, 1908 struct kvm_async_pf *work); 1909 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu); 1910 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); 1911 1912 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 1913 { 1914 kvm_x86_call(vcpu_blocking)(vcpu); 1915 } 1916 1917 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 1918 { 1919 kvm_x86_call(vcpu_unblocking)(vcpu); 1920 } 1921 1922 static inline bool kvm_arch_has_irq_bypass(void) 1923 { 1924 return enable_device_posted_irqs; 1925 } 1926 1927 #endif /* _ASM_X86_KVM_HOST_H */ 1928