1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This header defines architecture specific interfaces, x86 version 6 */ 7 8 #ifndef _ASM_X86_KVM_HOST_H 9 #define _ASM_X86_KVM_HOST_H 10 11 #include <linux/types.h> 12 #include <linux/mm.h> 13 #include <linux/mmu_notifier.h> 14 #include <linux/tracepoint.h> 15 #include <linux/cpumask.h> 16 #include <linux/irq_work.h> 17 #include <linux/irq.h> 18 #include <linux/workqueue.h> 19 20 #include <linux/kvm.h> 21 #include <linux/kvm_para.h> 22 #include <linux/kvm_types.h> 23 #include <linux/perf_event.h> 24 #include <linux/pvclock_gtod.h> 25 #include <linux/clocksource.h> 26 #include <linux/irqbypass.h> 27 #include <linux/kfifo.h> 28 #include <linux/sched/vhost_task.h> 29 #include <linux/call_once.h> 30 #include <linux/atomic.h> 31 32 #include <asm/apic.h> 33 #include <asm/pvclock-abi.h> 34 #include <asm/debugreg.h> 35 #include <asm/desc.h> 36 #include <asm/mtrr.h> 37 #include <asm/msr-index.h> 38 #include <asm/msr.h> 39 #include <asm/asm.h> 40 #include <asm/irq_remapping.h> 41 #include <asm/kvm_page_track.h> 42 #include <asm/kvm_vcpu_regs.h> 43 #include <asm/virt.h> 44 45 #include <hyperv/hvhdk.h> 46 47 #define __KVM_HAVE_ARCH_VCPU_DEBUGFS 48 49 /* 50 * CONFIG_KVM_MAX_NR_VCPUS is defined iff CONFIG_KVM!=n, provide a dummy max if 51 * KVM is disabled (arbitrarily use the default from CONFIG_KVM_MAX_NR_VCPUS). 52 */ 53 #ifdef CONFIG_KVM_MAX_NR_VCPUS 54 #define KVM_MAX_VCPUS CONFIG_KVM_MAX_NR_VCPUS 55 #else 56 #define KVM_MAX_VCPUS 1024 57 #endif 58 59 /* 60 * In x86, the VCPU ID corresponds to the APIC ID, and APIC IDs 61 * might be larger than the actual number of VCPUs because the 62 * APIC ID encodes CPU topology information. 63 * 64 * In the worst case, we'll need less than one extra bit for the 65 * Core ID, and less than one extra bit for the Package (Die) ID, 66 * so ratio of 4 should be enough. 67 */ 68 #define KVM_VCPU_ID_RATIO 4 69 #define KVM_MAX_VCPU_IDS (KVM_MAX_VCPUS * KVM_VCPU_ID_RATIO) 70 71 /* memory slots that are not exposed to userspace */ 72 #define KVM_INTERNAL_MEM_SLOTS 3 73 74 #define KVM_HALT_POLL_NS_DEFAULT 200000 75 76 #define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS 77 78 #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ 79 KVM_DIRTY_LOG_INITIALLY_SET) 80 81 /* x86-specific vcpu->requests bit members */ 82 #define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0) 83 #define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1) 84 #define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2) 85 #define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3) 86 #define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4) 87 #define KVM_REQ_LOAD_MMU_PGD KVM_ARCH_REQ(5) 88 #define KVM_REQ_EVENT KVM_ARCH_REQ(6) 89 #define KVM_REQ_APF_HALT KVM_ARCH_REQ(7) 90 #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8) 91 #define KVM_REQ_NMI KVM_ARCH_REQ(9) 92 #define KVM_REQ_PMU KVM_ARCH_REQ(10) 93 #define KVM_REQ_PMI KVM_ARCH_REQ(11) 94 #ifdef CONFIG_KVM_SMM 95 #define KVM_REQ_SMI KVM_ARCH_REQ(12) 96 #endif 97 #define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13) 98 #define KVM_REQ_MCLOCK_INPROGRESS \ 99 KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 100 #define KVM_REQ_SCAN_IOAPIC \ 101 KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 102 #define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16) 103 #define KVM_REQ_APIC_PAGE_RELOAD \ 104 KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 105 #define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18) 106 #define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19) 107 #define KVM_REQ_HV_RESET KVM_ARCH_REQ(20) 108 #define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21) 109 #define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22) 110 #define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23) 111 #define KVM_REQ_GET_NESTED_STATE_PAGES KVM_ARCH_REQ(24) 112 #define KVM_REQ_APICV_UPDATE \ 113 KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 114 #define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26) 115 #define KVM_REQ_TLB_FLUSH_GUEST \ 116 KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 117 #define KVM_REQ_APF_READY KVM_ARCH_REQ(28) 118 #define KVM_REQ_RECALC_INTERCEPTS KVM_ARCH_REQ(29) 119 #define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \ 120 KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 121 #define KVM_REQ_MMU_FREE_OBSOLETE_ROOTS \ 122 KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 123 #define KVM_REQ_HV_TLB_FLUSH \ 124 KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 125 #define KVM_REQ_VMSA_PAGE_RELOAD \ 126 KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 127 #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \ 128 KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT) 129 130 #define INVALID_PAGE (~(hpa_t)0) 131 #define VALID_PAGE(x) ((x) != INVALID_PAGE) 132 133 /* KVM Hugepage definitions for x86 */ 134 #define KVM_MAX_HUGEPAGE_LEVEL PG_LEVEL_1G 135 #define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - PG_LEVEL_4K + 1) 136 #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) 137 #define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x)) 138 #define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x)) 139 #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) 140 #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) 141 142 #define KVM_MAX_CPUID_ENTRIES 256 143 #define KVM_NR_VAR_MTRR 8 144 145 #define ASYNC_PF_PER_VCPU 64 146 147 enum kvm_reg { 148 VCPU_REGS_RAX = __VCPU_REGS_RAX, 149 VCPU_REGS_RCX = __VCPU_REGS_RCX, 150 VCPU_REGS_RDX = __VCPU_REGS_RDX, 151 VCPU_REGS_RBX = __VCPU_REGS_RBX, 152 VCPU_REGS_RSP = __VCPU_REGS_RSP, 153 VCPU_REGS_RBP = __VCPU_REGS_RBP, 154 VCPU_REGS_RSI = __VCPU_REGS_RSI, 155 VCPU_REGS_RDI = __VCPU_REGS_RDI, 156 #ifdef CONFIG_X86_64 157 VCPU_REGS_R8 = 8, 158 VCPU_REGS_R9, 159 VCPU_REGS_R10, 160 VCPU_REGS_R11, 161 VCPU_REGS_R12, 162 VCPU_REGS_R13, 163 VCPU_REGS_R14, 164 VCPU_REGS_R15, 165 #endif 166 NR_VCPU_GENERAL_PURPOSE_REGS, 167 168 VCPU_REG_RIP = NR_VCPU_GENERAL_PURPOSE_REGS, 169 170 VCPU_REG_PDPTR, 171 VCPU_REG_CR0, 172 /* 173 * Alias AMD's ERAPS (not a real register) to CR3 so that common code 174 * can trigger emulation of the RAP (Return Address Predictor) with 175 * minimal support required in common code. Piggyback CR3 as the RAP 176 * is cleared on writes to CR3, i.e. marking CR3 dirty will naturally 177 * mark ERAPS dirty as well. 178 */ 179 VCPU_REG_CR3, 180 VCPU_REG_ERAPS = VCPU_REG_CR3, 181 VCPU_REG_CR4, 182 VCPU_REG_RFLAGS, 183 VCPU_REG_SEGMENTS, 184 VCPU_REG_EXIT_INFO_1, 185 VCPU_REG_EXIT_INFO_2, 186 187 NR_VCPU_TOTAL_REGS, 188 }; 189 190 enum { 191 VCPU_SREG_ES, 192 VCPU_SREG_CS, 193 VCPU_SREG_SS, 194 VCPU_SREG_DS, 195 VCPU_SREG_FS, 196 VCPU_SREG_GS, 197 VCPU_SREG_TR, 198 VCPU_SREG_LDTR, 199 }; 200 201 enum exit_fastpath_completion { 202 EXIT_FASTPATH_NONE, 203 EXIT_FASTPATH_REENTER_GUEST, 204 EXIT_FASTPATH_EXIT_HANDLED, 205 EXIT_FASTPATH_EXIT_USERSPACE, 206 }; 207 typedef enum exit_fastpath_completion fastpath_t; 208 209 struct x86_emulate_ctxt; 210 struct x86_exception; 211 union kvm_smram; 212 enum x86_intercept; 213 enum x86_intercept_stage; 214 215 #define KVM_NR_DB_REGS 4 216 217 #define PFERR_PRESENT_MASK BIT(0) 218 #define PFERR_WRITE_MASK BIT(1) 219 #define PFERR_USER_MASK BIT(2) 220 #define PFERR_RSVD_MASK BIT(3) 221 #define PFERR_FETCH_MASK BIT(4) 222 #define PFERR_PK_MASK BIT(5) 223 #define PFERR_SS_MASK BIT(6) 224 #define PFERR_SGX_MASK BIT(15) 225 #define PFERR_GUEST_RMP_MASK BIT_ULL(31) 226 #define PFERR_GUEST_FINAL_MASK BIT_ULL(32) 227 #define PFERR_GUEST_PAGE_MASK BIT_ULL(33) 228 #define PFERR_GUEST_FAULT_STAGE_MASK \ 229 (PFERR_GUEST_FINAL_MASK | PFERR_GUEST_PAGE_MASK) 230 #define PFERR_GUEST_ENC_MASK BIT_ULL(34) 231 #define PFERR_GUEST_SIZEM_MASK BIT_ULL(35) 232 #define PFERR_GUEST_VMPL_MASK BIT_ULL(36) 233 234 /* 235 * IMPLICIT_ACCESS is a KVM-defined flag used to correctly perform SMAP checks 236 * when emulating instructions that triggers implicit access. 237 */ 238 #define PFERR_IMPLICIT_ACCESS BIT_ULL(48) 239 /* 240 * PRIVATE_ACCESS is a KVM-defined flag us to indicate that a fault occurred 241 * when the guest was accessing private memory. 242 */ 243 #define PFERR_PRIVATE_ACCESS BIT_ULL(49) 244 #define PFERR_SYNTHETIC_MASK (PFERR_IMPLICIT_ACCESS | PFERR_PRIVATE_ACCESS) 245 246 struct kvm_kernel_irqfd; 247 struct kvm_kernel_irq_routing_entry; 248 249 struct kvm_apic_map; 250 251 struct kvm_x86_msr_filter; 252 struct kvm_x86_pmu_event_filter; 253 254 struct kvm_caps { 255 /* control of guest tsc rate supported? */ 256 bool has_tsc_control; 257 /* maximum supported tsc_khz for guests */ 258 u32 max_guest_tsc_khz; 259 /* number of bits of the fractional part of the TSC scaling ratio */ 260 u8 tsc_scaling_ratio_frac_bits; 261 /* maximum allowed value of TSC scaling ratio */ 262 u64 max_tsc_scaling_ratio; 263 /* 1ull << kvm_caps.tsc_scaling_ratio_frac_bits */ 264 u64 default_tsc_scaling_ratio; 265 /* bus lock detection supported? */ 266 bool has_bus_lock_exit; 267 /* notify VM exit supported? */ 268 bool has_notify_vmexit; 269 /* bit mask of VM types */ 270 u32 supported_vm_types; 271 272 u64 supported_mce_cap; 273 u64 supported_xcr0; 274 u64 supported_xss; 275 u64 supported_perf_cap; 276 277 u64 supported_efer_bits; 278 279 u64 supported_quirks; 280 u64 inapplicable_quirks; 281 }; 282 extern struct kvm_caps kvm_caps; 283 284 struct kvm_host_values { 285 /* 286 * The host's raw MAXPHYADDR, i.e. the number of non-reserved physical 287 * address bits irrespective of features that repurpose legal bits, 288 * e.g. MKTME. 289 */ 290 u8 maxphyaddr; 291 292 u64 efer; 293 u64 xcr0; 294 u64 xss; 295 u64 s_cet; 296 u64 arch_capabilities; 297 }; 298 extern struct kvm_host_values kvm_host; 299 300 /* 301 * kvm_mmu_page_role tracks the properties of a shadow page (where shadow page 302 * also includes TDP pages) to determine whether or not a page can be used in 303 * the given MMU context. This is a subset of the overall kvm_cpu_role to 304 * minimize the size of kvm_memory_slot.arch.gfn_write_track, i.e. allows 305 * allocating 2 bytes per gfn instead of 4 bytes per gfn. 306 * 307 * Upper-level shadow pages having gptes are tracked for write-protection via 308 * gfn_write_track. As above, gfn_write_track is a 16 bit counter, so KVM must 309 * not create more than 2^16-1 upper-level shadow pages at a single gfn, 310 * otherwise gfn_write_track will overflow and explosions will ensue. 311 * 312 * A unique shadow page (SP) for a gfn is created if and only if an existing SP 313 * cannot be reused. The ability to reuse a SP is tracked by its role, which 314 * incorporates various mode bits and properties of the SP. Roughly speaking, 315 * the number of unique SPs that can theoretically be created is 2^n, where n 316 * is the number of bits that are used to compute the role. 317 * 318 * But, even though there are 21 bits in the mask below, not all combinations 319 * of modes and flags are possible: 320 * 321 * - invalid shadow pages are not accounted, mirror pages are not shadowed, 322 * so the bits are effectively 19. 323 * 324 * - quadrant will only be used if has_4_byte_gpte=1 (non-PAE paging); 325 * execonly and ad_disabled are only used for nested EPT which has 326 * has_4_byte_gpte=0. Therefore, 2 bits are always unused. 327 * 328 * - the 4 bits of level are effectively limited to the values 2/3/4/5, 329 * as 4k SPs are not tracked (allowed to go unsync). In addition non-PAE 330 * paging has exactly one upper level, making level completely redundant 331 * when has_4_byte_gpte=1. 332 * 333 * - on top of this, smap_andnot_wp is only set if cr0_wp=0, 334 * therefore these two bits only give rise to 3 possibilities. 335 * 336 * Therefore, the maximum number of possible upper-level shadow pages for a 337 * single gfn is a bit less than 2^14. 338 */ 339 union kvm_mmu_page_role { 340 u32 word; 341 struct { 342 unsigned level:4; 343 unsigned has_4_byte_gpte:1; 344 unsigned quadrant:2; 345 unsigned direct:1; 346 unsigned access:4; 347 unsigned invalid:1; 348 unsigned efer_nx:1; 349 unsigned cr0_wp:1; 350 unsigned smap_andnot_wp:1; 351 unsigned ad_disabled:1; 352 unsigned guest_mode:1; 353 unsigned passthrough:1; 354 unsigned is_mirror:1; 355 356 /* 357 * cr4_smep is also set for EPT MBEC. Because it affects 358 * which pages are considered non-present (bit 10 additionally 359 * must be zero if MBEC is on) it has to be in the base role. 360 * It also has to be in the base role for AMD GMET because 361 * kernel-executable pages need to have U=0 with GMET enabled. 362 */ 363 unsigned cr4_smep:1; 364 365 unsigned:3; 366 367 /* 368 * This is left at the top of the word so that 369 * kvm_memslots_for_spte_role can extract it with a 370 * simple shift. While there is room, give it a whole 371 * byte so it is also faster to load it from memory. 372 */ 373 unsigned smm:8; 374 }; 375 }; 376 377 /* 378 * kvm_mmu_extended_role complements kvm_mmu_page_role, tracking properties 379 * relevant to the current MMU configuration. When loading CR0, CR4, or EFER, 380 * including on nested transitions, if nothing in the full role changes then 381 * MMU re-configuration can be skipped. @valid bit is set on first usage so we 382 * don't treat all-zero structure as valid data. 383 * 384 * The properties that are tracked in the extended role but not the page role 385 * are for things that either (a) do not affect the validity of the shadow page 386 * or (b) are indirectly reflected in the shadow page's role. For example, 387 * CR4.PKE only affects permission checks for software walks of the guest page 388 * tables (because KVM doesn't support Protection Keys with shadow paging), and 389 * CR0.PG, CR4.PAE, and CR4.PSE are indirectly reflected in role.level. 390 * 391 * Note, SMAP is not redundant with smap_andnot_wp in the page role. If 392 * CR0.WP=1, KVM can reuse shadow pages for the guest regardless of SMAP, 393 * but the MMU's permission checks for software walks need to be SMAP 394 * aware regardless of CR0.WP. 395 */ 396 union kvm_mmu_extended_role { 397 u32 word; 398 struct { 399 unsigned int valid:1; 400 unsigned int execonly:1; 401 unsigned int cr4_pse:1; 402 unsigned int cr4_pke:1; 403 unsigned int cr4_smap:1; 404 unsigned int cr4_la57:1; 405 unsigned int efer_lma:1; 406 407 /* 408 * True if either CR4.SMEP or EFER.NXE are set. For AMD NPT 409 * this is the "real" host CR4.SMEP whereas cr4_smep is 410 * actually GMET. 411 */ 412 unsigned int has_pferr_fetch:1; 413 }; 414 }; 415 416 union kvm_cpu_role { 417 u64 as_u64; 418 struct { 419 union kvm_mmu_page_role base; 420 union kvm_mmu_extended_role ext; 421 }; 422 }; 423 424 struct kvm_rmap_head { 425 atomic_long_t val; 426 }; 427 428 struct kvm_pio_request { 429 unsigned long count; 430 int in; 431 int port; 432 int size; 433 }; 434 435 #define PT64_ROOT_MAX_LEVEL 5 436 437 struct kvm_page_format { 438 u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL]; 439 u64 bad_mt_xwr; 440 441 /* 442 * The pkru_mask indicates if protection key checks are needed. It 443 * consists of 16 domains indexed by page fault error code bits [4:1], 444 * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables. 445 * Each domain has 2 bits which are ANDed with AD and WD from PKRU. 446 */ 447 u32 pkru_mask; 448 449 /* 450 * Bitmap; bit set = permission fault 451 * Array index: page fault error code [4:1] 452 * Bit index: pte permissions in ACC_* format 453 */ 454 u16 permissions[16]; 455 }; 456 457 struct kvm_mmu_root_info { 458 gpa_t pgd; 459 hpa_t hpa; 460 }; 461 462 #define KVM_MMU_ROOT_INFO_INVALID \ 463 ((struct kvm_mmu_root_info) { .pgd = INVALID_PAGE, .hpa = INVALID_PAGE }) 464 465 #define KVM_MMU_NUM_PREV_ROOTS 3 466 467 #define KVM_MMU_ROOT_CURRENT BIT(0) 468 #define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i) 469 #define KVM_MMU_ROOTS_ALL (BIT(1 + KVM_MMU_NUM_PREV_ROOTS) - 1) 470 471 #define KVM_HAVE_MMU_RWLOCK 472 473 struct kvm_mmu_page; 474 struct kvm_page_fault; 475 476 /* 477 * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit, 478 * and 2-level 32-bit). The kvm_pagewalk structure abstracts the details of the 479 * current mmu mode. 480 */ 481 struct kvm_pagewalk { 482 unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu); 483 u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); 484 void (*inject_page_fault)(struct kvm_vcpu *vcpu, 485 struct x86_exception *fault, 486 bool from_hardware); 487 gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w, 488 gpa_t gva_or_gpa, u64 access, 489 struct x86_exception *exception); 490 491 union kvm_cpu_role cpu_role; 492 struct kvm_page_format fmt; 493 }; 494 495 struct kvm_mmu { 496 int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); 497 int (*sync_spte)(struct kvm_vcpu *vcpu, 498 struct kvm_mmu_page *sp, int i); 499 struct kvm_pagewalk *w; 500 501 struct kvm_mmu_root_info root; 502 hpa_t mirror_root_hpa; 503 union kvm_mmu_page_role root_role; 504 505 struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS]; 506 507 u64 *pae_root; 508 u64 *pml4_root; 509 u64 *pml5_root; 510 511 /* 512 * check zero bits on shadow page table entries, these 513 * bits include not only hardware reserved bits but also 514 * the bits spte never used. 515 */ 516 struct kvm_page_format fmt; 517 }; 518 519 enum pmc_type { 520 KVM_PMC_GP = 0, 521 KVM_PMC_FIXED, 522 }; 523 524 struct kvm_pmc { 525 enum pmc_type type; 526 u8 idx; 527 bool is_paused; 528 bool intr; 529 /* 530 * Base value of the PMC counter, relative to the *consumed* count in 531 * the associated perf_event. This value includes counter updates from 532 * the perf_event and emulated_count since the last time the counter 533 * was reprogrammed, but it is *not* the current value as seen by the 534 * guest or userspace. 535 * 536 * The count is relative to the associated perf_event so that KVM 537 * doesn't need to reprogram the perf_event every time the guest writes 538 * to the counter. 539 */ 540 u64 counter; 541 /* 542 * PMC events triggered by KVM emulation that haven't been fully 543 * processed, i.e. haven't undergone overflow detection. 544 */ 545 u64 emulated_counter; 546 u64 eventsel; 547 u64 eventsel_hw; 548 struct perf_event *perf_event; 549 struct kvm_vcpu *vcpu; 550 /* 551 * only for creating or reusing perf_event, 552 * eventsel value for general purpose counters, 553 * ctrl value for fixed counters. 554 */ 555 u64 current_config; 556 }; 557 558 /* More counters may conflict with other existing Architectural MSRs */ 559 #define KVM_MAX(a, b) ((a) >= (b) ? (a) : (b)) 560 #define KVM_MAX_NR_INTEL_GP_COUNTERS 8 561 #define KVM_MAX_NR_AMD_GP_COUNTERS 6 562 #define KVM_MAX_NR_GP_COUNTERS KVM_MAX(KVM_MAX_NR_INTEL_GP_COUNTERS, \ 563 KVM_MAX_NR_AMD_GP_COUNTERS) 564 565 #define KVM_MAX_NR_INTEL_FIXED_COUNTERS 3 566 #define KVM_MAX_NR_AMD_FIXED_COUNTERS 0 567 #define KVM_MAX_NR_FIXED_COUNTERS KVM_MAX(KVM_MAX_NR_INTEL_FIXED_COUNTERS, \ 568 KVM_MAX_NR_AMD_FIXED_COUNTERS) 569 570 struct kvm_pmu { 571 u8 version; 572 unsigned nr_arch_gp_counters; 573 unsigned nr_arch_fixed_counters; 574 unsigned available_event_types; 575 u64 fixed_ctr_ctrl; 576 u64 fixed_ctr_ctrl_hw; 577 u64 fixed_ctr_ctrl_rsvd; 578 u64 global_ctrl; 579 u64 global_status; 580 u64 counter_bitmask[2]; 581 u64 global_ctrl_rsvd; 582 u64 global_status_rsvd; 583 u64 reserved_bits; 584 u64 raw_event_mask; 585 struct kvm_pmc gp_counters[KVM_MAX_NR_GP_COUNTERS]; 586 struct kvm_pmc fixed_counters[KVM_MAX_NR_FIXED_COUNTERS]; 587 588 /* 589 * Overlay the bitmap with a 64-bit atomic so that all bits can be 590 * set in a single access, e.g. to reprogram all counters when the PMU 591 * filter changes. 592 */ 593 union { 594 DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX); 595 atomic64_t __reprogram_pmi; 596 }; 597 DECLARE_BITMAP(all_valid_pmc_idx, X86_PMC_IDX_MAX); 598 DECLARE_BITMAP(pmc_in_use, X86_PMC_IDX_MAX); 599 600 DECLARE_BITMAP(pmc_counting_instructions, X86_PMC_IDX_MAX); 601 DECLARE_BITMAP(pmc_counting_branches, X86_PMC_IDX_MAX); 602 603 DECLARE_BITMAP(pmc_has_mode_specific_enables, X86_PMC_IDX_MAX); 604 605 u64 ds_area; 606 u64 pebs_enable; 607 u64 pebs_enable_rsvd; 608 u64 pebs_data_cfg; 609 u64 pebs_data_cfg_rsvd; 610 611 /* 612 * The gate to release perf_events not marked in 613 * pmc_in_use only once in a vcpu time slice. 614 */ 615 bool need_cleanup; 616 617 /* 618 * The total number of programmed perf_events and it helps to avoid 619 * redundant check before cleanup if guest don't use vPMU at all. 620 */ 621 u8 event_count; 622 }; 623 624 struct kvm_pmu_ops; 625 626 enum { 627 KVM_DEBUGREG_BP_ENABLED = BIT(0), 628 KVM_DEBUGREG_WONT_EXIT = BIT(1), 629 /* 630 * Guest debug registers (DR0-3, DR6 and DR7) are saved/restored by 631 * hardware on exit from or enter to guest. KVM needn't switch them. 632 * DR0-3, DR6 and DR7 are set to their architectural INIT value on VM 633 * exit, host values need to be restored. 634 */ 635 KVM_DEBUGREG_AUTO_SWITCH = BIT(2), 636 }; 637 638 struct kvm_mtrr { 639 u64 var[KVM_NR_VAR_MTRR * 2]; 640 u64 fixed_64k; 641 u64 fixed_16k[2]; 642 u64 fixed_4k[8]; 643 u64 deftype; 644 }; 645 646 struct kvm_hypervisor_cpuid { 647 u32 base; 648 u32 limit; 649 }; 650 651 #ifdef CONFIG_KVM_XEN 652 /* Xen HVM per vcpu emulation context */ 653 struct kvm_vcpu_xen { 654 u64 hypercall_rip; 655 u32 current_runstate; 656 u8 upcall_vector; 657 struct gfn_to_pfn_cache vcpu_info_cache; 658 struct gfn_to_pfn_cache vcpu_time_info_cache; 659 struct gfn_to_pfn_cache runstate_cache; 660 struct gfn_to_pfn_cache runstate2_cache; 661 u64 last_steal; 662 u64 runstate_entry_time; 663 u64 runstate_times[4]; 664 unsigned long evtchn_pending_sel; 665 u32 vcpu_id; /* The Xen / ACPI vCPU ID */ 666 u32 timer_virq; 667 u64 timer_expires; /* In guest epoch */ 668 atomic_t timer_pending; 669 struct hrtimer timer; 670 int poll_evtchn; 671 struct timer_list poll_timer; 672 struct kvm_hypervisor_cpuid cpuid; 673 }; 674 #endif 675 676 struct kvm_vcpu_hv; 677 678 struct kvm_queued_exception { 679 bool pending; 680 bool injected; 681 bool has_error_code; 682 u8 vector; 683 u32 error_code; 684 unsigned long payload; 685 bool has_payload; 686 }; 687 688 /* 689 * Hardware-defined CPUID leafs that are either scattered by the kernel or are 690 * unknown to the kernel, but need to be directly used by KVM. Note, these 691 * word values conflict with the kernel's "bug" caps, but KVM doesn't use those. 692 */ 693 enum kvm_only_cpuid_leafs { 694 CPUID_12_EAX = NCAPINTS, 695 CPUID_7_1_EDX, 696 CPUID_8000_0007_EDX, 697 CPUID_8000_0022_EAX, 698 CPUID_7_2_EDX, 699 CPUID_24_0_EBX, 700 CPUID_8000_0021_ECX, 701 CPUID_7_1_ECX, 702 CPUID_1E_1_EAX, 703 CPUID_24_1_ECX, 704 NR_KVM_CPU_CAPS, 705 706 NKVMCAPINTS = NR_KVM_CPU_CAPS - NCAPINTS, 707 }; 708 709 struct kvm_vcpu_arch { 710 /* 711 * rip and regs accesses must go through 712 * kvm_{register,rip}_{read,write} functions. 713 */ 714 unsigned long regs[NR_VCPU_GENERAL_PURPOSE_REGS]; 715 unsigned long rip; 716 DECLARE_BITMAP(regs_avail, NR_VCPU_TOTAL_REGS); 717 DECLARE_BITMAP(regs_dirty, NR_VCPU_TOTAL_REGS); 718 719 unsigned long cr0; 720 unsigned long cr0_guest_owned_bits; 721 unsigned long cr2; 722 unsigned long cr3; 723 unsigned long cr4; 724 unsigned long cr4_guest_owned_bits; 725 unsigned long cr4_guest_rsvd_bits; 726 unsigned long cr8; 727 u32 host_pkru; 728 u32 pkru; 729 u32 hflags; 730 u64 efer; 731 u64 host_debugctl; 732 u64 apic_base; 733 struct kvm_lapic *apic; /* kernel irqchip context */ 734 bool load_eoi_exitmap_pending; 735 DECLARE_BITMAP(ioapic_handled_vectors, 256); 736 unsigned long apic_attention; 737 int32_t apic_arb_prio; 738 int mp_state; 739 u64 ia32_misc_enable_msr; 740 u64 smbase; 741 u64 smi_count; 742 bool at_instruction_boundary; 743 bool tpr_access_reporting; 744 bool xfd_no_write_intercept; 745 u64 microcode_version; 746 u64 arch_capabilities; 747 u64 perf_capabilities; 748 749 /* 750 * Paging state of the vcpu 751 * 752 * If the vcpu runs in guest mode with two level paging this still saves 753 * the paging mode of the l1 guest. This context is always used to 754 * handle faults. 755 */ 756 struct kvm_mmu *mmu; 757 758 /* Non-nested MMU for L1 */ 759 struct kvm_mmu root_mmu; 760 761 /* L1 TDP when running nested */ 762 struct kvm_mmu guest_mmu; 763 struct kvm_pagewalk ngpa_walk; 764 765 /* 766 * Pagewalk context used for gva_to_gpa translations. 767 */ 768 struct kvm_pagewalk gva_walk; 769 770 u64 pdptrs[4]; /* pae */ 771 772 struct kvm_mmu_memory_cache mmu_pte_list_desc_cache; 773 struct kvm_mmu_memory_cache mmu_shadow_page_cache; 774 struct kvm_mmu_memory_cache mmu_shadowed_info_cache; 775 struct kvm_mmu_memory_cache mmu_page_header_cache; 776 /* 777 * This cache is to allocate external page table. E.g. private EPT used 778 * by the TDX module. 779 */ 780 struct kvm_mmu_memory_cache mmu_external_spt_cache; 781 782 /* 783 * QEMU userspace and the guest each have their own FPU state. 784 * In vcpu_run, we switch between the user and guest FPU contexts. 785 * While running a VCPU, the VCPU thread will have the guest FPU 786 * context. 787 * 788 * Note that while the PKRU state lives inside the fpu registers, 789 * it is switched out separately at VMENTER and VMEXIT time. The 790 * "guest_fpstate" state here contains the guest FPU context, with the 791 * host PRKU bits. 792 */ 793 struct fpu_guest guest_fpu; 794 795 u64 xcr0; 796 u64 guest_supported_xcr0; 797 u64 ia32_xss; 798 u64 guest_supported_xss; 799 800 struct kvm_pio_request pio; 801 void *pio_data; 802 void *sev_pio_data; 803 unsigned sev_pio_count; 804 805 u8 event_exit_inst_len; 806 807 bool exception_from_userspace; 808 809 /* Exceptions to be injected to the guest. */ 810 struct kvm_queued_exception exception; 811 /* Exception VM-Exits to be synthesized to L1. */ 812 struct kvm_queued_exception exception_vmexit; 813 814 struct kvm_queued_interrupt { 815 bool injected; 816 bool soft; 817 u8 nr; 818 } interrupt; 819 820 int halt_request; /* real mode on Intel only */ 821 822 int cpuid_nent; 823 struct kvm_cpuid_entry2 *cpuid_entries; 824 bool cpuid_dynamic_bits_dirty; 825 bool is_amd_compatible; 826 827 /* 828 * cpu_caps holds the effective guest capabilities, i.e. the features 829 * the vCPU is allowed to use. Typically, but not always, features can 830 * be used by the guest if and only if both KVM and userspace want to 831 * expose the feature to the guest. 832 * 833 * A common exception is for virtualization holes, i.e. when KVM can't 834 * prevent the guest from using a feature, in which case the vCPU "has" 835 * the feature regardless of what KVM or userspace desires. 836 * 837 * Note, features that don't require KVM involvement in any way are 838 * NOT enforced/sanitized by KVM, i.e. are taken verbatim from the 839 * guest CPUID provided by userspace. 840 */ 841 u32 cpu_caps[NR_KVM_CPU_CAPS]; 842 843 u64 reserved_gpa_bits; 844 int maxphyaddr; 845 846 /* emulate context */ 847 848 struct x86_emulate_ctxt *emulate_ctxt; 849 bool emulate_regs_need_sync_to_vcpu; 850 bool emulate_regs_need_sync_from_vcpu; 851 int (*complete_userspace_io)(struct kvm_vcpu *vcpu); 852 unsigned long cui_linear_rip; 853 int cui_rdmsr_imm_reg; 854 855 gpa_t time; 856 s8 pvclock_tsc_shift; 857 u32 pvclock_tsc_mul; 858 unsigned int hw_tsc_khz; 859 struct gfn_to_pfn_cache pv_time; 860 /* set guest stopped flag in pvclock flags field */ 861 bool pvclock_set_guest_stopped_request; 862 863 struct { 864 u8 preempted; 865 u64 msr_val; 866 u64 last_steal; 867 struct gfn_to_hva_cache cache; 868 } st; 869 870 u64 l1_tsc_offset; 871 u64 tsc_offset; /* current tsc offset */ 872 u64 last_guest_tsc; 873 u64 last_host_tsc; 874 u64 tsc_offset_adjustment; 875 u64 this_tsc_nsec; 876 u64 this_tsc_write; 877 u64 this_tsc_generation; 878 bool tsc_catchup; 879 bool tsc_always_catchup; 880 s8 virtual_tsc_shift; 881 u32 virtual_tsc_mult; 882 u32 virtual_tsc_khz; 883 s64 ia32_tsc_adjust_msr; 884 u64 msr_ia32_power_ctl; 885 u64 l1_tsc_scaling_ratio; 886 u64 tsc_scaling_ratio; /* current scaling ratio */ 887 888 atomic_t nmi_queued; /* unprocessed asynchronous NMIs */ 889 /* Number of NMIs pending injection, not including hardware vNMIs. */ 890 unsigned int nmi_pending; 891 bool nmi_injected; /* Trying to inject an NMI this entry */ 892 bool smi_pending; /* SMI queued after currently running handler */ 893 u8 handling_intr_from_guest; 894 895 struct kvm_mtrr mtrr_state; 896 u64 pat; 897 898 unsigned switch_db_regs; 899 unsigned long db[KVM_NR_DB_REGS]; 900 unsigned long dr6; 901 unsigned long dr7; 902 unsigned long eff_db[KVM_NR_DB_REGS]; 903 unsigned long guest_debug_dr7; 904 u64 msr_platform_info; 905 u64 msr_misc_features_enables; 906 907 u64 mcg_cap; 908 u64 mcg_status; 909 u64 mcg_ctl; 910 u64 mcg_ext_ctl; 911 u64 *mce_banks; 912 u64 *mci_ctl2_banks; 913 914 /* Cache MMIO info */ 915 u64 mmio_gva; 916 unsigned mmio_access; 917 gfn_t mmio_gfn; 918 u64 mmio_gen; 919 920 struct kvm_pmu pmu; 921 922 /* used for guest single stepping over the given code position */ 923 unsigned long singlestep_rip; 924 925 #ifdef CONFIG_KVM_HYPERV 926 bool hyperv_enabled; 927 struct kvm_vcpu_hv *hyperv; 928 #endif 929 #ifdef CONFIG_KVM_XEN 930 struct kvm_vcpu_xen xen; 931 #endif 932 cpumask_var_t wbinvd_dirty_mask; 933 934 unsigned long last_retry_eip; 935 unsigned long last_retry_addr; 936 937 struct { 938 bool halted; 939 gfn_t gfns[ASYNC_PF_PER_VCPU]; 940 struct gfn_to_hva_cache data; 941 u64 msr_en_val; /* MSR_KVM_ASYNC_PF_EN */ 942 u64 msr_int_val; /* MSR_KVM_ASYNC_PF_INT */ 943 u16 vec; 944 u32 id; 945 u32 host_apf_flags; 946 bool pageready_pending; 947 } apf; 948 949 /* OSVW MSRs (AMD only) */ 950 struct { 951 u64 length; 952 u64 status; 953 } osvw; 954 955 struct { 956 u64 msr_val; 957 struct gfn_to_hva_cache data; 958 } pv_eoi; 959 960 u64 msr_kvm_poll_control; 961 962 /* pv related host specific info */ 963 struct { 964 bool pv_unhalted; 965 } pv; 966 967 int pending_ioapic_eoi; 968 int pending_external_vector; 969 int highest_stale_pending_ioapic_eoi; 970 971 /* be preempted when it's in kernel-mode(cpl=0) */ 972 bool preempted_in_kernel; 973 974 /* Host CPU on which VM-entry was most recently attempted */ 975 int last_vmentry_cpu; 976 977 /* AMD MSRC001_0015 Hardware Configuration */ 978 u64 msr_hwcr; 979 980 /* pv related cpuid info */ 981 struct { 982 /* 983 * value of the eax register in the KVM_CPUID_FEATURES CPUID 984 * leaf. 985 */ 986 u32 features; 987 988 /* 989 * indicates whether pv emulation should be disabled if features 990 * are not present in the guest's cpuid 991 */ 992 bool enforce; 993 } pv_cpuid; 994 995 /* Protected Guests */ 996 bool guest_state_protected; 997 bool guest_tsc_protected; 998 999 /* 1000 * Set when PDPTS were loaded directly by the userspace without 1001 * reading the guest memory 1002 */ 1003 bool pdptrs_from_userspace; 1004 1005 /* 1006 * Set if an emulated nested VM-Enter to L2 is pending completion. KVM 1007 * must not synthesize a VM-Exit to L1 before entering L2, as VM-Exits 1008 * can only occur at instruction boundaries. The only exception is 1009 * VMX's "notify" exits, which exist in large part to break the CPU out 1010 * of infinite ucode loops, but can corrupt vCPU state in the process! 1011 * 1012 * For all intents and purposes, this is a boolean, but it's tracked as 1013 * a u8 so that KVM can detect when userspace may have stuffed vCPU 1014 * state and generated an architecturally-impossible VM-Exit. 1015 */ 1016 #define KVM_NESTED_RUN_PENDING 1 1017 #define KVM_NESTED_RUN_PENDING_UNTRUSTED 2 1018 u8 nested_run_pending; 1019 1020 #if IS_ENABLED(CONFIG_HYPERV) 1021 hpa_t hv_root_tdp; 1022 #endif 1023 }; 1024 1025 struct kvm_lpage_info { 1026 int disallow_lpage; 1027 }; 1028 1029 struct kvm_arch_memory_slot { 1030 struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES]; 1031 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; 1032 unsigned short *gfn_write_track; 1033 }; 1034 1035 /* Hyper-V synthetic debugger (SynDbg)*/ 1036 struct kvm_hv_syndbg { 1037 struct { 1038 u64 control; 1039 u64 status; 1040 u64 send_page; 1041 u64 recv_page; 1042 u64 pending_page; 1043 } control; 1044 u64 options; 1045 }; 1046 1047 /* Current state of Hyper-V TSC page clocksource */ 1048 enum hv_tsc_page_status { 1049 /* TSC page was not set up or disabled */ 1050 HV_TSC_PAGE_UNSET = 0, 1051 /* TSC page MSR was written by the guest, update pending */ 1052 HV_TSC_PAGE_GUEST_CHANGED, 1053 /* TSC page update was triggered from the host side */ 1054 HV_TSC_PAGE_HOST_CHANGED, 1055 /* TSC page was properly set up and is currently active */ 1056 HV_TSC_PAGE_SET, 1057 /* TSC page was set up with an inaccessible GPA */ 1058 HV_TSC_PAGE_BROKEN, 1059 }; 1060 1061 #ifdef CONFIG_KVM_HYPERV 1062 /* Hyper-V emulation context */ 1063 struct kvm_hv { 1064 struct mutex hv_lock; 1065 u64 hv_guest_os_id; 1066 u64 hv_hypercall; 1067 u64 hv_tsc_page; 1068 enum hv_tsc_page_status hv_tsc_page_status; 1069 1070 /* Hyper-v based guest crash (NT kernel bugcheck) parameters */ 1071 u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS]; 1072 u64 hv_crash_ctl; 1073 1074 struct ms_hyperv_tsc_page tsc_ref; 1075 1076 struct idr conn_to_evt; 1077 1078 u64 hv_reenlightenment_control; 1079 u64 hv_tsc_emulation_control; 1080 u64 hv_tsc_emulation_status; 1081 u64 hv_invtsc_control; 1082 1083 /* How many vCPUs have VP index != vCPU index */ 1084 atomic_t num_mismatched_vp_indexes; 1085 1086 /* 1087 * How many SynICs use 'AutoEOI' feature 1088 * (protected by arch.apicv_update_lock) 1089 */ 1090 unsigned int synic_auto_eoi_used; 1091 1092 struct kvm_hv_syndbg hv_syndbg; 1093 1094 bool xsaves_xsavec_checked; 1095 }; 1096 #endif 1097 1098 #ifdef CONFIG_KVM_XEN 1099 /* Xen emulation context */ 1100 struct kvm_xen { 1101 struct mutex xen_lock; 1102 u32 xen_version; 1103 bool long_mode; 1104 bool runstate_update_flag; 1105 u8 upcall_vector; 1106 struct gfn_to_pfn_cache shinfo_cache; 1107 struct idr evtchn_ports; 1108 unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 1109 1110 struct kvm_xen_hvm_config hvm_config; 1111 }; 1112 #endif 1113 1114 enum kvm_irqchip_mode { 1115 KVM_IRQCHIP_NONE, 1116 #ifdef CONFIG_KVM_IOAPIC 1117 KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */ 1118 #endif 1119 KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */ 1120 }; 1121 1122 enum kvm_suppress_eoi_broadcast_mode { 1123 KVM_SUPPRESS_EOI_BROADCAST_QUIRKED, /* Legacy behavior */ 1124 KVM_SUPPRESS_EOI_BROADCAST_ENABLED, /* Enable Suppress EOI broadcast */ 1125 KVM_SUPPRESS_EOI_BROADCAST_DISABLED /* Disable Suppress EOI broadcast */ 1126 }; 1127 1128 struct kvm_possible_nx_huge_pages { 1129 /* 1130 * A list of kvm_mmu_page structs that, if zapped, could possibly be 1131 * replaced by an NX huge page. A shadow page is on this list if its 1132 * existence disallows an NX huge page (nx_huge_page_disallowed is set) 1133 * and there are no other conditions that prevent a huge page, e.g. 1134 * the backing host page is huge, dirtly logging is not enabled for its 1135 * memslot, etc... Note, zapping shadow pages on this list doesn't 1136 * guarantee an NX huge page will be created in its stead, e.g. if the 1137 * guest attempts to execute from the region then KVM obviously can't 1138 * create an NX huge page (without hanging the guest). 1139 */ 1140 struct list_head pages; 1141 u64 nr_pages; 1142 }; 1143 1144 enum kvm_mmu_type { 1145 KVM_SHADOW_MMU, 1146 #ifdef CONFIG_X86_64 1147 KVM_TDP_MMU, 1148 #endif 1149 KVM_NR_MMU_TYPES, 1150 }; 1151 1152 struct kvm_arch { 1153 unsigned long n_requested_mmu_pages; 1154 unsigned long n_max_mmu_pages; 1155 unsigned int indirect_shadow_pages; 1156 u8 mmu_valid_gen; 1157 u8 vm_type; 1158 bool has_private_mem; 1159 bool has_protected_state; 1160 bool has_protected_eoi; 1161 bool has_protected_pmu; 1162 bool pre_fault_allowed; 1163 struct hlist_head *mmu_page_hash; 1164 struct list_head active_mmu_pages; 1165 struct kvm_possible_nx_huge_pages possible_nx_huge_pages[KVM_NR_MMU_TYPES]; 1166 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING 1167 struct kvm_page_track_notifier_head track_notifier_head; 1168 #endif 1169 /* 1170 * Protects marking pages unsync during page faults, as TDP MMU page 1171 * faults only take mmu_lock for read. For simplicity, the unsync 1172 * pages lock is always taken when marking pages unsync regardless of 1173 * whether mmu_lock is held for read or write. 1174 */ 1175 spinlock_t mmu_unsync_pages_lock; 1176 1177 u64 shadow_mmio_value; 1178 1179 #define __KVM_HAVE_ARCH_NONCOHERENT_DMA 1180 atomic_t noncoherent_dma_count; 1181 unsigned long nr_possible_bypass_irqs; 1182 1183 #ifdef CONFIG_KVM_IOAPIC 1184 struct kvm_pic *vpic; 1185 struct kvm_ioapic *vioapic; 1186 struct kvm_pit *vpit; 1187 #endif 1188 atomic_t vapics_in_nmi_mode; 1189 1190 struct mutex apic_map_lock; 1191 struct kvm_apic_map __rcu *apic_map; 1192 atomic_t apic_map_dirty; 1193 1194 bool apic_access_memslot_enabled; 1195 bool apic_access_memslot_inhibited; 1196 1197 /* 1198 * Force apicv_update_lock and apicv_nr_irq_window_req to reside in a 1199 * dedicated cacheline. They are write-mostly, whereas most everything 1200 * else in kvm_arch is read-mostly. Note that apicv_inhibit_reasons is 1201 * read-mostly: toggling VM-wide inhibits is rare; _checking_ for 1202 * inhibits is common. 1203 */ 1204 ____cacheline_aligned 1205 /* 1206 * Protects apicv_inhibit_reasons and apicv_nr_irq_window_req (with an 1207 * asterisk, see kvm_inc_or_dec_irq_window_inhibit() for details). 1208 */ 1209 struct rw_semaphore apicv_update_lock; 1210 atomic_t apicv_nr_irq_window_req; 1211 ____cacheline_aligned 1212 1213 unsigned long apicv_inhibit_reasons; 1214 1215 gpa_t wall_clock; 1216 1217 u64 disabled_exits; 1218 1219 s64 kvmclock_offset; 1220 1221 /* 1222 * This also protects nr_vcpus_matched_tsc which is read from a 1223 * preemption-disabled region, so it must be a raw spinlock. 1224 */ 1225 raw_spinlock_t tsc_write_lock; 1226 u64 last_tsc_nsec; 1227 u64 last_tsc_write; 1228 u32 last_tsc_khz; 1229 u64 last_tsc_offset; 1230 u64 cur_tsc_nsec; 1231 u64 cur_tsc_write; 1232 u64 cur_tsc_offset; 1233 u64 cur_tsc_generation; 1234 int nr_vcpus_matched_tsc; 1235 1236 u32 default_tsc_khz; 1237 bool user_set_tsc; 1238 u64 apic_bus_cycle_ns; 1239 1240 seqcount_raw_spinlock_t pvclock_sc; 1241 bool use_master_clock; 1242 u64 master_kernel_ns; 1243 u64 master_cycle_now; 1244 struct ratelimit_state kvmclock_update_rs; 1245 1246 #ifdef CONFIG_KVM_HYPERV 1247 struct kvm_hv hyperv; 1248 #endif 1249 1250 #ifdef CONFIG_KVM_XEN 1251 struct kvm_xen xen; 1252 #endif 1253 1254 bool backwards_tsc_observed; 1255 bool boot_vcpu_runs_old_kvmclock; 1256 u32 bsp_vcpu_id; 1257 1258 u64 disabled_quirks; 1259 1260 enum kvm_irqchip_mode irqchip_mode; 1261 u8 nr_reserved_ioapic_pins; 1262 1263 bool disabled_lapic_found; 1264 1265 bool x2apic_format; 1266 bool x2apic_broadcast_quirk_disabled; 1267 enum kvm_suppress_eoi_broadcast_mode suppress_eoi_broadcast_mode; 1268 1269 bool has_mapped_host_mmio; 1270 bool guest_can_read_msr_platform_info; 1271 bool exception_payload_enabled; 1272 1273 bool triple_fault_event; 1274 1275 bool bus_lock_detection_enabled; 1276 bool enable_pmu; 1277 bool created_mediated_pmu; 1278 1279 u32 notify_window; 1280 u32 notify_vmexit_flags; 1281 /* 1282 * If exit_on_emulation_error is set, and the in-kernel instruction 1283 * emulator fails to emulate an instruction, allow userspace 1284 * the opportunity to look at it. 1285 */ 1286 bool exit_on_emulation_error; 1287 1288 /* Deflect RDMSR and WRMSR to user space when they trigger a #GP */ 1289 u32 user_space_msr_mask; 1290 struct kvm_x86_msr_filter __rcu *msr_filter; 1291 1292 u32 hypercall_exit_enabled; 1293 1294 /* Guest can access the SGX PROVISIONKEY. */ 1295 bool sgx_provisioning_allowed; 1296 1297 struct kvm_x86_pmu_event_filter __rcu *pmu_event_filter; 1298 struct vhost_task *nx_huge_page_recovery_thread; 1299 u64 nx_huge_page_last; 1300 struct once nx_once; 1301 1302 #ifdef CONFIG_X86_64 1303 #ifdef CONFIG_KVM_PROVE_MMU 1304 /* 1305 * The number of TDP MMU pages across all roots. Used only to sanity 1306 * check that KVM isn't leaking TDP MMU pages. 1307 */ 1308 atomic64_t tdp_mmu_pages; 1309 #endif 1310 1311 /* 1312 * List of struct kvm_mmu_pages being used as roots. 1313 * All struct kvm_mmu_pages in the list should have 1314 * tdp_mmu_page set. 1315 * 1316 * For reads, this list is protected by: 1317 * RCU alone or 1318 * the MMU lock in read mode + RCU or 1319 * the MMU lock in write mode 1320 * 1321 * For writes, this list is protected by tdp_mmu_pages_lock; see 1322 * below for the details. 1323 * 1324 * Roots will remain in the list until their tdp_mmu_root_count 1325 * drops to zero, at which point the thread that decremented the 1326 * count to zero should removed the root from the list and clean 1327 * it up, freeing the root after an RCU grace period. 1328 */ 1329 struct list_head tdp_mmu_roots; 1330 1331 /* 1332 * Protects accesses to the following fields when the MMU lock 1333 * is held in read mode: 1334 * - tdp_mmu_roots (above) 1335 * - the link field of kvm_mmu_page structs used by the TDP MMU 1336 * - possible_nx_huge_pages[KVM_TDP_MMU]; 1337 * - the possible_nx_huge_page_link field of kvm_mmu_page structs used 1338 * by the TDP MMU 1339 * Because the lock is only taken within the MMU lock, strictly 1340 * speaking it is redundant to acquire this lock when the thread 1341 * holds the MMU lock in write mode. However it often simplifies 1342 * the code to do so. 1343 */ 1344 spinlock_t tdp_mmu_pages_lock; 1345 #endif /* CONFIG_X86_64 */ 1346 1347 /* 1348 * If set, at least one shadow root has been allocated. This flag 1349 * is used as one input when determining whether certain memslot 1350 * related allocations are necessary. 1351 */ 1352 bool shadow_root_allocated; 1353 1354 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING 1355 /* 1356 * If set, the VM has (or had) an external write tracking user, and 1357 * thus all write tracking metadata has been allocated, even if KVM 1358 * itself isn't using write tracking. 1359 */ 1360 bool external_write_tracking_enabled; 1361 #endif 1362 1363 #if IS_ENABLED(CONFIG_HYPERV) 1364 hpa_t hv_root_tdp; 1365 spinlock_t hv_root_tdp_lock; 1366 struct hv_partition_assist_pg *hv_pa_pg; 1367 #endif 1368 /* 1369 * VM-scope maximum vCPU ID. Used to determine the size of structures 1370 * that increase along with the maximum vCPU ID, in which case, using 1371 * the global KVM_MAX_VCPU_IDS may lead to significant memory waste. 1372 */ 1373 u32 max_vcpu_ids; 1374 1375 bool disable_nx_huge_pages; 1376 1377 /* 1378 * Memory caches used to allocate shadow pages when performing eager 1379 * page splitting. No need for a shadowed_info_cache since eager page 1380 * splitting only allocates direct shadow pages. 1381 * 1382 * Protected by kvm->slots_lock. 1383 */ 1384 struct kvm_mmu_memory_cache split_shadow_page_cache; 1385 struct kvm_mmu_memory_cache split_page_header_cache; 1386 1387 /* 1388 * Memory cache used to allocate pte_list_desc structs while splitting 1389 * huge pages. In the worst case, to split one huge page, 512 1390 * pte_list_desc structs are needed to add each lower level leaf sptep 1391 * to the rmap plus 1 to extend the parent_ptes rmap of the lower level 1392 * page table. 1393 * 1394 * Protected by kvm->slots_lock. 1395 */ 1396 #define SPLIT_DESC_CACHE_MIN_NR_OBJECTS (SPTE_ENT_PER_PAGE + 1) 1397 struct kvm_mmu_memory_cache split_desc_cache; 1398 1399 gfn_t gfn_direct_bits; 1400 1401 /* 1402 * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero 1403 * value indicates CPU dirty logging is unsupported or disabled in 1404 * current VM. 1405 */ 1406 int cpu_dirty_log_size; 1407 }; 1408 1409 struct kvm_vm_stat { 1410 struct kvm_vm_stat_generic generic; 1411 u64 mmu_shadow_zapped; 1412 u64 mmu_pte_write; 1413 u64 mmu_pde_zapped; 1414 u64 mmu_flooded; 1415 u64 mmu_recycled; 1416 u64 mmu_cache_miss; 1417 u64 mmu_unsync; 1418 u64 mmu_shadow_pages; 1419 union { 1420 struct { 1421 atomic64_t pages_4k; 1422 atomic64_t pages_2m; 1423 atomic64_t pages_1g; 1424 }; 1425 atomic64_t pages[KVM_NR_PAGE_SIZES]; 1426 }; 1427 u64 nx_lpage_splits; 1428 u64 max_mmu_page_hash_collisions; 1429 u64 max_mmu_rmap_size; 1430 }; 1431 1432 struct kvm_vcpu_stat { 1433 struct kvm_vcpu_stat_generic generic; 1434 u64 pf_taken; 1435 u64 pf_fixed; 1436 u64 pf_emulate; 1437 u64 pf_spurious; 1438 u64 pf_fast; 1439 u64 pf_mmio_spte_created; 1440 u64 pf_guest; 1441 u64 tlb_flush; 1442 u64 invlpg; 1443 1444 u64 exits; 1445 u64 io_exits; 1446 u64 mmio_exits; 1447 u64 signal_exits; 1448 u64 irq_window_exits; 1449 u64 nmi_window_exits; 1450 u64 l1d_flush; 1451 u64 halt_exits; 1452 u64 request_irq_exits; 1453 u64 irq_exits; 1454 u64 host_state_reload; 1455 u64 fpu_reload; 1456 u64 insn_emulation; 1457 u64 insn_emulation_fail; 1458 u64 hypercalls; 1459 u64 irq_injections; 1460 u64 nmi_injections; 1461 u64 req_event; 1462 u64 nested_run; 1463 u64 directed_yield_attempted; 1464 u64 directed_yield_successful; 1465 u64 preemption_reported; 1466 u64 preemption_other; 1467 u64 guest_mode; 1468 u64 notify_window_exits; 1469 }; 1470 1471 struct x86_instruction_info; 1472 1473 struct msr_data { 1474 bool host_initiated; 1475 u32 index; 1476 u64 data; 1477 }; 1478 1479 struct kvm_lapic_irq { 1480 u32 vector; 1481 u16 delivery_mode; 1482 u16 dest_mode; 1483 bool level; 1484 u16 trig_mode; 1485 u32 shorthand; 1486 u32 dest_id; 1487 bool msi_redir_hint; 1488 }; 1489 1490 enum kvm_x86_run_flags { 1491 KVM_RUN_FORCE_IMMEDIATE_EXIT = BIT(0), 1492 KVM_RUN_LOAD_GUEST_DR6 = BIT(1), 1493 KVM_RUN_LOAD_DEBUGCTL = BIT(2), 1494 }; 1495 1496 struct kvm_x86_ops { 1497 const char *name; 1498 1499 int (*check_processor_compatibility)(void); 1500 1501 int (*enable_virtualization_cpu)(void); 1502 void (*disable_virtualization_cpu)(void); 1503 cpu_emergency_virt_cb *emergency_disable_virtualization_cpu; 1504 1505 void (*hardware_unsetup)(void); 1506 bool (*has_emulated_msr)(struct kvm *kvm, u32 index); 1507 void (*vcpu_after_set_cpuid)(struct kvm_vcpu *vcpu); 1508 1509 unsigned int vm_size; 1510 int (*vm_init)(struct kvm *kvm); 1511 void (*vm_destroy)(struct kvm *kvm); 1512 void (*vm_pre_destroy)(struct kvm *kvm); 1513 1514 /* Create, but do not attach this VCPU */ 1515 int (*vcpu_precreate)(struct kvm *kvm); 1516 int (*vcpu_create)(struct kvm_vcpu *vcpu); 1517 void (*vcpu_free)(struct kvm_vcpu *vcpu); 1518 void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event); 1519 1520 void (*prepare_switch_to_guest)(struct kvm_vcpu *vcpu); 1521 void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); 1522 void (*vcpu_put)(struct kvm_vcpu *vcpu); 1523 1524 /* 1525 * Mask of DEBUGCTL bits that are owned by the host, i.e. that need to 1526 * match the host's value even while the guest is active. 1527 */ 1528 const u64 HOST_OWNED_DEBUGCTL; 1529 1530 void (*update_exception_bitmap)(struct kvm_vcpu *vcpu); 1531 int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); 1532 int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); 1533 u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); 1534 void (*get_segment)(struct kvm_vcpu *vcpu, 1535 struct kvm_segment *var, int seg); 1536 int (*get_cpl)(struct kvm_vcpu *vcpu); 1537 int (*get_cpl_no_cache)(struct kvm_vcpu *vcpu); 1538 void (*set_segment)(struct kvm_vcpu *vcpu, 1539 struct kvm_segment *var, int seg); 1540 void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); 1541 bool (*is_valid_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); 1542 void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); 1543 void (*post_set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); 1544 bool (*is_valid_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); 1545 void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); 1546 int (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); 1547 void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1548 void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1549 void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1550 void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1551 void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu); 1552 void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value); 1553 void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg); 1554 unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); 1555 void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); 1556 bool (*get_if_flag)(struct kvm_vcpu *vcpu); 1557 1558 void (*flush_tlb_all)(struct kvm_vcpu *vcpu); 1559 void (*flush_tlb_current)(struct kvm_vcpu *vcpu); 1560 #if IS_ENABLED(CONFIG_HYPERV) 1561 int (*flush_remote_tlbs)(struct kvm *kvm); 1562 int (*flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn, 1563 gfn_t nr_pages); 1564 #endif 1565 1566 /* 1567 * Flush any TLB entries associated with the given GVA. 1568 * Does not need to flush GPA->HPA mappings. 1569 * Can potentially get non-canonical addresses through INVLPGs, which 1570 * the implementation may choose to ignore if appropriate. 1571 */ 1572 void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full); 1573 1574 /* 1575 * Flush any TLB entries created by the guest. Like tlb_flush_gva(), 1576 * does not need to flush GPA->HPA mappings. 1577 */ 1578 void (*flush_tlb_guest)(struct kvm_vcpu *vcpu); 1579 1580 bool (*vcpu_needs_initialization)(struct kvm_vcpu *vcpu); 1581 enum exit_fastpath_completion (*vcpu_run)(struct kvm_vcpu *vcpu, 1582 u64 run_flags); 1583 int (*handle_exit)(struct kvm_vcpu *vcpu, 1584 enum exit_fastpath_completion exit_fastpath); 1585 int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); 1586 void (*update_emulated_instruction)(struct kvm_vcpu *vcpu); 1587 bool (*unhandleable_emulation_required)(struct kvm_vcpu *vcpu); 1588 1589 void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask); 1590 u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu); 1591 void (*patch_hypercall)(struct kvm_vcpu *vcpu, 1592 unsigned char *hypercall_addr); 1593 void (*inject_irq)(struct kvm_vcpu *vcpu, bool reinjected); 1594 void (*inject_nmi)(struct kvm_vcpu *vcpu); 1595 void (*inject_exception)(struct kvm_vcpu *vcpu); 1596 void (*cancel_injection)(struct kvm_vcpu *vcpu); 1597 int (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection); 1598 int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); 1599 bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); 1600 void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); 1601 /* Whether or not a virtual NMI is pending in hardware. */ 1602 bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu); 1603 /* 1604 * Attempt to pend a virtual NMI in hardware. Returns %true on success 1605 * to allow using static_call_ret0 as the fallback. 1606 */ 1607 bool (*set_vnmi_pending)(struct kvm_vcpu *vcpu); 1608 void (*enable_nmi_window)(struct kvm_vcpu *vcpu); 1609 void (*enable_irq_window)(struct kvm_vcpu *vcpu); 1610 void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr); 1611 1612 const bool x2apic_icr_is_split; 1613 const unsigned long required_apicv_inhibits; 1614 bool allow_apicv_in_x2apic_without_x2apic_virtualization; 1615 void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu); 1616 void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr); 1617 void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); 1618 void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); 1619 void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu); 1620 void (*deliver_interrupt)(struct kvm_lapic *apic, int delivery_mode, 1621 int trig_mode, int vector); 1622 int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu); 1623 int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); 1624 int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr); 1625 u8 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio); 1626 bool (*tdp_has_smep)(struct kvm *kvm); 1627 1628 void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, hpa_t root_hpa, 1629 int root_level); 1630 1631 /* Update the external page table from spte getting set. */ 1632 int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, u64 old_spte, 1633 u64 new_spte, enum pg_level level); 1634 1635 /* Update external page tables for page table about to be freed. */ 1636 void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp); 1637 1638 int (*topup_external_cache)(struct kvm_vcpu *vcpu, int min_nr_spts); 1639 1640 bool (*has_wbinvd_exit)(void); 1641 1642 u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu); 1643 u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu); 1644 void (*write_tsc_offset)(struct kvm_vcpu *vcpu); 1645 void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu); 1646 1647 /* 1648 * Retrieve somewhat arbitrary exit/entry information. Intended to 1649 * be used only from within tracepoints or error paths. 1650 */ 1651 void (*get_exit_info)(struct kvm_vcpu *vcpu, u32 *reason, 1652 u64 *info1, u64 *info2, 1653 u32 *intr_info, u32 *error_code); 1654 1655 void (*get_entry_info)(struct kvm_vcpu *vcpu, 1656 u32 *intr_info, u32 *error_code); 1657 1658 int (*check_intercept)(struct kvm_vcpu *vcpu, 1659 struct x86_instruction_info *info, 1660 enum x86_intercept_stage stage, 1661 struct x86_exception *exception); 1662 void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); 1663 1664 void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); 1665 1666 void (*vcpu_blocking)(struct kvm_vcpu *vcpu); 1667 void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); 1668 1669 int (*pi_update_irte)(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, 1670 unsigned int host_irq, uint32_t guest_irq, 1671 struct kvm_vcpu *vcpu, u32 vector); 1672 void (*pi_start_bypass)(struct kvm *kvm); 1673 void (*apicv_pre_state_restore)(struct kvm_vcpu *vcpu); 1674 void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu); 1675 bool (*dy_apicv_has_pending_interrupt)(struct kvm_vcpu *vcpu); 1676 bool (*protected_apic_has_interrupt)(struct kvm_vcpu *vcpu); 1677 1678 int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 1679 bool *expired); 1680 void (*cancel_hv_timer)(struct kvm_vcpu *vcpu); 1681 1682 void (*setup_mce)(struct kvm_vcpu *vcpu); 1683 1684 #ifdef CONFIG_KVM_SMM 1685 int (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); 1686 int (*enter_smm)(struct kvm_vcpu *vcpu, union kvm_smram *smram); 1687 int (*leave_smm)(struct kvm_vcpu *vcpu, const union kvm_smram *smram); 1688 void (*enable_smi_window)(struct kvm_vcpu *vcpu); 1689 #endif 1690 1691 int (*dev_get_attr)(u32 group, u64 attr, u64 *val); 1692 int (*mem_enc_ioctl)(struct kvm *kvm, void __user *argp); 1693 int (*vcpu_mem_enc_ioctl)(struct kvm_vcpu *vcpu, void __user *argp); 1694 int (*vcpu_mem_enc_unlocked_ioctl)(struct kvm_vcpu *vcpu, void __user *argp); 1695 int (*mem_enc_register_region)(struct kvm *kvm, struct kvm_enc_region *argp); 1696 int (*mem_enc_unregister_region)(struct kvm *kvm, struct kvm_enc_region *argp); 1697 int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); 1698 int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); 1699 void (*guest_memory_reclaimed)(struct kvm *kvm); 1700 void (*reload_vmsa)(struct kvm_vcpu *vcpu); 1701 1702 int (*get_feature_msr)(u32 msr, u64 *data); 1703 1704 int (*check_emulate_instruction)(struct kvm_vcpu *vcpu, int emul_type, 1705 void *insn, int insn_len); 1706 1707 bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu); 1708 int (*enable_l2_tlb_flush)(struct kvm_vcpu *vcpu); 1709 1710 void (*migrate_timers)(struct kvm_vcpu *vcpu); 1711 void (*recalc_intercepts)(struct kvm_vcpu *vcpu); 1712 int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err); 1713 1714 void (*vcpu_deliver_sipi_vector)(struct kvm_vcpu *vcpu, u8 vector); 1715 1716 /* 1717 * Returns vCPU specific APICv inhibit reasons 1718 */ 1719 unsigned long (*vcpu_get_apicv_inhibit_reasons)(struct kvm_vcpu *vcpu); 1720 1721 gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); 1722 void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); 1723 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT 1724 int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, 1725 kvm_pfn_t nr_pages); 1726 #endif 1727 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM 1728 void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); 1729 #endif 1730 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE 1731 void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); 1732 #endif 1733 int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); 1734 }; 1735 1736 struct kvm_x86_nested_ops { 1737 bool enabled; 1738 1739 void (*leave_nested)(struct kvm_vcpu *vcpu); 1740 bool (*is_exception_vmexit)(struct kvm_vcpu *vcpu, u8 vector, 1741 u32 error_code); 1742 int (*check_events)(struct kvm_vcpu *vcpu); 1743 bool (*has_events)(struct kvm_vcpu *vcpu, bool for_injection); 1744 void (*triple_fault)(struct kvm_vcpu *vcpu); 1745 int (*get_state)(struct kvm_vcpu *vcpu, 1746 struct kvm_nested_state __user *user_kvm_nested_state, 1747 unsigned user_data_size); 1748 int (*set_state)(struct kvm_vcpu *vcpu, 1749 struct kvm_nested_state __user *user_kvm_nested_state, 1750 struct kvm_nested_state *kvm_state); 1751 bool (*get_nested_state_pages)(struct kvm_vcpu *vcpu); 1752 int (*write_log_dirty)(struct kvm_vcpu *vcpu, gpa_t l2_gpa); 1753 gpa_t (*translate_nested_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, 1754 u64 access, 1755 struct x86_exception *exception, 1756 u64 pte_access); 1757 1758 int (*enable_evmcs)(struct kvm_vcpu *vcpu, 1759 uint16_t *vmcs_version); 1760 uint16_t (*get_evmcs_version)(struct kvm_vcpu *vcpu); 1761 void (*hv_inject_synthetic_vmexit_post_tlb_flush)(struct kvm_vcpu *vcpu); 1762 }; 1763 1764 struct kvm_x86_init_ops { 1765 int (*hardware_setup)(void); 1766 unsigned int (*handle_intel_pt_intr)(void); 1767 1768 struct kvm_x86_ops *runtime_ops; 1769 struct kvm_pmu_ops *pmu_ops; 1770 struct kvm_x86_nested_ops *nested_ops; 1771 }; 1772 1773 struct kvm_arch_async_pf { 1774 u32 token; 1775 gfn_t gfn; 1776 unsigned long cr3; 1777 bool direct_map; 1778 u64 error_code; 1779 }; 1780 1781 extern bool __read_mostly allow_smaller_maxphyaddr; 1782 extern bool __read_mostly enable_apicv; 1783 extern bool __read_mostly enable_ipiv; 1784 extern bool __read_mostly enable_device_posted_irqs; 1785 extern struct kvm_x86_ops kvm_x86_ops; 1786 extern struct kvm_x86_nested_ops kvm_nested_ops __read_mostly; 1787 1788 #define kvm_x86_call(func) static_call(kvm_x86_##func) 1789 1790 #define KVM_X86_OP(func) \ 1791 DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func)); 1792 #define KVM_X86_OP_OPTIONAL KVM_X86_OP 1793 #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP 1794 #include <asm/kvm-x86-ops.h> 1795 1796 #define kvm_nested_call(func) static_call(kvm_x86_nested_##func) 1797 1798 #define KVM_X86_NESTED_OP(func) \ 1799 DECLARE_STATIC_CALL(kvm_x86_nested_##func, *(((struct kvm_x86_nested_ops *)0)->func)); 1800 #define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP 1801 #define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP 1802 #include <asm/kvm-x86-nested-ops.h> 1803 1804 #define __KVM_HAVE_ARCH_VM_ALLOC 1805 static inline struct kvm *kvm_arch_alloc_vm(void) 1806 { 1807 return kvzalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT); 1808 } 1809 1810 #define __KVM_HAVE_ARCH_VM_FREE 1811 void kvm_arch_free_vm(struct kvm *kvm); 1812 1813 #if IS_ENABLED(CONFIG_HYPERV) 1814 #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS 1815 static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm) 1816 { 1817 if (kvm_x86_ops.flush_remote_tlbs && 1818 !kvm_x86_call(flush_remote_tlbs)(kvm)) 1819 return 0; 1820 else 1821 return -ENOTSUPP; 1822 } 1823 1824 #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE 1825 static inline int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, 1826 u64 nr_pages) 1827 { 1828 if (!kvm_x86_ops.flush_remote_tlbs_range) 1829 return -EOPNOTSUPP; 1830 1831 return kvm_x86_call(flush_remote_tlbs_range)(kvm, gfn, nr_pages); 1832 } 1833 #endif /* CONFIG_HYPERV */ 1834 1835 enum kvm_intr_type { 1836 /* Values are arbitrary, but must be non-zero. */ 1837 KVM_HANDLING_IRQ = 1, 1838 KVM_HANDLING_NMI, 1839 }; 1840 1841 /* Enable perf NMI and timer modes to work, and minimise false positives. */ 1842 #define kvm_arch_pmi_in_guest(vcpu) \ 1843 ((vcpu) && (vcpu)->arch.handling_intr_from_guest && \ 1844 (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI))) 1845 1846 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 1847 #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) 1848 #endif 1849 1850 #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state) 1851 1852 #ifdef CONFIG_X86_64 1853 static inline unsigned long read_msr(unsigned long msr) 1854 { 1855 u64 value; 1856 1857 rdmsrq(msr, value); 1858 return value; 1859 } 1860 #endif 1861 1862 #define HF_GUEST_MASK (1 << 0) /* VCPU is in guest-mode */ 1863 1864 #ifdef CONFIG_KVM_SMM 1865 #define HF_SMM_MASK (1 << 1) 1866 #define HF_SMM_INSIDE_NMI_MASK (1 << 2) 1867 1868 # define KVM_MAX_NR_ADDRESS_SPACES 2 1869 /* SMM is currently unsupported for guests with private memory. */ 1870 # define kvm_arch_nr_memslot_as_ids(kvm) (kvm_arch_has_private_mem(kvm) ? 1 : 2) 1871 # define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0) 1872 # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm) 1873 #else 1874 # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0) 1875 #endif 1876 1877 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 1878 struct kvm_async_pf *work); 1879 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 1880 struct kvm_async_pf *work); 1881 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, 1882 struct kvm_async_pf *work); 1883 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu); 1884 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); 1885 1886 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 1887 { 1888 kvm_x86_call(vcpu_blocking)(vcpu); 1889 } 1890 1891 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 1892 { 1893 kvm_x86_call(vcpu_unblocking)(vcpu); 1894 } 1895 1896 static inline bool kvm_arch_has_irq_bypass(void) 1897 { 1898 return enable_device_posted_irqs; 1899 } 1900 1901 #endif /* _ASM_X86_KVM_HOST_H */ 1902