1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This header defines architecture specific interfaces, x86 version 6 */ 7 8 #ifndef _ASM_X86_KVM_HOST_H 9 #define _ASM_X86_KVM_HOST_H 10 11 #include <linux/types.h> 12 #include <linux/mm.h> 13 #include <linux/mmu_notifier.h> 14 #include <linux/tracepoint.h> 15 #include <linux/cpumask.h> 16 #include <linux/irq_work.h> 17 #include <linux/irq.h> 18 #include <linux/workqueue.h> 19 20 #include <linux/kvm.h> 21 #include <linux/kvm_para.h> 22 #include <linux/kvm_types.h> 23 #include <linux/perf_event.h> 24 #include <linux/pvclock_gtod.h> 25 #include <linux/clocksource.h> 26 #include <linux/irqbypass.h> 27 #include <linux/kfifo.h> 28 #include <linux/sched/vhost_task.h> 29 #include <linux/call_once.h> 30 #include <linux/atomic.h> 31 32 #include <asm/apic.h> 33 #include <asm/pvclock-abi.h> 34 #include <asm/debugreg.h> 35 #include <asm/desc.h> 36 #include <asm/mtrr.h> 37 #include <asm/msr-index.h> 38 #include <asm/msr.h> 39 #include <asm/asm.h> 40 #include <asm/irq_remapping.h> 41 #include <asm/kvm_page_track.h> 42 #include <asm/kvm_vcpu_regs.h> 43 #include <asm/virt.h> 44 45 #include <hyperv/hvhdk.h> 46 47 #define __KVM_HAVE_ARCH_VCPU_DEBUGFS 48 49 /* 50 * CONFIG_KVM_MAX_NR_VCPUS is defined iff CONFIG_KVM!=n, provide a dummy max if 51 * KVM is disabled (arbitrarily use the default from CONFIG_KVM_MAX_NR_VCPUS). 52 */ 53 #ifdef CONFIG_KVM_MAX_NR_VCPUS 54 #define KVM_MAX_VCPUS CONFIG_KVM_MAX_NR_VCPUS 55 #else 56 #define KVM_MAX_VCPUS 1024 57 #endif 58 59 /* 60 * In x86, the VCPU ID corresponds to the APIC ID, and APIC IDs 61 * might be larger than the actual number of VCPUs because the 62 * APIC ID encodes CPU topology information. 63 * 64 * In the worst case, we'll need less than one extra bit for the 65 * Core ID, and less than one extra bit for the Package (Die) ID, 66 * so ratio of 4 should be enough. 67 */ 68 #define KVM_VCPU_ID_RATIO 4 69 #define KVM_MAX_VCPU_IDS (KVM_MAX_VCPUS * KVM_VCPU_ID_RATIO) 70 71 /* memory slots that are not exposed to userspace */ 72 #define KVM_INTERNAL_MEM_SLOTS 3 73 74 #define KVM_HALT_POLL_NS_DEFAULT 200000 75 76 #define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS 77 78 #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ 79 KVM_DIRTY_LOG_INITIALLY_SET) 80 81 /* x86-specific vcpu->requests bit members */ 82 #define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0) 83 #define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1) 84 #define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2) 85 #define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3) 86 #define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4) 87 #define KVM_REQ_LOAD_MMU_PGD KVM_ARCH_REQ(5) 88 #define KVM_REQ_EVENT KVM_ARCH_REQ(6) 89 #define KVM_REQ_APF_HALT KVM_ARCH_REQ(7) 90 #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8) 91 #define KVM_REQ_NMI KVM_ARCH_REQ(9) 92 #define KVM_REQ_PMU KVM_ARCH_REQ(10) 93 #define KVM_REQ_PMI KVM_ARCH_REQ(11) 94 #ifdef CONFIG_KVM_SMM 95 #define KVM_REQ_SMI KVM_ARCH_REQ(12) 96 #endif 97 #define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13) 98 #define KVM_REQ_MCLOCK_INPROGRESS \ 99 KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 100 #define KVM_REQ_SCAN_IOAPIC \ 101 KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 102 #define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16) 103 #define KVM_REQ_APIC_PAGE_RELOAD \ 104 KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 105 #define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18) 106 #define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19) 107 #define KVM_REQ_HV_RESET KVM_ARCH_REQ(20) 108 #define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21) 109 #define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22) 110 #define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23) 111 #define KVM_REQ_GET_NESTED_STATE_PAGES KVM_ARCH_REQ(24) 112 #define KVM_REQ_APICV_UPDATE \ 113 KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 114 #define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26) 115 #define KVM_REQ_TLB_FLUSH_GUEST \ 116 KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 117 #define KVM_REQ_APF_READY KVM_ARCH_REQ(28) 118 #define KVM_REQ_RECALC_INTERCEPTS KVM_ARCH_REQ(29) 119 #define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \ 120 KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 121 #define KVM_REQ_MMU_FREE_OBSOLETE_ROOTS \ 122 KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 123 #define KVM_REQ_HV_TLB_FLUSH \ 124 KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 125 #define KVM_REQ_VMSA_PAGE_RELOAD \ 126 KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 127 #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \ 128 KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT) 129 130 #define INVALID_PAGE (~(hpa_t)0) 131 #define VALID_PAGE(x) ((x) != INVALID_PAGE) 132 133 /* KVM Hugepage definitions for x86 */ 134 #define KVM_MAX_HUGEPAGE_LEVEL PG_LEVEL_1G 135 #define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - PG_LEVEL_4K + 1) 136 #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) 137 #define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x)) 138 #define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x)) 139 #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) 140 #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) 141 142 #define KVM_MAX_CPUID_ENTRIES 256 143 #define KVM_NR_VAR_MTRR 8 144 145 #define ASYNC_PF_PER_VCPU 64 146 147 enum kvm_reg { 148 VCPU_REGS_RAX = __VCPU_REGS_RAX, 149 VCPU_REGS_RCX = __VCPU_REGS_RCX, 150 VCPU_REGS_RDX = __VCPU_REGS_RDX, 151 VCPU_REGS_RBX = __VCPU_REGS_RBX, 152 VCPU_REGS_RSP = __VCPU_REGS_RSP, 153 VCPU_REGS_RBP = __VCPU_REGS_RBP, 154 VCPU_REGS_RSI = __VCPU_REGS_RSI, 155 VCPU_REGS_RDI = __VCPU_REGS_RDI, 156 #ifdef CONFIG_X86_64 157 VCPU_REGS_R8 = 8, 158 VCPU_REGS_R9, 159 VCPU_REGS_R10, 160 VCPU_REGS_R11, 161 VCPU_REGS_R12, 162 VCPU_REGS_R13, 163 VCPU_REGS_R14, 164 VCPU_REGS_R15, 165 #endif 166 NR_VCPU_GENERAL_PURPOSE_REGS, 167 168 VCPU_REG_RIP = NR_VCPU_GENERAL_PURPOSE_REGS, 169 170 VCPU_REG_PDPTR, 171 VCPU_REG_CR0, 172 /* 173 * Alias AMD's ERAPS (not a real register) to CR3 so that common code 174 * can trigger emulation of the RAP (Return Address Predictor) with 175 * minimal support required in common code. Piggyback CR3 as the RAP 176 * is cleared on writes to CR3, i.e. marking CR3 dirty will naturally 177 * mark ERAPS dirty as well. 178 */ 179 VCPU_REG_CR3, 180 VCPU_REG_ERAPS = VCPU_REG_CR3, 181 VCPU_REG_CR4, 182 VCPU_REG_RFLAGS, 183 VCPU_REG_SEGMENTS, 184 VCPU_REG_EXIT_INFO_1, 185 VCPU_REG_EXIT_INFO_2, 186 187 NR_VCPU_TOTAL_REGS, 188 }; 189 190 enum { 191 VCPU_SREG_ES, 192 VCPU_SREG_CS, 193 VCPU_SREG_SS, 194 VCPU_SREG_DS, 195 VCPU_SREG_FS, 196 VCPU_SREG_GS, 197 VCPU_SREG_TR, 198 VCPU_SREG_LDTR, 199 }; 200 201 enum exit_fastpath_completion { 202 EXIT_FASTPATH_NONE, 203 EXIT_FASTPATH_REENTER_GUEST, 204 EXIT_FASTPATH_EXIT_HANDLED, 205 EXIT_FASTPATH_EXIT_USERSPACE, 206 }; 207 typedef enum exit_fastpath_completion fastpath_t; 208 209 struct x86_emulate_ctxt; 210 struct x86_exception; 211 union kvm_smram; 212 enum x86_intercept; 213 enum x86_intercept_stage; 214 215 #define KVM_NR_DB_REGS 4 216 217 #define PFERR_PRESENT_MASK BIT(0) 218 #define PFERR_WRITE_MASK BIT(1) 219 #define PFERR_USER_MASK BIT(2) 220 #define PFERR_RSVD_MASK BIT(3) 221 #define PFERR_FETCH_MASK BIT(4) 222 #define PFERR_PK_MASK BIT(5) 223 #define PFERR_SS_MASK BIT(6) 224 #define PFERR_SGX_MASK BIT(15) 225 #define PFERR_GUEST_RMP_MASK BIT_ULL(31) 226 #define PFERR_GUEST_FINAL_MASK BIT_ULL(32) 227 #define PFERR_GUEST_PAGE_MASK BIT_ULL(33) 228 #define PFERR_GUEST_FAULT_STAGE_MASK \ 229 (PFERR_GUEST_FINAL_MASK | PFERR_GUEST_PAGE_MASK) 230 #define PFERR_GUEST_ENC_MASK BIT_ULL(34) 231 #define PFERR_GUEST_SIZEM_MASK BIT_ULL(35) 232 #define PFERR_GUEST_VMPL_MASK BIT_ULL(36) 233 234 /* 235 * IMPLICIT_ACCESS is a KVM-defined flag used to correctly perform SMAP checks 236 * when emulating instructions that triggers implicit access. 237 */ 238 #define PFERR_IMPLICIT_ACCESS BIT_ULL(48) 239 /* 240 * PRIVATE_ACCESS is a KVM-defined flag us to indicate that a fault occurred 241 * when the guest was accessing private memory. 242 */ 243 #define PFERR_PRIVATE_ACCESS BIT_ULL(49) 244 #define PFERR_SYNTHETIC_MASK (PFERR_IMPLICIT_ACCESS | PFERR_PRIVATE_ACCESS) 245 246 struct kvm_kernel_irqfd; 247 struct kvm_kernel_irq_routing_entry; 248 249 struct kvm_apic_map; 250 251 struct kvm_x86_msr_filter; 252 struct kvm_x86_pmu_event_filter; 253 254 struct kvm_caps { 255 /* control of guest tsc rate supported? */ 256 bool has_tsc_control; 257 /* maximum supported tsc_khz for guests */ 258 u32 max_guest_tsc_khz; 259 /* number of bits of the fractional part of the TSC scaling ratio */ 260 u8 tsc_scaling_ratio_frac_bits; 261 /* maximum allowed value of TSC scaling ratio */ 262 u64 max_tsc_scaling_ratio; 263 /* 1ull << kvm_caps.tsc_scaling_ratio_frac_bits */ 264 u64 default_tsc_scaling_ratio; 265 /* bus lock detection supported? */ 266 bool has_bus_lock_exit; 267 /* notify VM exit supported? */ 268 bool has_notify_vmexit; 269 /* bit mask of VM types */ 270 u32 supported_vm_types; 271 272 u64 supported_mce_cap; 273 u64 supported_xcr0; 274 u64 supported_xss; 275 u64 supported_perf_cap; 276 277 u64 supported_efer_bits; 278 279 u64 supported_quirks; 280 u64 inapplicable_quirks; 281 }; 282 extern struct kvm_caps kvm_caps; 283 284 struct kvm_host_values { 285 /* 286 * The host's raw MAXPHYADDR, i.e. the number of non-reserved physical 287 * address bits irrespective of features that repurpose legal bits, 288 * e.g. MKTME. 289 */ 290 u8 maxphyaddr; 291 292 u64 efer; 293 u64 xcr0; 294 u64 xss; 295 u64 s_cet; 296 u64 arch_capabilities; 297 }; 298 extern struct kvm_host_values kvm_host; 299 300 /* 301 * kvm_mmu_page_role tracks the properties of a shadow page (where shadow page 302 * also includes TDP pages) to determine whether or not a page can be used in 303 * the given MMU context. This is a subset of the overall kvm_cpu_role to 304 * minimize the size of kvm_memory_slot.arch.gfn_write_track, i.e. allows 305 * allocating 2 bytes per gfn instead of 4 bytes per gfn. 306 * 307 * Upper-level shadow pages having gptes are tracked for write-protection via 308 * gfn_write_track. As above, gfn_write_track is a 16 bit counter, so KVM must 309 * not create more than 2^16-1 upper-level shadow pages at a single gfn, 310 * otherwise gfn_write_track will overflow and explosions will ensue. 311 * 312 * A unique shadow page (SP) for a gfn is created if and only if an existing SP 313 * cannot be reused. The ability to reuse a SP is tracked by its role, which 314 * incorporates various mode bits and properties of the SP. Roughly speaking, 315 * the number of unique SPs that can theoretically be created is 2^n, where n 316 * is the number of bits that are used to compute the role. 317 * 318 * But, even though there are 21 bits in the mask below, not all combinations 319 * of modes and flags are possible: 320 * 321 * - invalid shadow pages are not accounted, mirror pages are not shadowed, 322 * so the bits are effectively 19. 323 * 324 * - quadrant will only be used if has_4_byte_gpte=1 (non-PAE paging); 325 * execonly and ad_disabled are only used for nested EPT which has 326 * has_4_byte_gpte=0. Therefore, 2 bits are always unused. 327 * 328 * - the 4 bits of level are effectively limited to the values 2/3/4/5, 329 * as 4k SPs are not tracked (allowed to go unsync). In addition non-PAE 330 * paging has exactly one upper level, making level completely redundant 331 * when has_4_byte_gpte=1. 332 * 333 * - on top of this, smap_andnot_wp is only set if cr0_wp=0, 334 * therefore these two bits only give rise to 3 possibilities. 335 * 336 * Therefore, the maximum number of possible upper-level shadow pages for a 337 * single gfn is a bit less than 2^14. 338 */ 339 union kvm_mmu_page_role { 340 u32 word; 341 struct { 342 unsigned level:4; 343 unsigned has_4_byte_gpte:1; 344 unsigned quadrant:2; 345 unsigned direct:1; 346 unsigned access:4; 347 unsigned invalid:1; 348 unsigned efer_nx:1; 349 unsigned cr0_wp:1; 350 unsigned smap_andnot_wp:1; 351 unsigned ad_disabled:1; 352 unsigned guest_mode:1; 353 unsigned passthrough:1; 354 unsigned is_mirror:1; 355 356 /* 357 * cr4_smep is also set for EPT MBEC. Because it affects 358 * which pages are considered non-present (bit 10 additionally 359 * must be zero if MBEC is on) it has to be in the base role. 360 * It also has to be in the base role for AMD GMET because 361 * kernel-executable pages need to have U=0 with GMET enabled. 362 */ 363 unsigned cr4_smep:1; 364 365 unsigned:3; 366 367 /* 368 * This is left at the top of the word so that 369 * kvm_memslots_for_spte_role can extract it with a 370 * simple shift. While there is room, give it a whole 371 * byte so it is also faster to load it from memory. 372 */ 373 unsigned smm:8; 374 }; 375 }; 376 377 /* 378 * kvm_mmu_extended_role complements kvm_mmu_page_role, tracking properties 379 * relevant to the current MMU configuration. When loading CR0, CR4, or EFER, 380 * including on nested transitions, if nothing in the full role changes then 381 * MMU re-configuration can be skipped. @valid bit is set on first usage so we 382 * don't treat all-zero structure as valid data. 383 * 384 * The properties that are tracked in the extended role but not the page role 385 * are for things that either (a) do not affect the validity of the shadow page 386 * or (b) are indirectly reflected in the shadow page's role. For example, 387 * CR4.PKE only affects permission checks for software walks of the guest page 388 * tables (because KVM doesn't support Protection Keys with shadow paging), and 389 * CR0.PG, CR4.PAE, and CR4.PSE are indirectly reflected in role.level. 390 * 391 * Note, SMAP is not redundant with smap_andnot_wp in the page role. If 392 * CR0.WP=1, KVM can reuse shadow pages for the guest regardless of SMAP, 393 * but the MMU's permission checks for software walks need to be SMAP 394 * aware regardless of CR0.WP. 395 */ 396 union kvm_mmu_extended_role { 397 u32 word; 398 struct { 399 unsigned int valid:1; 400 unsigned int execonly:1; 401 unsigned int cr4_pse:1; 402 unsigned int cr4_pke:1; 403 unsigned int cr4_smap:1; 404 unsigned int cr4_la57:1; 405 unsigned int efer_lma:1; 406 407 /* 408 * True if either CR4.SMEP or EFER.NXE are set. For AMD NPT 409 * this is the "real" host CR4.SMEP whereas cr4_smep is 410 * actually GMET. 411 */ 412 unsigned int has_pferr_fetch:1; 413 }; 414 }; 415 416 union kvm_cpu_role { 417 u64 as_u64; 418 struct { 419 union kvm_mmu_page_role base; 420 union kvm_mmu_extended_role ext; 421 }; 422 }; 423 424 struct kvm_rmap_head { 425 atomic_long_t val; 426 }; 427 428 struct kvm_pio_request { 429 unsigned long count; 430 int in; 431 int port; 432 int size; 433 }; 434 435 #define PT64_ROOT_MAX_LEVEL 5 436 437 struct kvm_page_format { 438 u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL]; 439 u64 bad_mt_xwr; 440 441 /* 442 * The pkru_mask indicates if protection key checks are needed. It 443 * consists of 16 domains indexed by page fault error code bits [4:1], 444 * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables. 445 * Each domain has 2 bits which are ANDed with AD and WD from PKRU. 446 */ 447 u32 pkru_mask; 448 449 /* 450 * Bitmap; bit set = permission fault 451 * Array index: page fault error code [4:1] 452 * Bit index: pte permissions in ACC_* format 453 */ 454 u16 permissions[16]; 455 }; 456 457 struct kvm_mmu_root_info { 458 gpa_t pgd; 459 hpa_t hpa; 460 }; 461 462 #define KVM_MMU_ROOT_INFO_INVALID \ 463 ((struct kvm_mmu_root_info) { .pgd = INVALID_PAGE, .hpa = INVALID_PAGE }) 464 465 #define KVM_MMU_NUM_PREV_ROOTS 3 466 467 #define KVM_MMU_ROOT_CURRENT BIT(0) 468 #define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i) 469 #define KVM_MMU_ROOTS_ALL (BIT(1 + KVM_MMU_NUM_PREV_ROOTS) - 1) 470 471 #define KVM_HAVE_MMU_RWLOCK 472 473 struct kvm_mmu_page; 474 struct kvm_page_fault; 475 476 /* 477 * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit, 478 * and 2-level 32-bit). The kvm_pagewalk structure abstracts the details of the 479 * current mmu mode. 480 */ 481 struct kvm_pagewalk { 482 unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu); 483 u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); 484 void (*inject_page_fault)(struct kvm_vcpu *vcpu, 485 struct x86_exception *fault, 486 bool from_hardware); 487 gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w, 488 gpa_t gva_or_gpa, u64 access, 489 struct x86_exception *exception); 490 491 union kvm_cpu_role cpu_role; 492 struct kvm_page_format fmt; 493 }; 494 495 struct kvm_mmu { 496 int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); 497 int (*sync_spte)(struct kvm_vcpu *vcpu, 498 struct kvm_mmu_page *sp, int i); 499 struct kvm_pagewalk *w; 500 501 struct kvm_mmu_root_info root; 502 hpa_t mirror_root_hpa; 503 union kvm_mmu_page_role root_role; 504 505 struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS]; 506 507 u64 *pae_root; 508 u64 *pml4_root; 509 u64 *pml5_root; 510 511 /* 512 * check zero bits on shadow page table entries, these 513 * bits include not only hardware reserved bits but also 514 * the bits spte never used. 515 */ 516 struct kvm_page_format fmt; 517 }; 518 519 enum pmc_type { 520 KVM_PMC_GP = 0, 521 KVM_PMC_FIXED, 522 }; 523 524 struct kvm_pmc { 525 enum pmc_type type; 526 u8 idx; 527 bool is_paused; 528 bool intr; 529 /* 530 * Base value of the PMC counter, relative to the *consumed* count in 531 * the associated perf_event. This value includes counter updates from 532 * the perf_event and emulated_count since the last time the counter 533 * was reprogrammed, but it is *not* the current value as seen by the 534 * guest or userspace. 535 * 536 * The count is relative to the associated perf_event so that KVM 537 * doesn't need to reprogram the perf_event every time the guest writes 538 * to the counter. 539 */ 540 u64 counter; 541 /* 542 * PMC events triggered by KVM emulation that haven't been fully 543 * processed, i.e. haven't undergone overflow detection. 544 */ 545 u64 emulated_counter; 546 u64 eventsel; 547 u64 eventsel_hw; 548 struct perf_event *perf_event; 549 struct kvm_vcpu *vcpu; 550 /* 551 * only for creating or reusing perf_event, 552 * eventsel value for general purpose counters, 553 * ctrl value for fixed counters. 554 */ 555 u64 current_config; 556 }; 557 558 /* More counters may conflict with other existing Architectural MSRs */ 559 #define KVM_MAX(a, b) ((a) >= (b) ? (a) : (b)) 560 #define KVM_MAX_NR_INTEL_GP_COUNTERS 8 561 #define KVM_MAX_NR_AMD_GP_COUNTERS 6 562 #define KVM_MAX_NR_GP_COUNTERS KVM_MAX(KVM_MAX_NR_INTEL_GP_COUNTERS, \ 563 KVM_MAX_NR_AMD_GP_COUNTERS) 564 565 #define KVM_MAX_NR_INTEL_FIXED_COUNTERS 3 566 #define KVM_MAX_NR_AMD_FIXED_COUNTERS 0 567 #define KVM_MAX_NR_FIXED_COUNTERS KVM_MAX(KVM_MAX_NR_INTEL_FIXED_COUNTERS, \ 568 KVM_MAX_NR_AMD_FIXED_COUNTERS) 569 570 struct kvm_pmu { 571 u8 version; 572 unsigned nr_arch_gp_counters; 573 unsigned nr_arch_fixed_counters; 574 unsigned available_event_types; 575 u64 fixed_ctr_ctrl; 576 u64 fixed_ctr_ctrl_hw; 577 u64 fixed_ctr_ctrl_rsvd; 578 u64 global_ctrl; 579 u64 global_status; 580 u64 counter_bitmask[2]; 581 u64 global_ctrl_rsvd; 582 u64 global_status_rsvd; 583 u64 reserved_bits; 584 u64 raw_event_mask; 585 struct kvm_pmc gp_counters[KVM_MAX_NR_GP_COUNTERS]; 586 struct kvm_pmc fixed_counters[KVM_MAX_NR_FIXED_COUNTERS]; 587 588 /* 589 * Overlay the bitmap with a 64-bit atomic so that all bits can be 590 * set in a single access, e.g. to reprogram all counters when the PMU 591 * filter changes. 592 */ 593 union { 594 DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX); 595 atomic64_t __reprogram_pmi; 596 }; 597 DECLARE_BITMAP(all_valid_pmc_idx, X86_PMC_IDX_MAX); 598 DECLARE_BITMAP(pmc_in_use, X86_PMC_IDX_MAX); 599 600 DECLARE_BITMAP(pmc_counting_instructions, X86_PMC_IDX_MAX); 601 DECLARE_BITMAP(pmc_counting_branches, X86_PMC_IDX_MAX); 602 603 DECLARE_BITMAP(pmc_has_mode_specific_enables, X86_PMC_IDX_MAX); 604 605 u64 ds_area; 606 u64 pebs_enable; 607 u64 pebs_enable_rsvd; 608 u64 pebs_data_cfg; 609 u64 pebs_data_cfg_rsvd; 610 611 /* 612 * If a guest counter is cross-mapped to host counter with different 613 * index, its PEBS capability will be temporarily disabled. 614 * 615 * The user should make sure that this mask is updated 616 * after disabling interrupts and before perf_guest_get_msrs(); 617 */ 618 u64 host_cross_mapped_mask; 619 620 /* 621 * The gate to release perf_events not marked in 622 * pmc_in_use only once in a vcpu time slice. 623 */ 624 bool need_cleanup; 625 626 /* 627 * The total number of programmed perf_events and it helps to avoid 628 * redundant check before cleanup if guest don't use vPMU at all. 629 */ 630 u8 event_count; 631 }; 632 633 struct kvm_pmu_ops; 634 635 enum { 636 KVM_DEBUGREG_BP_ENABLED = BIT(0), 637 KVM_DEBUGREG_WONT_EXIT = BIT(1), 638 /* 639 * Guest debug registers (DR0-3, DR6 and DR7) are saved/restored by 640 * hardware on exit from or enter to guest. KVM needn't switch them. 641 * DR0-3, DR6 and DR7 are set to their architectural INIT value on VM 642 * exit, host values need to be restored. 643 */ 644 KVM_DEBUGREG_AUTO_SWITCH = BIT(2), 645 }; 646 647 struct kvm_mtrr { 648 u64 var[KVM_NR_VAR_MTRR * 2]; 649 u64 fixed_64k; 650 u64 fixed_16k[2]; 651 u64 fixed_4k[8]; 652 u64 deftype; 653 }; 654 655 struct kvm_hypervisor_cpuid { 656 u32 base; 657 u32 limit; 658 }; 659 660 #ifdef CONFIG_KVM_XEN 661 /* Xen HVM per vcpu emulation context */ 662 struct kvm_vcpu_xen { 663 u64 hypercall_rip; 664 u32 current_runstate; 665 u8 upcall_vector; 666 struct gfn_to_pfn_cache vcpu_info_cache; 667 struct gfn_to_pfn_cache vcpu_time_info_cache; 668 struct gfn_to_pfn_cache runstate_cache; 669 struct gfn_to_pfn_cache runstate2_cache; 670 u64 last_steal; 671 u64 runstate_entry_time; 672 u64 runstate_times[4]; 673 unsigned long evtchn_pending_sel; 674 u32 vcpu_id; /* The Xen / ACPI vCPU ID */ 675 u32 timer_virq; 676 u64 timer_expires; /* In guest epoch */ 677 atomic_t timer_pending; 678 struct hrtimer timer; 679 int poll_evtchn; 680 struct timer_list poll_timer; 681 struct kvm_hypervisor_cpuid cpuid; 682 }; 683 #endif 684 685 struct kvm_vcpu_hv; 686 687 struct kvm_queued_exception { 688 bool pending; 689 bool injected; 690 bool has_error_code; 691 u8 vector; 692 u32 error_code; 693 unsigned long payload; 694 bool has_payload; 695 }; 696 697 /* 698 * Hardware-defined CPUID leafs that are either scattered by the kernel or are 699 * unknown to the kernel, but need to be directly used by KVM. Note, these 700 * word values conflict with the kernel's "bug" caps, but KVM doesn't use those. 701 */ 702 enum kvm_only_cpuid_leafs { 703 CPUID_12_EAX = NCAPINTS, 704 CPUID_7_1_EDX, 705 CPUID_8000_0007_EDX, 706 CPUID_8000_0022_EAX, 707 CPUID_7_2_EDX, 708 CPUID_24_0_EBX, 709 CPUID_8000_0021_ECX, 710 CPUID_7_1_ECX, 711 CPUID_1E_1_EAX, 712 CPUID_24_1_ECX, 713 NR_KVM_CPU_CAPS, 714 715 NKVMCAPINTS = NR_KVM_CPU_CAPS - NCAPINTS, 716 }; 717 718 struct kvm_vcpu_arch { 719 /* 720 * rip and regs accesses must go through 721 * kvm_{register,rip}_{read,write} functions. 722 */ 723 unsigned long regs[NR_VCPU_GENERAL_PURPOSE_REGS]; 724 unsigned long rip; 725 DECLARE_BITMAP(regs_avail, NR_VCPU_TOTAL_REGS); 726 DECLARE_BITMAP(regs_dirty, NR_VCPU_TOTAL_REGS); 727 728 unsigned long cr0; 729 unsigned long cr0_guest_owned_bits; 730 unsigned long cr2; 731 unsigned long cr3; 732 unsigned long cr4; 733 unsigned long cr4_guest_owned_bits; 734 unsigned long cr4_guest_rsvd_bits; 735 unsigned long cr8; 736 u32 host_pkru; 737 u32 pkru; 738 u32 hflags; 739 u64 efer; 740 u64 host_debugctl; 741 u64 apic_base; 742 struct kvm_lapic *apic; /* kernel irqchip context */ 743 bool load_eoi_exitmap_pending; 744 DECLARE_BITMAP(ioapic_handled_vectors, 256); 745 unsigned long apic_attention; 746 int32_t apic_arb_prio; 747 int mp_state; 748 u64 ia32_misc_enable_msr; 749 u64 smbase; 750 u64 smi_count; 751 bool at_instruction_boundary; 752 bool tpr_access_reporting; 753 bool xfd_no_write_intercept; 754 u64 microcode_version; 755 u64 arch_capabilities; 756 u64 perf_capabilities; 757 758 /* 759 * Paging state of the vcpu 760 * 761 * If the vcpu runs in guest mode with two level paging this still saves 762 * the paging mode of the l1 guest. This context is always used to 763 * handle faults. 764 */ 765 struct kvm_mmu *mmu; 766 767 /* Non-nested MMU for L1 */ 768 struct kvm_mmu root_mmu; 769 770 /* L1 TDP when running nested */ 771 struct kvm_mmu guest_mmu; 772 struct kvm_pagewalk ngpa_walk; 773 774 /* 775 * Pagewalk context used for gva_to_gpa translations. 776 */ 777 struct kvm_pagewalk gva_walk; 778 779 u64 pdptrs[4]; /* pae */ 780 781 struct kvm_mmu_memory_cache mmu_pte_list_desc_cache; 782 struct kvm_mmu_memory_cache mmu_shadow_page_cache; 783 struct kvm_mmu_memory_cache mmu_shadowed_info_cache; 784 struct kvm_mmu_memory_cache mmu_page_header_cache; 785 /* 786 * This cache is to allocate external page table. E.g. private EPT used 787 * by the TDX module. 788 */ 789 struct kvm_mmu_memory_cache mmu_external_spt_cache; 790 791 /* 792 * QEMU userspace and the guest each have their own FPU state. 793 * In vcpu_run, we switch between the user and guest FPU contexts. 794 * While running a VCPU, the VCPU thread will have the guest FPU 795 * context. 796 * 797 * Note that while the PKRU state lives inside the fpu registers, 798 * it is switched out separately at VMENTER and VMEXIT time. The 799 * "guest_fpstate" state here contains the guest FPU context, with the 800 * host PRKU bits. 801 */ 802 struct fpu_guest guest_fpu; 803 804 u64 xcr0; 805 u64 guest_supported_xcr0; 806 u64 ia32_xss; 807 u64 guest_supported_xss; 808 809 struct kvm_pio_request pio; 810 void *pio_data; 811 void *sev_pio_data; 812 unsigned sev_pio_count; 813 814 u8 event_exit_inst_len; 815 816 bool exception_from_userspace; 817 818 /* Exceptions to be injected to the guest. */ 819 struct kvm_queued_exception exception; 820 /* Exception VM-Exits to be synthesized to L1. */ 821 struct kvm_queued_exception exception_vmexit; 822 823 struct kvm_queued_interrupt { 824 bool injected; 825 bool soft; 826 u8 nr; 827 } interrupt; 828 829 int halt_request; /* real mode on Intel only */ 830 831 int cpuid_nent; 832 struct kvm_cpuid_entry2 *cpuid_entries; 833 bool cpuid_dynamic_bits_dirty; 834 bool is_amd_compatible; 835 836 /* 837 * cpu_caps holds the effective guest capabilities, i.e. the features 838 * the vCPU is allowed to use. Typically, but not always, features can 839 * be used by the guest if and only if both KVM and userspace want to 840 * expose the feature to the guest. 841 * 842 * A common exception is for virtualization holes, i.e. when KVM can't 843 * prevent the guest from using a feature, in which case the vCPU "has" 844 * the feature regardless of what KVM or userspace desires. 845 * 846 * Note, features that don't require KVM involvement in any way are 847 * NOT enforced/sanitized by KVM, i.e. are taken verbatim from the 848 * guest CPUID provided by userspace. 849 */ 850 u32 cpu_caps[NR_KVM_CPU_CAPS]; 851 852 u64 reserved_gpa_bits; 853 int maxphyaddr; 854 855 /* emulate context */ 856 857 struct x86_emulate_ctxt *emulate_ctxt; 858 bool emulate_regs_need_sync_to_vcpu; 859 bool emulate_regs_need_sync_from_vcpu; 860 int (*complete_userspace_io)(struct kvm_vcpu *vcpu); 861 unsigned long cui_linear_rip; 862 int cui_rdmsr_imm_reg; 863 864 gpa_t time; 865 s8 pvclock_tsc_shift; 866 u32 pvclock_tsc_mul; 867 unsigned int hw_tsc_khz; 868 struct gfn_to_pfn_cache pv_time; 869 /* set guest stopped flag in pvclock flags field */ 870 bool pvclock_set_guest_stopped_request; 871 872 struct { 873 u8 preempted; 874 u64 msr_val; 875 u64 last_steal; 876 struct gfn_to_hva_cache cache; 877 } st; 878 879 u64 l1_tsc_offset; 880 u64 tsc_offset; /* current tsc offset */ 881 u64 last_guest_tsc; 882 u64 last_host_tsc; 883 u64 tsc_offset_adjustment; 884 u64 this_tsc_nsec; 885 u64 this_tsc_write; 886 u64 this_tsc_generation; 887 bool tsc_catchup; 888 bool tsc_always_catchup; 889 s8 virtual_tsc_shift; 890 u32 virtual_tsc_mult; 891 u32 virtual_tsc_khz; 892 s64 ia32_tsc_adjust_msr; 893 u64 msr_ia32_power_ctl; 894 u64 l1_tsc_scaling_ratio; 895 u64 tsc_scaling_ratio; /* current scaling ratio */ 896 897 atomic_t nmi_queued; /* unprocessed asynchronous NMIs */ 898 /* Number of NMIs pending injection, not including hardware vNMIs. */ 899 unsigned int nmi_pending; 900 bool nmi_injected; /* Trying to inject an NMI this entry */ 901 bool smi_pending; /* SMI queued after currently running handler */ 902 u8 handling_intr_from_guest; 903 904 struct kvm_mtrr mtrr_state; 905 u64 pat; 906 907 unsigned switch_db_regs; 908 unsigned long db[KVM_NR_DB_REGS]; 909 unsigned long dr6; 910 unsigned long dr7; 911 unsigned long eff_db[KVM_NR_DB_REGS]; 912 unsigned long guest_debug_dr7; 913 u64 msr_platform_info; 914 u64 msr_misc_features_enables; 915 916 u64 mcg_cap; 917 u64 mcg_status; 918 u64 mcg_ctl; 919 u64 mcg_ext_ctl; 920 u64 *mce_banks; 921 u64 *mci_ctl2_banks; 922 923 /* Cache MMIO info */ 924 u64 mmio_gva; 925 unsigned mmio_access; 926 gfn_t mmio_gfn; 927 u64 mmio_gen; 928 929 struct kvm_pmu pmu; 930 931 /* used for guest single stepping over the given code position */ 932 unsigned long singlestep_rip; 933 934 #ifdef CONFIG_KVM_HYPERV 935 bool hyperv_enabled; 936 struct kvm_vcpu_hv *hyperv; 937 #endif 938 #ifdef CONFIG_KVM_XEN 939 struct kvm_vcpu_xen xen; 940 #endif 941 cpumask_var_t wbinvd_dirty_mask; 942 943 unsigned long last_retry_eip; 944 unsigned long last_retry_addr; 945 946 struct { 947 bool halted; 948 gfn_t gfns[ASYNC_PF_PER_VCPU]; 949 struct gfn_to_hva_cache data; 950 u64 msr_en_val; /* MSR_KVM_ASYNC_PF_EN */ 951 u64 msr_int_val; /* MSR_KVM_ASYNC_PF_INT */ 952 u16 vec; 953 u32 id; 954 u32 host_apf_flags; 955 bool pageready_pending; 956 } apf; 957 958 /* OSVW MSRs (AMD only) */ 959 struct { 960 u64 length; 961 u64 status; 962 } osvw; 963 964 struct { 965 u64 msr_val; 966 struct gfn_to_hva_cache data; 967 } pv_eoi; 968 969 u64 msr_kvm_poll_control; 970 971 /* pv related host specific info */ 972 struct { 973 bool pv_unhalted; 974 } pv; 975 976 int pending_ioapic_eoi; 977 int pending_external_vector; 978 int highest_stale_pending_ioapic_eoi; 979 980 /* be preempted when it's in kernel-mode(cpl=0) */ 981 bool preempted_in_kernel; 982 983 /* Host CPU on which VM-entry was most recently attempted */ 984 int last_vmentry_cpu; 985 986 /* AMD MSRC001_0015 Hardware Configuration */ 987 u64 msr_hwcr; 988 989 /* pv related cpuid info */ 990 struct { 991 /* 992 * value of the eax register in the KVM_CPUID_FEATURES CPUID 993 * leaf. 994 */ 995 u32 features; 996 997 /* 998 * indicates whether pv emulation should be disabled if features 999 * are not present in the guest's cpuid 1000 */ 1001 bool enforce; 1002 } pv_cpuid; 1003 1004 /* Protected Guests */ 1005 bool guest_state_protected; 1006 bool guest_tsc_protected; 1007 1008 /* 1009 * Set when PDPTS were loaded directly by the userspace without 1010 * reading the guest memory 1011 */ 1012 bool pdptrs_from_userspace; 1013 1014 /* 1015 * Set if an emulated nested VM-Enter to L2 is pending completion. KVM 1016 * must not synthesize a VM-Exit to L1 before entering L2, as VM-Exits 1017 * can only occur at instruction boundaries. The only exception is 1018 * VMX's "notify" exits, which exist in large part to break the CPU out 1019 * of infinite ucode loops, but can corrupt vCPU state in the process! 1020 * 1021 * For all intents and purposes, this is a boolean, but it's tracked as 1022 * a u8 so that KVM can detect when userspace may have stuffed vCPU 1023 * state and generated an architecturally-impossible VM-Exit. 1024 */ 1025 #define KVM_NESTED_RUN_PENDING 1 1026 #define KVM_NESTED_RUN_PENDING_UNTRUSTED 2 1027 u8 nested_run_pending; 1028 1029 #if IS_ENABLED(CONFIG_HYPERV) 1030 hpa_t hv_root_tdp; 1031 #endif 1032 }; 1033 1034 struct kvm_lpage_info { 1035 int disallow_lpage; 1036 }; 1037 1038 struct kvm_arch_memory_slot { 1039 struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES]; 1040 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; 1041 unsigned short *gfn_write_track; 1042 }; 1043 1044 /* Hyper-V synthetic debugger (SynDbg)*/ 1045 struct kvm_hv_syndbg { 1046 struct { 1047 u64 control; 1048 u64 status; 1049 u64 send_page; 1050 u64 recv_page; 1051 u64 pending_page; 1052 } control; 1053 u64 options; 1054 }; 1055 1056 /* Current state of Hyper-V TSC page clocksource */ 1057 enum hv_tsc_page_status { 1058 /* TSC page was not set up or disabled */ 1059 HV_TSC_PAGE_UNSET = 0, 1060 /* TSC page MSR was written by the guest, update pending */ 1061 HV_TSC_PAGE_GUEST_CHANGED, 1062 /* TSC page update was triggered from the host side */ 1063 HV_TSC_PAGE_HOST_CHANGED, 1064 /* TSC page was properly set up and is currently active */ 1065 HV_TSC_PAGE_SET, 1066 /* TSC page was set up with an inaccessible GPA */ 1067 HV_TSC_PAGE_BROKEN, 1068 }; 1069 1070 #ifdef CONFIG_KVM_HYPERV 1071 /* Hyper-V emulation context */ 1072 struct kvm_hv { 1073 struct mutex hv_lock; 1074 u64 hv_guest_os_id; 1075 u64 hv_hypercall; 1076 u64 hv_tsc_page; 1077 enum hv_tsc_page_status hv_tsc_page_status; 1078 1079 /* Hyper-v based guest crash (NT kernel bugcheck) parameters */ 1080 u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS]; 1081 u64 hv_crash_ctl; 1082 1083 struct ms_hyperv_tsc_page tsc_ref; 1084 1085 struct idr conn_to_evt; 1086 1087 u64 hv_reenlightenment_control; 1088 u64 hv_tsc_emulation_control; 1089 u64 hv_tsc_emulation_status; 1090 u64 hv_invtsc_control; 1091 1092 /* How many vCPUs have VP index != vCPU index */ 1093 atomic_t num_mismatched_vp_indexes; 1094 1095 /* 1096 * How many SynICs use 'AutoEOI' feature 1097 * (protected by arch.apicv_update_lock) 1098 */ 1099 unsigned int synic_auto_eoi_used; 1100 1101 struct kvm_hv_syndbg hv_syndbg; 1102 1103 bool xsaves_xsavec_checked; 1104 }; 1105 #endif 1106 1107 #ifdef CONFIG_KVM_XEN 1108 /* Xen emulation context */ 1109 struct kvm_xen { 1110 struct mutex xen_lock; 1111 u32 xen_version; 1112 bool long_mode; 1113 bool runstate_update_flag; 1114 u8 upcall_vector; 1115 struct gfn_to_pfn_cache shinfo_cache; 1116 struct idr evtchn_ports; 1117 unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 1118 1119 struct kvm_xen_hvm_config hvm_config; 1120 }; 1121 #endif 1122 1123 enum kvm_irqchip_mode { 1124 KVM_IRQCHIP_NONE, 1125 #ifdef CONFIG_KVM_IOAPIC 1126 KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */ 1127 #endif 1128 KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */ 1129 }; 1130 1131 enum kvm_suppress_eoi_broadcast_mode { 1132 KVM_SUPPRESS_EOI_BROADCAST_QUIRKED, /* Legacy behavior */ 1133 KVM_SUPPRESS_EOI_BROADCAST_ENABLED, /* Enable Suppress EOI broadcast */ 1134 KVM_SUPPRESS_EOI_BROADCAST_DISABLED /* Disable Suppress EOI broadcast */ 1135 }; 1136 1137 struct kvm_possible_nx_huge_pages { 1138 /* 1139 * A list of kvm_mmu_page structs that, if zapped, could possibly be 1140 * replaced by an NX huge page. A shadow page is on this list if its 1141 * existence disallows an NX huge page (nx_huge_page_disallowed is set) 1142 * and there are no other conditions that prevent a huge page, e.g. 1143 * the backing host page is huge, dirtly logging is not enabled for its 1144 * memslot, etc... Note, zapping shadow pages on this list doesn't 1145 * guarantee an NX huge page will be created in its stead, e.g. if the 1146 * guest attempts to execute from the region then KVM obviously can't 1147 * create an NX huge page (without hanging the guest). 1148 */ 1149 struct list_head pages; 1150 u64 nr_pages; 1151 }; 1152 1153 enum kvm_mmu_type { 1154 KVM_SHADOW_MMU, 1155 #ifdef CONFIG_X86_64 1156 KVM_TDP_MMU, 1157 #endif 1158 KVM_NR_MMU_TYPES, 1159 }; 1160 1161 struct kvm_arch { 1162 unsigned long n_requested_mmu_pages; 1163 unsigned long n_max_mmu_pages; 1164 unsigned int indirect_shadow_pages; 1165 u8 mmu_valid_gen; 1166 u8 vm_type; 1167 bool has_private_mem; 1168 bool has_protected_state; 1169 bool has_protected_eoi; 1170 bool has_protected_pmu; 1171 bool pre_fault_allowed; 1172 struct hlist_head *mmu_page_hash; 1173 struct list_head active_mmu_pages; 1174 struct kvm_possible_nx_huge_pages possible_nx_huge_pages[KVM_NR_MMU_TYPES]; 1175 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING 1176 struct kvm_page_track_notifier_head track_notifier_head; 1177 #endif 1178 /* 1179 * Protects marking pages unsync during page faults, as TDP MMU page 1180 * faults only take mmu_lock for read. For simplicity, the unsync 1181 * pages lock is always taken when marking pages unsync regardless of 1182 * whether mmu_lock is held for read or write. 1183 */ 1184 spinlock_t mmu_unsync_pages_lock; 1185 1186 u64 shadow_mmio_value; 1187 1188 #define __KVM_HAVE_ARCH_NONCOHERENT_DMA 1189 atomic_t noncoherent_dma_count; 1190 unsigned long nr_possible_bypass_irqs; 1191 1192 #ifdef CONFIG_KVM_IOAPIC 1193 struct kvm_pic *vpic; 1194 struct kvm_ioapic *vioapic; 1195 struct kvm_pit *vpit; 1196 #endif 1197 atomic_t vapics_in_nmi_mode; 1198 1199 struct mutex apic_map_lock; 1200 struct kvm_apic_map __rcu *apic_map; 1201 atomic_t apic_map_dirty; 1202 1203 bool apic_access_memslot_enabled; 1204 bool apic_access_memslot_inhibited; 1205 1206 /* 1207 * Force apicv_update_lock and apicv_nr_irq_window_req to reside in a 1208 * dedicated cacheline. They are write-mostly, whereas most everything 1209 * else in kvm_arch is read-mostly. Note that apicv_inhibit_reasons is 1210 * read-mostly: toggling VM-wide inhibits is rare; _checking_ for 1211 * inhibits is common. 1212 */ 1213 ____cacheline_aligned 1214 /* 1215 * Protects apicv_inhibit_reasons and apicv_nr_irq_window_req (with an 1216 * asterisk, see kvm_inc_or_dec_irq_window_inhibit() for details). 1217 */ 1218 struct rw_semaphore apicv_update_lock; 1219 atomic_t apicv_nr_irq_window_req; 1220 ____cacheline_aligned 1221 1222 unsigned long apicv_inhibit_reasons; 1223 1224 gpa_t wall_clock; 1225 1226 u64 disabled_exits; 1227 1228 s64 kvmclock_offset; 1229 1230 /* 1231 * This also protects nr_vcpus_matched_tsc which is read from a 1232 * preemption-disabled region, so it must be a raw spinlock. 1233 */ 1234 raw_spinlock_t tsc_write_lock; 1235 u64 last_tsc_nsec; 1236 u64 last_tsc_write; 1237 u32 last_tsc_khz; 1238 u64 last_tsc_offset; 1239 u64 cur_tsc_nsec; 1240 u64 cur_tsc_write; 1241 u64 cur_tsc_offset; 1242 u64 cur_tsc_generation; 1243 int nr_vcpus_matched_tsc; 1244 1245 u32 default_tsc_khz; 1246 bool user_set_tsc; 1247 u64 apic_bus_cycle_ns; 1248 1249 seqcount_raw_spinlock_t pvclock_sc; 1250 bool use_master_clock; 1251 u64 master_kernel_ns; 1252 u64 master_cycle_now; 1253 struct ratelimit_state kvmclock_update_rs; 1254 1255 #ifdef CONFIG_KVM_HYPERV 1256 struct kvm_hv hyperv; 1257 #endif 1258 1259 #ifdef CONFIG_KVM_XEN 1260 struct kvm_xen xen; 1261 #endif 1262 1263 bool backwards_tsc_observed; 1264 bool boot_vcpu_runs_old_kvmclock; 1265 u32 bsp_vcpu_id; 1266 1267 u64 disabled_quirks; 1268 1269 enum kvm_irqchip_mode irqchip_mode; 1270 u8 nr_reserved_ioapic_pins; 1271 1272 bool disabled_lapic_found; 1273 1274 bool x2apic_format; 1275 bool x2apic_broadcast_quirk_disabled; 1276 enum kvm_suppress_eoi_broadcast_mode suppress_eoi_broadcast_mode; 1277 1278 bool has_mapped_host_mmio; 1279 bool guest_can_read_msr_platform_info; 1280 bool exception_payload_enabled; 1281 1282 bool triple_fault_event; 1283 1284 bool bus_lock_detection_enabled; 1285 bool enable_pmu; 1286 bool created_mediated_pmu; 1287 1288 u32 notify_window; 1289 u32 notify_vmexit_flags; 1290 /* 1291 * If exit_on_emulation_error is set, and the in-kernel instruction 1292 * emulator fails to emulate an instruction, allow userspace 1293 * the opportunity to look at it. 1294 */ 1295 bool exit_on_emulation_error; 1296 1297 /* Deflect RDMSR and WRMSR to user space when they trigger a #GP */ 1298 u32 user_space_msr_mask; 1299 struct kvm_x86_msr_filter __rcu *msr_filter; 1300 1301 u32 hypercall_exit_enabled; 1302 1303 /* Guest can access the SGX PROVISIONKEY. */ 1304 bool sgx_provisioning_allowed; 1305 1306 struct kvm_x86_pmu_event_filter __rcu *pmu_event_filter; 1307 struct vhost_task *nx_huge_page_recovery_thread; 1308 u64 nx_huge_page_last; 1309 struct once nx_once; 1310 1311 #ifdef CONFIG_X86_64 1312 #ifdef CONFIG_KVM_PROVE_MMU 1313 /* 1314 * The number of TDP MMU pages across all roots. Used only to sanity 1315 * check that KVM isn't leaking TDP MMU pages. 1316 */ 1317 atomic64_t tdp_mmu_pages; 1318 #endif 1319 1320 /* 1321 * List of struct kvm_mmu_pages being used as roots. 1322 * All struct kvm_mmu_pages in the list should have 1323 * tdp_mmu_page set. 1324 * 1325 * For reads, this list is protected by: 1326 * RCU alone or 1327 * the MMU lock in read mode + RCU or 1328 * the MMU lock in write mode 1329 * 1330 * For writes, this list is protected by tdp_mmu_pages_lock; see 1331 * below for the details. 1332 * 1333 * Roots will remain in the list until their tdp_mmu_root_count 1334 * drops to zero, at which point the thread that decremented the 1335 * count to zero should removed the root from the list and clean 1336 * it up, freeing the root after an RCU grace period. 1337 */ 1338 struct list_head tdp_mmu_roots; 1339 1340 /* 1341 * Protects accesses to the following fields when the MMU lock 1342 * is held in read mode: 1343 * - tdp_mmu_roots (above) 1344 * - the link field of kvm_mmu_page structs used by the TDP MMU 1345 * - possible_nx_huge_pages[KVM_TDP_MMU]; 1346 * - the possible_nx_huge_page_link field of kvm_mmu_page structs used 1347 * by the TDP MMU 1348 * Because the lock is only taken within the MMU lock, strictly 1349 * speaking it is redundant to acquire this lock when the thread 1350 * holds the MMU lock in write mode. However it often simplifies 1351 * the code to do so. 1352 */ 1353 spinlock_t tdp_mmu_pages_lock; 1354 #endif /* CONFIG_X86_64 */ 1355 1356 /* 1357 * If set, at least one shadow root has been allocated. This flag 1358 * is used as one input when determining whether certain memslot 1359 * related allocations are necessary. 1360 */ 1361 bool shadow_root_allocated; 1362 1363 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING 1364 /* 1365 * If set, the VM has (or had) an external write tracking user, and 1366 * thus all write tracking metadata has been allocated, even if KVM 1367 * itself isn't using write tracking. 1368 */ 1369 bool external_write_tracking_enabled; 1370 #endif 1371 1372 #if IS_ENABLED(CONFIG_HYPERV) 1373 hpa_t hv_root_tdp; 1374 spinlock_t hv_root_tdp_lock; 1375 struct hv_partition_assist_pg *hv_pa_pg; 1376 #endif 1377 /* 1378 * VM-scope maximum vCPU ID. Used to determine the size of structures 1379 * that increase along with the maximum vCPU ID, in which case, using 1380 * the global KVM_MAX_VCPU_IDS may lead to significant memory waste. 1381 */ 1382 u32 max_vcpu_ids; 1383 1384 bool disable_nx_huge_pages; 1385 1386 /* 1387 * Memory caches used to allocate shadow pages when performing eager 1388 * page splitting. No need for a shadowed_info_cache since eager page 1389 * splitting only allocates direct shadow pages. 1390 * 1391 * Protected by kvm->slots_lock. 1392 */ 1393 struct kvm_mmu_memory_cache split_shadow_page_cache; 1394 struct kvm_mmu_memory_cache split_page_header_cache; 1395 1396 /* 1397 * Memory cache used to allocate pte_list_desc structs while splitting 1398 * huge pages. In the worst case, to split one huge page, 512 1399 * pte_list_desc structs are needed to add each lower level leaf sptep 1400 * to the rmap plus 1 to extend the parent_ptes rmap of the lower level 1401 * page table. 1402 * 1403 * Protected by kvm->slots_lock. 1404 */ 1405 #define SPLIT_DESC_CACHE_MIN_NR_OBJECTS (SPTE_ENT_PER_PAGE + 1) 1406 struct kvm_mmu_memory_cache split_desc_cache; 1407 1408 gfn_t gfn_direct_bits; 1409 1410 /* 1411 * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero 1412 * value indicates CPU dirty logging is unsupported or disabled in 1413 * current VM. 1414 */ 1415 int cpu_dirty_log_size; 1416 }; 1417 1418 struct kvm_vm_stat { 1419 struct kvm_vm_stat_generic generic; 1420 u64 mmu_shadow_zapped; 1421 u64 mmu_pte_write; 1422 u64 mmu_pde_zapped; 1423 u64 mmu_flooded; 1424 u64 mmu_recycled; 1425 u64 mmu_cache_miss; 1426 u64 mmu_unsync; 1427 u64 mmu_shadow_pages; 1428 union { 1429 struct { 1430 atomic64_t pages_4k; 1431 atomic64_t pages_2m; 1432 atomic64_t pages_1g; 1433 }; 1434 atomic64_t pages[KVM_NR_PAGE_SIZES]; 1435 }; 1436 u64 nx_lpage_splits; 1437 u64 max_mmu_page_hash_collisions; 1438 u64 max_mmu_rmap_size; 1439 }; 1440 1441 struct kvm_vcpu_stat { 1442 struct kvm_vcpu_stat_generic generic; 1443 u64 pf_taken; 1444 u64 pf_fixed; 1445 u64 pf_emulate; 1446 u64 pf_spurious; 1447 u64 pf_fast; 1448 u64 pf_mmio_spte_created; 1449 u64 pf_guest; 1450 u64 tlb_flush; 1451 u64 invlpg; 1452 1453 u64 exits; 1454 u64 io_exits; 1455 u64 mmio_exits; 1456 u64 signal_exits; 1457 u64 irq_window_exits; 1458 u64 nmi_window_exits; 1459 u64 l1d_flush; 1460 u64 halt_exits; 1461 u64 request_irq_exits; 1462 u64 irq_exits; 1463 u64 host_state_reload; 1464 u64 fpu_reload; 1465 u64 insn_emulation; 1466 u64 insn_emulation_fail; 1467 u64 hypercalls; 1468 u64 irq_injections; 1469 u64 nmi_injections; 1470 u64 req_event; 1471 u64 nested_run; 1472 u64 directed_yield_attempted; 1473 u64 directed_yield_successful; 1474 u64 preemption_reported; 1475 u64 preemption_other; 1476 u64 guest_mode; 1477 u64 notify_window_exits; 1478 }; 1479 1480 struct x86_instruction_info; 1481 1482 struct msr_data { 1483 bool host_initiated; 1484 u32 index; 1485 u64 data; 1486 }; 1487 1488 struct kvm_lapic_irq { 1489 u32 vector; 1490 u16 delivery_mode; 1491 u16 dest_mode; 1492 bool level; 1493 u16 trig_mode; 1494 u32 shorthand; 1495 u32 dest_id; 1496 bool msi_redir_hint; 1497 }; 1498 1499 enum kvm_x86_run_flags { 1500 KVM_RUN_FORCE_IMMEDIATE_EXIT = BIT(0), 1501 KVM_RUN_LOAD_GUEST_DR6 = BIT(1), 1502 KVM_RUN_LOAD_DEBUGCTL = BIT(2), 1503 }; 1504 1505 struct kvm_x86_ops { 1506 const char *name; 1507 1508 int (*check_processor_compatibility)(void); 1509 1510 int (*enable_virtualization_cpu)(void); 1511 void (*disable_virtualization_cpu)(void); 1512 cpu_emergency_virt_cb *emergency_disable_virtualization_cpu; 1513 1514 void (*hardware_unsetup)(void); 1515 bool (*has_emulated_msr)(struct kvm *kvm, u32 index); 1516 void (*vcpu_after_set_cpuid)(struct kvm_vcpu *vcpu); 1517 1518 unsigned int vm_size; 1519 int (*vm_init)(struct kvm *kvm); 1520 void (*vm_destroy)(struct kvm *kvm); 1521 void (*vm_pre_destroy)(struct kvm *kvm); 1522 1523 /* Create, but do not attach this VCPU */ 1524 int (*vcpu_precreate)(struct kvm *kvm); 1525 int (*vcpu_create)(struct kvm_vcpu *vcpu); 1526 void (*vcpu_free)(struct kvm_vcpu *vcpu); 1527 void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event); 1528 1529 void (*prepare_switch_to_guest)(struct kvm_vcpu *vcpu); 1530 void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); 1531 void (*vcpu_put)(struct kvm_vcpu *vcpu); 1532 1533 /* 1534 * Mask of DEBUGCTL bits that are owned by the host, i.e. that need to 1535 * match the host's value even while the guest is active. 1536 */ 1537 const u64 HOST_OWNED_DEBUGCTL; 1538 1539 void (*update_exception_bitmap)(struct kvm_vcpu *vcpu); 1540 int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); 1541 int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); 1542 u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); 1543 void (*get_segment)(struct kvm_vcpu *vcpu, 1544 struct kvm_segment *var, int seg); 1545 int (*get_cpl)(struct kvm_vcpu *vcpu); 1546 int (*get_cpl_no_cache)(struct kvm_vcpu *vcpu); 1547 void (*set_segment)(struct kvm_vcpu *vcpu, 1548 struct kvm_segment *var, int seg); 1549 void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); 1550 bool (*is_valid_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); 1551 void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); 1552 void (*post_set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); 1553 bool (*is_valid_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); 1554 void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); 1555 int (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); 1556 void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1557 void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1558 void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1559 void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); 1560 void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu); 1561 void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value); 1562 void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg); 1563 unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); 1564 void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); 1565 bool (*get_if_flag)(struct kvm_vcpu *vcpu); 1566 1567 void (*flush_tlb_all)(struct kvm_vcpu *vcpu); 1568 void (*flush_tlb_current)(struct kvm_vcpu *vcpu); 1569 #if IS_ENABLED(CONFIG_HYPERV) 1570 int (*flush_remote_tlbs)(struct kvm *kvm); 1571 int (*flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn, 1572 gfn_t nr_pages); 1573 #endif 1574 1575 /* 1576 * Flush any TLB entries associated with the given GVA. 1577 * Does not need to flush GPA->HPA mappings. 1578 * Can potentially get non-canonical addresses through INVLPGs, which 1579 * the implementation may choose to ignore if appropriate. 1580 */ 1581 void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full); 1582 1583 /* 1584 * Flush any TLB entries created by the guest. Like tlb_flush_gva(), 1585 * does not need to flush GPA->HPA mappings. 1586 */ 1587 void (*flush_tlb_guest)(struct kvm_vcpu *vcpu); 1588 1589 bool (*vcpu_needs_initialization)(struct kvm_vcpu *vcpu); 1590 enum exit_fastpath_completion (*vcpu_run)(struct kvm_vcpu *vcpu, 1591 u64 run_flags); 1592 int (*handle_exit)(struct kvm_vcpu *vcpu, 1593 enum exit_fastpath_completion exit_fastpath); 1594 int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); 1595 void (*update_emulated_instruction)(struct kvm_vcpu *vcpu); 1596 bool (*unhandleable_emulation_required)(struct kvm_vcpu *vcpu); 1597 1598 void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask); 1599 u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu); 1600 void (*patch_hypercall)(struct kvm_vcpu *vcpu, 1601 unsigned char *hypercall_addr); 1602 void (*inject_irq)(struct kvm_vcpu *vcpu, bool reinjected); 1603 void (*inject_nmi)(struct kvm_vcpu *vcpu); 1604 void (*inject_exception)(struct kvm_vcpu *vcpu); 1605 void (*cancel_injection)(struct kvm_vcpu *vcpu); 1606 int (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection); 1607 int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); 1608 bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); 1609 void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); 1610 /* Whether or not a virtual NMI is pending in hardware. */ 1611 bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu); 1612 /* 1613 * Attempt to pend a virtual NMI in hardware. Returns %true on success 1614 * to allow using static_call_ret0 as the fallback. 1615 */ 1616 bool (*set_vnmi_pending)(struct kvm_vcpu *vcpu); 1617 void (*enable_nmi_window)(struct kvm_vcpu *vcpu); 1618 void (*enable_irq_window)(struct kvm_vcpu *vcpu); 1619 void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr); 1620 1621 const bool x2apic_icr_is_split; 1622 const unsigned long required_apicv_inhibits; 1623 bool allow_apicv_in_x2apic_without_x2apic_virtualization; 1624 void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu); 1625 void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr); 1626 void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); 1627 void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); 1628 void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu); 1629 void (*deliver_interrupt)(struct kvm_lapic *apic, int delivery_mode, 1630 int trig_mode, int vector); 1631 int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu); 1632 int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); 1633 int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr); 1634 u8 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio); 1635 bool (*tdp_has_smep)(struct kvm *kvm); 1636 1637 void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, hpa_t root_hpa, 1638 int root_level); 1639 1640 /* Update the external page table from spte getting set. */ 1641 int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, u64 old_spte, 1642 u64 new_spte, enum pg_level level); 1643 1644 /* Update external page tables for page table about to be freed. */ 1645 void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp); 1646 1647 1648 bool (*has_wbinvd_exit)(void); 1649 1650 u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu); 1651 u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu); 1652 void (*write_tsc_offset)(struct kvm_vcpu *vcpu); 1653 void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu); 1654 1655 /* 1656 * Retrieve somewhat arbitrary exit/entry information. Intended to 1657 * be used only from within tracepoints or error paths. 1658 */ 1659 void (*get_exit_info)(struct kvm_vcpu *vcpu, u32 *reason, 1660 u64 *info1, u64 *info2, 1661 u32 *intr_info, u32 *error_code); 1662 1663 void (*get_entry_info)(struct kvm_vcpu *vcpu, 1664 u32 *intr_info, u32 *error_code); 1665 1666 int (*check_intercept)(struct kvm_vcpu *vcpu, 1667 struct x86_instruction_info *info, 1668 enum x86_intercept_stage stage, 1669 struct x86_exception *exception); 1670 void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); 1671 1672 void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); 1673 1674 void (*vcpu_blocking)(struct kvm_vcpu *vcpu); 1675 void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); 1676 1677 int (*pi_update_irte)(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, 1678 unsigned int host_irq, uint32_t guest_irq, 1679 struct kvm_vcpu *vcpu, u32 vector); 1680 void (*pi_start_bypass)(struct kvm *kvm); 1681 void (*apicv_pre_state_restore)(struct kvm_vcpu *vcpu); 1682 void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu); 1683 bool (*dy_apicv_has_pending_interrupt)(struct kvm_vcpu *vcpu); 1684 bool (*protected_apic_has_interrupt)(struct kvm_vcpu *vcpu); 1685 1686 int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 1687 bool *expired); 1688 void (*cancel_hv_timer)(struct kvm_vcpu *vcpu); 1689 1690 void (*setup_mce)(struct kvm_vcpu *vcpu); 1691 1692 #ifdef CONFIG_KVM_SMM 1693 int (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); 1694 int (*enter_smm)(struct kvm_vcpu *vcpu, union kvm_smram *smram); 1695 int (*leave_smm)(struct kvm_vcpu *vcpu, const union kvm_smram *smram); 1696 void (*enable_smi_window)(struct kvm_vcpu *vcpu); 1697 #endif 1698 1699 int (*dev_get_attr)(u32 group, u64 attr, u64 *val); 1700 int (*mem_enc_ioctl)(struct kvm *kvm, void __user *argp); 1701 int (*vcpu_mem_enc_ioctl)(struct kvm_vcpu *vcpu, void __user *argp); 1702 int (*vcpu_mem_enc_unlocked_ioctl)(struct kvm_vcpu *vcpu, void __user *argp); 1703 int (*mem_enc_register_region)(struct kvm *kvm, struct kvm_enc_region *argp); 1704 int (*mem_enc_unregister_region)(struct kvm *kvm, struct kvm_enc_region *argp); 1705 int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); 1706 int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); 1707 void (*guest_memory_reclaimed)(struct kvm *kvm); 1708 void (*reload_vmsa)(struct kvm_vcpu *vcpu); 1709 1710 int (*get_feature_msr)(u32 msr, u64 *data); 1711 1712 int (*check_emulate_instruction)(struct kvm_vcpu *vcpu, int emul_type, 1713 void *insn, int insn_len); 1714 1715 bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu); 1716 int (*enable_l2_tlb_flush)(struct kvm_vcpu *vcpu); 1717 1718 void (*migrate_timers)(struct kvm_vcpu *vcpu); 1719 void (*recalc_intercepts)(struct kvm_vcpu *vcpu); 1720 int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err); 1721 1722 void (*vcpu_deliver_sipi_vector)(struct kvm_vcpu *vcpu, u8 vector); 1723 1724 /* 1725 * Returns vCPU specific APICv inhibit reasons 1726 */ 1727 unsigned long (*vcpu_get_apicv_inhibit_reasons)(struct kvm_vcpu *vcpu); 1728 1729 gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); 1730 void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); 1731 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT 1732 int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, 1733 kvm_pfn_t nr_pages); 1734 #endif 1735 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM 1736 void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); 1737 #endif 1738 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE 1739 void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); 1740 #endif 1741 int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); 1742 }; 1743 1744 struct kvm_x86_nested_ops { 1745 bool enabled; 1746 1747 void (*leave_nested)(struct kvm_vcpu *vcpu); 1748 bool (*is_exception_vmexit)(struct kvm_vcpu *vcpu, u8 vector, 1749 u32 error_code); 1750 int (*check_events)(struct kvm_vcpu *vcpu); 1751 bool (*has_events)(struct kvm_vcpu *vcpu, bool for_injection); 1752 void (*triple_fault)(struct kvm_vcpu *vcpu); 1753 int (*get_state)(struct kvm_vcpu *vcpu, 1754 struct kvm_nested_state __user *user_kvm_nested_state, 1755 unsigned user_data_size); 1756 int (*set_state)(struct kvm_vcpu *vcpu, 1757 struct kvm_nested_state __user *user_kvm_nested_state, 1758 struct kvm_nested_state *kvm_state); 1759 bool (*get_nested_state_pages)(struct kvm_vcpu *vcpu); 1760 int (*write_log_dirty)(struct kvm_vcpu *vcpu, gpa_t l2_gpa); 1761 gpa_t (*translate_nested_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, 1762 u64 access, 1763 struct x86_exception *exception, 1764 u64 pte_access); 1765 1766 int (*enable_evmcs)(struct kvm_vcpu *vcpu, 1767 uint16_t *vmcs_version); 1768 uint16_t (*get_evmcs_version)(struct kvm_vcpu *vcpu); 1769 void (*hv_inject_synthetic_vmexit_post_tlb_flush)(struct kvm_vcpu *vcpu); 1770 }; 1771 1772 struct kvm_x86_init_ops { 1773 int (*hardware_setup)(void); 1774 unsigned int (*handle_intel_pt_intr)(void); 1775 1776 struct kvm_x86_ops *runtime_ops; 1777 struct kvm_pmu_ops *pmu_ops; 1778 struct kvm_x86_nested_ops *nested_ops; 1779 }; 1780 1781 struct kvm_arch_async_pf { 1782 u32 token; 1783 gfn_t gfn; 1784 unsigned long cr3; 1785 bool direct_map; 1786 u64 error_code; 1787 }; 1788 1789 extern bool __read_mostly allow_smaller_maxphyaddr; 1790 extern bool __read_mostly enable_apicv; 1791 extern bool __read_mostly enable_ipiv; 1792 extern bool __read_mostly enable_device_posted_irqs; 1793 extern struct kvm_x86_ops kvm_x86_ops; 1794 extern struct kvm_x86_nested_ops kvm_nested_ops __read_mostly; 1795 1796 #define kvm_x86_call(func) static_call(kvm_x86_##func) 1797 1798 #define KVM_X86_OP(func) \ 1799 DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func)); 1800 #define KVM_X86_OP_OPTIONAL KVM_X86_OP 1801 #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP 1802 #include <asm/kvm-x86-ops.h> 1803 1804 #define kvm_nested_call(func) static_call(kvm_x86_nested_##func) 1805 1806 #define KVM_X86_NESTED_OP(func) \ 1807 DECLARE_STATIC_CALL(kvm_x86_nested_##func, *(((struct kvm_x86_nested_ops *)0)->func)); 1808 #define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP 1809 #define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP 1810 #include <asm/kvm-x86-nested-ops.h> 1811 1812 #define __KVM_HAVE_ARCH_VM_ALLOC 1813 static inline struct kvm *kvm_arch_alloc_vm(void) 1814 { 1815 return kvzalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT); 1816 } 1817 1818 #define __KVM_HAVE_ARCH_VM_FREE 1819 void kvm_arch_free_vm(struct kvm *kvm); 1820 1821 #if IS_ENABLED(CONFIG_HYPERV) 1822 #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS 1823 static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm) 1824 { 1825 if (kvm_x86_ops.flush_remote_tlbs && 1826 !kvm_x86_call(flush_remote_tlbs)(kvm)) 1827 return 0; 1828 else 1829 return -ENOTSUPP; 1830 } 1831 1832 #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE 1833 static inline int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, 1834 u64 nr_pages) 1835 { 1836 if (!kvm_x86_ops.flush_remote_tlbs_range) 1837 return -EOPNOTSUPP; 1838 1839 return kvm_x86_call(flush_remote_tlbs_range)(kvm, gfn, nr_pages); 1840 } 1841 #endif /* CONFIG_HYPERV */ 1842 1843 enum kvm_intr_type { 1844 /* Values are arbitrary, but must be non-zero. */ 1845 KVM_HANDLING_IRQ = 1, 1846 KVM_HANDLING_NMI, 1847 }; 1848 1849 /* Enable perf NMI and timer modes to work, and minimise false positives. */ 1850 #define kvm_arch_pmi_in_guest(vcpu) \ 1851 ((vcpu) && (vcpu)->arch.handling_intr_from_guest && \ 1852 (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI))) 1853 1854 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 1855 #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) 1856 #endif 1857 1858 #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state) 1859 1860 #ifdef CONFIG_X86_64 1861 static inline unsigned long read_msr(unsigned long msr) 1862 { 1863 u64 value; 1864 1865 rdmsrq(msr, value); 1866 return value; 1867 } 1868 #endif 1869 1870 #define HF_GUEST_MASK (1 << 0) /* VCPU is in guest-mode */ 1871 1872 #ifdef CONFIG_KVM_SMM 1873 #define HF_SMM_MASK (1 << 1) 1874 #define HF_SMM_INSIDE_NMI_MASK (1 << 2) 1875 1876 # define KVM_MAX_NR_ADDRESS_SPACES 2 1877 /* SMM is currently unsupported for guests with private memory. */ 1878 # define kvm_arch_nr_memslot_as_ids(kvm) (kvm_arch_has_private_mem(kvm) ? 1 : 2) 1879 # define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0) 1880 # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm) 1881 #else 1882 # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0) 1883 #endif 1884 1885 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 1886 struct kvm_async_pf *work); 1887 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 1888 struct kvm_async_pf *work); 1889 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, 1890 struct kvm_async_pf *work); 1891 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu); 1892 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); 1893 1894 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 1895 { 1896 kvm_x86_call(vcpu_blocking)(vcpu); 1897 } 1898 1899 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 1900 { 1901 kvm_x86_call(vcpu_unblocking)(vcpu); 1902 } 1903 1904 static inline bool kvm_arch_has_irq_bypass(void) 1905 { 1906 return enable_device_posted_irqs; 1907 } 1908 1909 #endif /* _ASM_X86_KVM_HOST_H */ 1910