1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * derived from drivers/kvm/kvm_main.c 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright (C) 2008 Qumranet, Inc. 9 * Copyright IBM Corporation, 2008 10 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 11 * 12 * Authors: 13 * Avi Kivity <avi@qumranet.com> 14 * Yaniv Kamay <yaniv@qumranet.com> 15 * Amit Shah <amit.shah@qumranet.com> 16 * Ben-Ami Yassour <benami@il.ibm.com> 17 */ 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/kvm_host.h> 21 #include "irq.h" 22 #include "ioapic.h" 23 #include "mmu.h" 24 #include "i8254.h" 25 #include "tss.h" 26 #include "regs.h" 27 #include "kvm_emulate.h" 28 #include "mmu/page_track.h" 29 #include "x86.h" 30 #include "cpuid.h" 31 #include "pmu.h" 32 #include "hyperv.h" 33 #include "lapic.h" 34 #include "xen.h" 35 #include "smm.h" 36 37 #include <linux/clocksource.h> 38 #include <linux/timekeeping.h> 39 #include <linux/interrupt.h> 40 #include <linux/kvm.h> 41 #include <linux/fs.h> 42 #include <linux/vmalloc.h> 43 #include <linux/export.h> 44 #include <linux/moduleparam.h> 45 #include <linux/mman.h> 46 #include <linux/highmem.h> 47 #include <linux/iommu.h> 48 #include <linux/cpufreq.h> 49 #include <linux/user-return-notifier.h> 50 #include <linux/srcu.h> 51 #include <linux/slab.h> 52 #include <linux/perf_event.h> 53 #include <linux/uaccess.h> 54 #include <linux/hash.h> 55 #include <linux/pci.h> 56 #include <linux/timekeeper_internal.h> 57 #include <linux/pvclock_gtod.h> 58 #include <linux/kvm_irqfd.h> 59 #include <linux/irqbypass.h> 60 #include <linux/sched/stat.h> 61 #include <linux/sched/isolation.h> 62 #include <linux/mem_encrypt.h> 63 #include <linux/suspend.h> 64 #include <linux/smp.h> 65 #include <linux/units.h> 66 67 #include <trace/events/ipi.h> 68 #include <trace/events/kvm.h> 69 70 #include <asm/debugreg.h> 71 #include <asm/msr.h> 72 #include <asm/desc.h> 73 #include <asm/mce.h> 74 #include <asm/pkru.h> 75 #include <linux/kernel_stat.h> 76 #include <asm/fpu/api.h> 77 #include <asm/fpu/xcr.h> 78 #include <asm/fpu/xstate.h> 79 #include <asm/pvclock.h> 80 #include <asm/div64.h> 81 #include <asm/irq_remapping.h> 82 #include <asm/mshyperv.h> 83 #include <asm/hypervisor.h> 84 #include <asm/tlbflush.h> 85 #include <asm/emulate_prefix.h> 86 #include <asm/sgx.h> 87 #include <asm/virt.h> 88 89 #include <clocksource/hyperv_timer.h> 90 91 #define CREATE_TRACE_POINTS 92 #include "trace.h" 93 94 /* 95 * Note, kvm_caps fields should *never* have default values, all fields must be 96 * recomputed from scratch during vendor module load, e.g. to account for a 97 * vendor module being reloaded with different module parameters. 98 */ 99 struct kvm_caps kvm_caps __read_mostly; 100 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_caps); 101 102 struct kvm_host_values kvm_host __read_mostly; 103 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_host); 104 105 #define ERR_PTR_USR(e) ((void __user *)ERR_PTR(e)) 106 107 #define emul_to_vcpu(ctxt) \ 108 ((struct kvm_vcpu *)(ctxt)->vcpu) 109 110 /* 111 * KVM previously used a u32 field in kvm_run to indicate the hypercall was 112 * initiated from long mode. KVM now sets bit 0 to indicate long mode, but the 113 * remaining 31 lower bits must be 0 to preserve ABI. 114 */ 115 #define KVM_EXIT_HYPERCALL_MBZ GENMASK_ULL(31, 1) 116 #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) 117 118 #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE 119 120 #define KVM_GUESTDBG_VALID_MASK \ 121 (KVM_GUESTDBG_ENABLE | \ 122 KVM_GUESTDBG_SINGLESTEP | \ 123 KVM_GUESTDBG_USE_HW_BP | \ 124 KVM_GUESTDBG_USE_SW_BP | \ 125 KVM_GUESTDBG_INJECT_BP | \ 126 KVM_GUESTDBG_INJECT_DB | \ 127 KVM_GUESTDBG_BLOCKIRQ) 128 129 #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ 130 KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK | \ 131 KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \ 132 KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) 133 134 #define KVM_CLOCK_VALID_FLAGS \ 135 (KVM_CLOCK_TSC_STABLE | KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC) 136 137 #define KVM_X86_VALID_QUIRKS \ 138 (KVM_X86_QUIRK_LINT0_REENABLED | \ 139 KVM_X86_QUIRK_CD_NW_CLEARED | \ 140 KVM_X86_QUIRK_LAPIC_MMIO_HOLE | \ 141 KVM_X86_QUIRK_OUT_7E_INC_RIP | \ 142 KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT | \ 143 KVM_X86_QUIRK_FIX_HYPERCALL_INSN | \ 144 KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS | \ 145 KVM_X86_QUIRK_SLOT_ZAP_ALL | \ 146 KVM_X86_QUIRK_STUFF_FEATURE_MSRS | \ 147 KVM_X86_QUIRK_IGNORE_GUEST_PAT | \ 148 KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM | \ 149 KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT) 150 151 #define KVM_X86_CONDITIONAL_QUIRKS \ 152 (KVM_X86_QUIRK_CD_NW_CLEARED | \ 153 KVM_X86_QUIRK_IGNORE_GUEST_PAT) 154 155 #define KVM_BUS_LOCK_DETECTION_VALID_MODE (KVM_BUS_LOCK_DETECTION_OFF | \ 156 KVM_BUS_LOCK_DETECTION_EXIT) 157 158 #define KVM_X86_NOTIFY_VMEXIT_VALID_BITS (KVM_X86_NOTIFY_VMEXIT_ENABLED | \ 159 KVM_X86_NOTIFY_VMEXIT_USER) 160 161 static void process_nmi(struct kvm_vcpu *vcpu); 162 static void store_regs(struct kvm_vcpu *vcpu); 163 static int sync_regs(struct kvm_vcpu *vcpu); 164 165 static DEFINE_MUTEX(vendor_module_lock); 166 167 struct kvm_x86_ops kvm_x86_ops __read_mostly; 168 struct kvm_x86_nested_ops kvm_nested_ops __read_mostly; 169 170 #define KVM_X86_OP(func) \ 171 DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ 172 *(((struct kvm_x86_ops *)0)->func)); 173 #define KVM_X86_OP_OPTIONAL KVM_X86_OP 174 #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP 175 #include <asm/kvm-x86-ops.h> 176 EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits); 177 EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg); 178 EXPORT_STATIC_CALL_GPL(kvm_x86_get_cpl); 179 180 #define KVM_X86_NESTED_OP(func) \ 181 DEFINE_STATIC_CALL_NULL(kvm_x86_nested_##func, \ 182 *(((struct kvm_x86_nested_ops *)0)->func)); 183 #define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP 184 #define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP 185 #include <asm/kvm-x86-nested-ops.h> 186 187 unsigned int min_timer_period_us = 200; 188 module_param(min_timer_period_us, uint, 0644); 189 190 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */ 191 static u32 __read_mostly tsc_tolerance_ppm = 250; 192 module_param(tsc_tolerance_ppm, uint, 0644); 193 194 bool __read_mostly enable_vmware_backdoor = false; 195 module_param(enable_vmware_backdoor, bool, 0444); 196 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_vmware_backdoor); 197 198 /* 199 * Flags to manipulate forced emulation behavior (any non-zero value will 200 * enable forced emulation). 201 */ 202 #define KVM_FEP_CLEAR_RFLAGS_RF BIT(1) 203 static int __read_mostly force_emulation_prefix; 204 module_param(force_emulation_prefix, int, 0644); 205 206 int __read_mostly pi_inject_timer = -1; 207 module_param(pi_inject_timer, bint, 0644); 208 209 /* Enable/disable SMT_RSB bug mitigation */ 210 static bool __read_mostly mitigate_smt_rsb; 211 module_param(mitigate_smt_rsb, bool, 0444); 212 213 #define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ 214 | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ 215 | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ 216 | XFEATURE_MASK_PKRU | XFEATURE_MASK_XTILE) 217 218 #define XFEATURE_MASK_CET_ALL (XFEATURE_MASK_CET_USER | XFEATURE_MASK_CET_KERNEL) 219 /* 220 * Note, KVM supports exposing PT to the guest, but does not support context 221 * switching PT via XSTATE (KVM's PT virtualization relies on perf; swapping 222 * PT via guest XSTATE would clobber perf state), i.e. KVM doesn't support 223 * IA32_XSS[bit 8] (guests can/must use RDMSR/WRMSR to save/restore PT MSRs). 224 */ 225 #define KVM_SUPPORTED_XSS (XFEATURE_MASK_CET_ALL) 226 227 bool __read_mostly allow_smaller_maxphyaddr = 0; 228 EXPORT_SYMBOL_FOR_KVM_INTERNAL(allow_smaller_maxphyaddr); 229 230 bool __read_mostly enable_apicv = true; 231 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_apicv); 232 233 bool __read_mostly enable_ipiv = true; 234 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_ipiv); 235 236 bool __read_mostly enable_device_posted_irqs = true; 237 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_device_posted_irqs); 238 239 const struct kvm_stats_desc kvm_vm_stats_desc[] = { 240 KVM_GENERIC_VM_STATS(), 241 STATS_DESC_COUNTER(VM, mmu_shadow_zapped), 242 STATS_DESC_COUNTER(VM, mmu_pte_write), 243 STATS_DESC_COUNTER(VM, mmu_pde_zapped), 244 STATS_DESC_COUNTER(VM, mmu_flooded), 245 STATS_DESC_COUNTER(VM, mmu_recycled), 246 STATS_DESC_COUNTER(VM, mmu_cache_miss), 247 STATS_DESC_ICOUNTER(VM, mmu_unsync), 248 STATS_DESC_ICOUNTER(VM, mmu_shadow_pages), 249 STATS_DESC_ICOUNTER(VM, pages_4k), 250 STATS_DESC_ICOUNTER(VM, pages_2m), 251 STATS_DESC_ICOUNTER(VM, pages_1g), 252 STATS_DESC_ICOUNTER(VM, nx_lpage_splits), 253 STATS_DESC_PCOUNTER(VM, max_mmu_rmap_size), 254 STATS_DESC_PCOUNTER(VM, max_mmu_page_hash_collisions) 255 }; 256 257 const struct kvm_stats_header kvm_vm_stats_header = { 258 .name_size = KVM_STATS_NAME_SIZE, 259 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), 260 .id_offset = sizeof(struct kvm_stats_header), 261 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 262 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 263 sizeof(kvm_vm_stats_desc), 264 }; 265 266 const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { 267 KVM_GENERIC_VCPU_STATS(), 268 STATS_DESC_COUNTER(VCPU, pf_taken), 269 STATS_DESC_COUNTER(VCPU, pf_fixed), 270 STATS_DESC_COUNTER(VCPU, pf_emulate), 271 STATS_DESC_COUNTER(VCPU, pf_spurious), 272 STATS_DESC_COUNTER(VCPU, pf_fast), 273 STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created), 274 STATS_DESC_COUNTER(VCPU, pf_guest), 275 STATS_DESC_COUNTER(VCPU, tlb_flush), 276 STATS_DESC_COUNTER(VCPU, invlpg), 277 STATS_DESC_COUNTER(VCPU, exits), 278 STATS_DESC_COUNTER(VCPU, io_exits), 279 STATS_DESC_COUNTER(VCPU, mmio_exits), 280 STATS_DESC_COUNTER(VCPU, signal_exits), 281 STATS_DESC_COUNTER(VCPU, irq_window_exits), 282 STATS_DESC_COUNTER(VCPU, nmi_window_exits), 283 STATS_DESC_COUNTER(VCPU, l1d_flush), 284 STATS_DESC_COUNTER(VCPU, halt_exits), 285 STATS_DESC_COUNTER(VCPU, request_irq_exits), 286 STATS_DESC_COUNTER(VCPU, irq_exits), 287 STATS_DESC_COUNTER(VCPU, host_state_reload), 288 STATS_DESC_COUNTER(VCPU, fpu_reload), 289 STATS_DESC_COUNTER(VCPU, insn_emulation), 290 STATS_DESC_COUNTER(VCPU, insn_emulation_fail), 291 STATS_DESC_COUNTER(VCPU, hypercalls), 292 STATS_DESC_COUNTER(VCPU, irq_injections), 293 STATS_DESC_COUNTER(VCPU, nmi_injections), 294 STATS_DESC_COUNTER(VCPU, req_event), 295 STATS_DESC_COUNTER(VCPU, nested_run), 296 STATS_DESC_COUNTER(VCPU, directed_yield_attempted), 297 STATS_DESC_COUNTER(VCPU, directed_yield_successful), 298 STATS_DESC_COUNTER(VCPU, preemption_reported), 299 STATS_DESC_COUNTER(VCPU, preemption_other), 300 STATS_DESC_IBOOLEAN(VCPU, guest_mode), 301 STATS_DESC_COUNTER(VCPU, notify_window_exits), 302 }; 303 304 const struct kvm_stats_header kvm_vcpu_stats_header = { 305 .name_size = KVM_STATS_NAME_SIZE, 306 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 307 .id_offset = sizeof(struct kvm_stats_header), 308 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 309 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 310 sizeof(kvm_vcpu_stats_desc), 311 }; 312 313 static struct kmem_cache *x86_emulator_cache; 314 315 static struct kmem_cache *kvm_alloc_emulator_cache(void) 316 { 317 unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src); 318 unsigned int size = sizeof(struct x86_emulate_ctxt); 319 320 return kmem_cache_create_usercopy("x86_emulator", size, 321 __alignof__(struct x86_emulate_ctxt), 322 SLAB_ACCOUNT, useroffset, 323 size - useroffset, NULL); 324 } 325 326 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt); 327 328 /* 329 * Handle a fault on a hardware virtualization (VMX or SVM) instruction. 330 * 331 * Hardware virtualization extension instructions may fault if a reboot turns 332 * off virtualization while processes are running. Usually after catching the 333 * fault we just panic; during reboot instead the instruction is ignored. 334 */ 335 noinstr void kvm_spurious_fault(void) 336 { 337 /* Fault while not rebooting. We want the trace. */ 338 BUG_ON(!virt_rebooting); 339 } 340 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_spurious_fault); 341 342 #define EXCPT_BENIGN 0 343 #define EXCPT_CONTRIBUTORY 1 344 #define EXCPT_PF 2 345 346 static int exception_class(int vector) 347 { 348 switch (vector) { 349 case PF_VECTOR: 350 return EXCPT_PF; 351 case DE_VECTOR: 352 case TS_VECTOR: 353 case NP_VECTOR: 354 case SS_VECTOR: 355 case GP_VECTOR: 356 return EXCPT_CONTRIBUTORY; 357 default: 358 break; 359 } 360 return EXCPT_BENIGN; 361 } 362 363 #define EXCPT_FAULT 0 364 #define EXCPT_TRAP 1 365 #define EXCPT_ABORT 2 366 #define EXCPT_INTERRUPT 3 367 #define EXCPT_DB 4 368 369 static int exception_type(int vector) 370 { 371 unsigned int mask; 372 373 if (WARN_ON(vector > 31 || vector == NMI_VECTOR)) 374 return EXCPT_INTERRUPT; 375 376 mask = 1 << vector; 377 378 /* 379 * #DBs can be trap-like or fault-like, the caller must check other CPU 380 * state, e.g. DR6, to determine whether a #DB is a trap or fault. 381 */ 382 if (mask & (1 << DB_VECTOR)) 383 return EXCPT_DB; 384 385 if (mask & ((1 << BP_VECTOR) | (1 << OF_VECTOR))) 386 return EXCPT_TRAP; 387 388 if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR))) 389 return EXCPT_ABORT; 390 391 /* Reserved exceptions will result in fault */ 392 return EXCPT_FAULT; 393 } 394 395 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, 396 struct kvm_queued_exception *ex) 397 { 398 if (!ex->has_payload) 399 return; 400 401 switch (ex->vector) { 402 case DB_VECTOR: 403 /* 404 * DR6 is a mess. Reserved/unused bits are fixed-to-1, and so 405 * to maintain backwards compatibility with existing software, 406 * features that use previously-reserved bits have active-low 407 * semantics, i.e. clear the bit when the feature is present in 408 * the payload. 409 * 410 * Further complicating matters, some DR6 bits are preserved by 411 * hardware, while others are explicitly modified on every #DB. 412 * The trap bits are always set based on the payload, as is the 413 * RTM flag (but it's active low). All other bits are modified 414 * if and only if a relevant debug exception occurs, e.g. BD, 415 * BS, and BT are never cleared by hardware, and BLD is never 416 * set by hardware (when supported, excepting RESET). 417 * 418 * Lastly, the payload does NOT have active-low semantics, e.g. 419 * so that it's compatible VMX's pending debug exceptions and 420 * qualification fields, and to avoid bleeding the DR6 madness 421 * into other KVM code. 422 * 423 * To compute DR6: 424 * 425 * 1. "Reset" the bits that are modified on all #DBs 426 * 2. Clear active-low bits that are present in the payload. 427 * 3. Set active-high bits that are present in the payload. 428 * 4. Clear fixed-0 bits. 429 * 5. Set fixed-1 bits. 430 */ 431 vcpu->arch.dr6 &= ~DR_TRAP_BITS; 432 vcpu->arch.dr6 |= DR6_RTM; 433 vcpu->arch.dr6 &= ~(ex->payload & DR6_ACTIVE_LOW); 434 vcpu->arch.dr6 |= (ex->payload & ~DR6_ACTIVE_LOW); 435 436 /* 437 * The #DB payload is defined as compatible with the 'pending 438 * debug exceptions' field under VMX, not DR6. While bit 12 is 439 * defined in the 'pending debug exceptions' field (enabled 440 * breakpoint), it is reserved and must be zero in DR6. 441 */ 442 vcpu->arch.dr6 &= ~BIT(12); 443 vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); 444 break; 445 case PF_VECTOR: 446 vcpu->arch.cr2 = ex->payload; 447 break; 448 } 449 450 ex->has_payload = false; 451 ex->payload = 0; 452 } 453 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_deliver_exception_payload); 454 455 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector, 456 bool has_error_code, u32 error_code, 457 bool has_payload, unsigned long payload) 458 { 459 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 460 461 kvm_make_request(KVM_REQ_EVENT, vcpu); 462 463 ex->vector = vector; 464 ex->injected = false; 465 ex->pending = true; 466 ex->has_error_code = has_error_code; 467 ex->error_code = error_code; 468 ex->has_payload = has_payload; 469 ex->payload = payload; 470 } 471 472 static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr, 473 bool has_error, u32 error_code, 474 bool has_payload, unsigned long payload) 475 { 476 u32 prev_nr; 477 int class1, class2; 478 479 kvm_make_request(KVM_REQ_EVENT, vcpu); 480 481 /* 482 * If the exception is destined for L2, morph it to a VM-Exit if L1 483 * wants to intercept the exception. 484 */ 485 if (is_guest_mode(vcpu) && 486 kvm_nested_call(is_exception_vmexit)(vcpu, nr, error_code)) { 487 kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code, 488 has_payload, payload); 489 return; 490 } 491 492 if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) { 493 queue: 494 vcpu->arch.exception.pending = true; 495 vcpu->arch.exception.injected = false; 496 497 vcpu->arch.exception.has_error_code = has_error; 498 vcpu->arch.exception.vector = nr; 499 vcpu->arch.exception.error_code = error_code; 500 vcpu->arch.exception.has_payload = has_payload; 501 vcpu->arch.exception.payload = payload; 502 return; 503 } 504 505 /* to check exception */ 506 prev_nr = vcpu->arch.exception.vector; 507 if (prev_nr == DF_VECTOR) { 508 /* triple fault -> shutdown */ 509 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 510 return; 511 } 512 class1 = exception_class(prev_nr); 513 class2 = exception_class(nr); 514 if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) || 515 (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) { 516 /* 517 * Synthesize #DF. Clear the previously injected or pending 518 * exception so as not to incorrectly trigger shutdown. 519 */ 520 vcpu->arch.exception.injected = false; 521 vcpu->arch.exception.pending = false; 522 523 kvm_queue_exception_e(vcpu, DF_VECTOR, 0); 524 } else { 525 /* replace previous exception with a new one in a hope 526 that instruction re-execution will regenerate lost 527 exception */ 528 goto queue; 529 } 530 } 531 532 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) 533 { 534 kvm_multiple_exception(vcpu, nr, false, 0, false, 0); 535 } 536 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_queue_exception); 537 538 539 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, 540 unsigned long payload) 541 { 542 kvm_multiple_exception(vcpu, nr, false, 0, true, payload); 543 } 544 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_queue_exception_p); 545 546 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr, 547 u32 error_code, unsigned long payload) 548 { 549 kvm_multiple_exception(vcpu, nr, true, error_code, true, payload); 550 } 551 552 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr, 553 bool has_error_code, u32 error_code) 554 { 555 556 /* 557 * On VM-Entry, an exception can be pending if and only if event 558 * injection was blocked by nested_run_pending. In that case, however, 559 * vcpu_enter_guest() requests an immediate exit, and the guest 560 * shouldn't proceed far enough to need reinjection. 561 */ 562 WARN_ON_ONCE(kvm_is_exception_pending(vcpu)); 563 564 /* 565 * Do not check for interception when injecting an event for L2, as the 566 * exception was checked for intercept when it was original queued, and 567 * re-checking is incorrect if _L1_ injected the exception, in which 568 * case it's exempt from interception. 569 */ 570 kvm_make_request(KVM_REQ_EVENT, vcpu); 571 572 vcpu->arch.exception.injected = true; 573 vcpu->arch.exception.has_error_code = has_error_code; 574 vcpu->arch.exception.vector = nr; 575 vcpu->arch.exception.error_code = error_code; 576 vcpu->arch.exception.has_payload = false; 577 vcpu->arch.exception.payload = 0; 578 } 579 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_requeue_exception); 580 581 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) 582 { 583 if (err) 584 kvm_inject_gp(vcpu, 0); 585 else 586 return kvm_skip_emulated_instruction(vcpu); 587 588 return 1; 589 } 590 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_complete_insn_gp); 591 592 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault, 593 bool from_hardware) 594 { 595 ++vcpu->stat.pf_guest; 596 597 /* 598 * Async #PF in L2 is always forwarded to L1 as a VM-Exit regardless of 599 * whether or not L1 wants to intercept "regular" #PF. 600 */ 601 if (is_guest_mode(vcpu) && fault->async_page_fault) 602 kvm_queue_exception_vmexit(vcpu, PF_VECTOR, 603 true, fault->error_code, 604 true, fault->address); 605 else 606 kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code, 607 fault->address); 608 } 609 610 void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, 611 struct x86_exception *fault, 612 bool from_hardware) 613 { 614 struct kvm_pagewalk *fault_walk; 615 616 WARN_ON_ONCE(fault->vector != PF_VECTOR); 617 618 fault_walk = fault->nested_page_fault ? &vcpu->arch.ngpa_walk : 619 &vcpu->arch.gva_walk; 620 621 /* 622 * Invalidate the TLB entry for the faulting address, if it exists, 623 * else the access will fault indefinitely (and to emulate hardware). 624 */ 625 if ((fault->error_code & PFERR_PRESENT_MASK) && 626 !(fault->error_code & PFERR_RSVD_MASK)) 627 kvm_mmu_invalidate_addr(vcpu, fault_walk, fault->address, 628 KVM_MMU_ROOT_CURRENT); 629 630 fault_walk->inject_page_fault(vcpu, fault, from_hardware); 631 } 632 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_inject_emulated_page_fault); 633 634 void kvm_inject_nmi(struct kvm_vcpu *vcpu) 635 { 636 atomic_inc(&vcpu->arch.nmi_queued); 637 kvm_make_request(KVM_REQ_NMI, vcpu); 638 } 639 640 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) 641 { 642 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0); 643 } 644 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_queue_exception_e); 645 646 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) 647 { 648 if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE)) 649 return true; 650 651 kvm_queue_exception(vcpu, UD_VECTOR); 652 return false; 653 } 654 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr); 655 656 static void kvm_load_xfeatures(struct kvm_vcpu *vcpu, bool load_guest) 657 { 658 if (vcpu->arch.guest_state_protected) 659 return; 660 661 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) 662 return; 663 664 if (vcpu->arch.xcr0 != kvm_host.xcr0) 665 xsetbv(XCR_XFEATURE_ENABLED_MASK, 666 load_guest ? vcpu->arch.xcr0 : kvm_host.xcr0); 667 668 if (guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) && 669 vcpu->arch.ia32_xss != kvm_host.xss) 670 wrmsrq(MSR_IA32_XSS, load_guest ? vcpu->arch.ia32_xss : kvm_host.xss); 671 } 672 673 static void kvm_load_guest_pkru(struct kvm_vcpu *vcpu) 674 { 675 if (vcpu->arch.guest_state_protected) 676 return; 677 678 if (cpu_feature_enabled(X86_FEATURE_PKU) && 679 vcpu->arch.pkru != vcpu->arch.host_pkru && 680 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || 681 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) 682 wrpkru(vcpu->arch.pkru); 683 } 684 685 static void kvm_load_host_pkru(struct kvm_vcpu *vcpu) 686 { 687 if (vcpu->arch.guest_state_protected) 688 return; 689 690 if (cpu_feature_enabled(X86_FEATURE_PKU) && 691 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || 692 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) { 693 vcpu->arch.pkru = rdpkru(); 694 if (vcpu->arch.pkru != vcpu->arch.host_pkru) 695 wrpkru(vcpu->arch.host_pkru); 696 } 697 } 698 699 int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) 700 { 701 u64 xcr0 = xcr; 702 u64 old_xcr0 = vcpu->arch.xcr0; 703 u64 valid_bits; 704 705 /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now */ 706 if (index != XCR_XFEATURE_ENABLED_MASK) 707 return 1; 708 if (!(xcr0 & XFEATURE_MASK_FP)) 709 return 1; 710 if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE)) 711 return 1; 712 713 /* 714 * Do not allow the guest to set bits that we do not support 715 * saving. However, xcr0 bit 0 is always set, even if the 716 * emulated CPU does not support XSAVE (see kvm_vcpu_reset()). 717 */ 718 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP; 719 if (xcr0 & ~valid_bits) 720 return 1; 721 722 if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) != 723 (!(xcr0 & XFEATURE_MASK_BNDCSR))) 724 return 1; 725 726 if (xcr0 & XFEATURE_MASK_AVX512) { 727 if (!(xcr0 & XFEATURE_MASK_YMM)) 728 return 1; 729 if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512) 730 return 1; 731 } 732 733 if ((xcr0 & XFEATURE_MASK_XTILE) && 734 ((xcr0 & XFEATURE_MASK_XTILE) != XFEATURE_MASK_XTILE)) 735 return 1; 736 737 vcpu->arch.xcr0 = xcr0; 738 739 if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND) 740 vcpu->arch.cpuid_dynamic_bits_dirty = true; 741 return 0; 742 } 743 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_set_xcr); 744 745 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu) 746 { 747 /* Note, #UD due to CR4.OSXSAVE=0 has priority over the intercept. */ 748 if (kvm_x86_call(get_cpl)(vcpu) != 0 || 749 __kvm_set_xcr(vcpu, kvm_ecx_read(vcpu), kvm_read_edx_eax(vcpu))) { 750 kvm_inject_gp(vcpu, 0); 751 return 1; 752 } 753 754 return kvm_skip_emulated_instruction(vcpu); 755 } 756 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_xsetbv); 757 758 void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) 759 { 760 struct kvm_mmu *mmu = vcpu->arch.mmu; 761 unsigned long roots_to_free = 0; 762 int i; 763 764 /* 765 * MOV CR3 and INVPCID are usually not intercepted when using TDP, but 766 * this is reachable when running EPT=1 and unrestricted_guest=0, and 767 * also via the emulator. KVM's TDP page tables are not in the scope of 768 * the invalidation, but the guest's TLB entries need to be flushed as 769 * the CPU may have cached entries in its TLB for the target PCID. 770 */ 771 if (unlikely(tdp_enabled)) { 772 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 773 return; 774 } 775 776 /* 777 * If neither the current CR3 nor any of the prev_roots use the given 778 * PCID, then nothing needs to be done here because a resync will 779 * happen anyway before switching to any other CR3. 780 */ 781 if (kvm_get_active_pcid(vcpu) == pcid) { 782 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 783 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 784 } 785 786 /* 787 * If PCID is disabled, there is no need to free prev_roots even if the 788 * PCIDs for them are also 0, because MOV to CR3 always flushes the TLB 789 * with PCIDE=0. 790 */ 791 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) 792 return; 793 794 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 795 if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid) 796 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 797 798 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); 799 } 800 801 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) 802 { 803 u32 pmc = kvm_ecx_read(vcpu); 804 u64 data; 805 806 if (kvm_pmu_rdpmc(vcpu, pmc, &data)) { 807 kvm_inject_gp(vcpu, 0); 808 return 1; 809 } 810 811 kvm_eax_write(vcpu, data); 812 kvm_edx_write(vcpu, data >> 32); 813 return kvm_skip_emulated_instruction(vcpu); 814 } 815 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdpmc); 816 817 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu) 818 { 819 return kvm_skip_emulated_instruction(vcpu); 820 } 821 822 int kvm_emulate_invd(struct kvm_vcpu *vcpu) 823 { 824 /* Treat an INVD instruction as a NOP and just skip it. */ 825 return kvm_emulate_as_nop(vcpu); 826 } 827 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_invd); 828 829 fastpath_t handle_fastpath_invd(struct kvm_vcpu *vcpu) 830 { 831 if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu)) 832 return EXIT_FASTPATH_NONE; 833 834 if (!kvm_emulate_invd(vcpu)) 835 return EXIT_FASTPATH_EXIT_USERSPACE; 836 837 return EXIT_FASTPATH_REENTER_GUEST; 838 } 839 EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_invd); 840 841 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu) 842 { 843 kvm_queue_exception(vcpu, UD_VECTOR); 844 return 1; 845 } 846 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_invalid_op); 847 848 849 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn) 850 { 851 bool enabled; 852 853 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS)) 854 goto emulate_as_nop; 855 856 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT)) 857 enabled = guest_cpu_cap_has(vcpu, X86_FEATURE_MWAIT); 858 else 859 enabled = vcpu->arch.ia32_misc_enable_msr & MSR_IA32_MISC_ENABLE_MWAIT; 860 861 if (!enabled) 862 return kvm_handle_invalid_op(vcpu); 863 864 emulate_as_nop: 865 pr_warn_once("%s instruction emulated as NOP!\n", insn); 866 return kvm_emulate_as_nop(vcpu); 867 } 868 int kvm_emulate_mwait(struct kvm_vcpu *vcpu) 869 { 870 return kvm_emulate_monitor_mwait(vcpu, "MWAIT"); 871 } 872 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_mwait); 873 874 int kvm_emulate_monitor(struct kvm_vcpu *vcpu) 875 { 876 return kvm_emulate_monitor_mwait(vcpu, "MONITOR"); 877 } 878 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_monitor); 879 880 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu) 881 { 882 xfer_to_guest_mode_prepare(); 883 884 return READ_ONCE(vcpu->mode) == EXITING_GUEST_MODE || 885 kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending(); 886 } 887 888 #ifdef CONFIG_X86_64 889 struct pvclock_clock { 890 int vclock_mode; 891 u64 cycle_last; 892 u64 mask; 893 u32 mult; 894 u32 shift; 895 u64 base_cycles; 896 u64 offset; 897 }; 898 899 struct pvclock_gtod_data { 900 seqcount_t seq; 901 902 struct pvclock_clock clock; /* extract of a clocksource struct */ 903 struct pvclock_clock raw_clock; /* extract of a clocksource struct */ 904 905 ktime_t offs_boot; 906 u64 wall_time_sec; 907 }; 908 909 static struct pvclock_gtod_data pvclock_gtod_data; 910 911 static void update_pvclock_gtod(struct timekeeper *tk) 912 { 913 struct pvclock_gtod_data *vdata = &pvclock_gtod_data; 914 915 write_seqcount_begin(&vdata->seq); 916 917 /* copy pvclock gtod data */ 918 vdata->clock.vclock_mode = tk->tkr_mono.clock->vdso_clock_mode; 919 vdata->clock.cycle_last = tk->tkr_mono.cycle_last; 920 vdata->clock.mask = tk->tkr_mono.mask; 921 vdata->clock.mult = tk->tkr_mono.mult; 922 vdata->clock.shift = tk->tkr_mono.shift; 923 vdata->clock.base_cycles = tk->tkr_mono.xtime_nsec; 924 vdata->clock.offset = tk->tkr_mono.base; 925 926 vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->vdso_clock_mode; 927 vdata->raw_clock.cycle_last = tk->tkr_raw.cycle_last; 928 vdata->raw_clock.mask = tk->tkr_raw.mask; 929 vdata->raw_clock.mult = tk->tkr_raw.mult; 930 vdata->raw_clock.shift = tk->tkr_raw.shift; 931 vdata->raw_clock.base_cycles = tk->tkr_raw.xtime_nsec; 932 vdata->raw_clock.offset = tk->tkr_raw.base; 933 934 vdata->wall_time_sec = tk->xtime_sec; 935 936 vdata->offs_boot = tk->offs_boot; 937 938 write_seqcount_end(&vdata->seq); 939 } 940 #endif 941 942 static s64 get_kvmclock_base_ns(void) 943 { 944 /* Count up from boot time, but with the frequency of the raw clock. */ 945 return ktime_to_ns(ktime_mono_to_any(ktime_get_raw(), TK_OFFS_BOOT)); 946 } 947 948 static uint32_t div_frac(uint32_t dividend, uint32_t divisor) 949 { 950 do_shl32_div32(dividend, divisor); 951 return dividend; 952 } 953 954 static void kvm_get_time_scale(u64 scaled_hz, u64 base_hz, 955 s8 *pshift, u32 *pmultiplier) 956 { 957 u64 scaled_hz_u64 = scaled_hz; 958 s32 shift = 0; 959 u64 base_hz_u64; 960 u32 base32; 961 962 /* 963 * This function calculates a fixed-point multiplier and shift such 964 * that: 965 * time_ns = (tsc_cycles << shift) * multiplier >> 32 966 * 967 * Where tsc_cycles tick at base_hz, and time_ns should count at 968 * scaled_hz (typically NSEC_PER_SEC for a TSC→nanoseconds conversion). 969 * 970 * The multiplier is: (scaled_hz << 32) / base_hz, adjusted by shift 971 * to keep everything in range. 972 */ 973 974 base_hz_u64 = base_hz; 975 976 /* 977 * Start by shifting base_hz right until it fits in 32 bits, and 978 * is lower than double the target rate. This introduces a negative 979 * shift value which would result in pvclock_scale_delta() shifting 980 * the actual tick count right before performing the multiplication. 981 */ 982 while (base_hz_u64 > scaled_hz_u64 * 2 || base_hz_u64 >> 32) { 983 base_hz_u64 >>= 1; 984 shift--; 985 } 986 987 /* Now the shifted base_hz fits in 32 bits. */ 988 base32 = (u32)base_hz_u64; 989 990 /* 991 * Next, shift scaled_hz right until it fits in 32 bits, and ensure 992 * that the shifted base_hz is strictly larger (so that the result of the 993 * final division also fits in 32 bits). 994 */ 995 while (base32 <= scaled_hz_u64 || scaled_hz_u64 >> 32) { 996 if (scaled_hz_u64 >> 32 || base32 & BIT(31)) 997 scaled_hz_u64 >>= 1; 998 else 999 base32 <<= 1; 1000 shift++; 1001 } 1002 1003 *pshift = shift; 1004 *pmultiplier = div_frac(scaled_hz_u64, base32); 1005 } 1006 1007 #ifdef CONFIG_X86_64 1008 static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0); 1009 #endif 1010 1011 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz); 1012 static unsigned long max_tsc_khz; 1013 1014 static u32 adjust_tsc_khz(u32 khz, s32 ppm) 1015 { 1016 u64 v = (u64)khz * (1000000 + ppm); 1017 do_div(v, 1000000); 1018 return v; 1019 } 1020 1021 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier); 1022 1023 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale) 1024 { 1025 u64 ratio; 1026 1027 /* Guest TSC same frequency as host TSC? */ 1028 if (!scale) { 1029 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio); 1030 return 0; 1031 } 1032 1033 /* TSC scaling supported? */ 1034 if (!kvm_caps.has_tsc_control) { 1035 if (user_tsc_khz > tsc_khz) { 1036 vcpu->arch.tsc_catchup = 1; 1037 vcpu->arch.tsc_always_catchup = 1; 1038 return 0; 1039 } else { 1040 pr_warn_ratelimited("user requested TSC rate below hardware speed\n"); 1041 return -1; 1042 } 1043 } 1044 1045 /* TSC scaling required - calculate ratio */ 1046 ratio = mul_u64_u32_div(1ULL << kvm_caps.tsc_scaling_ratio_frac_bits, 1047 user_tsc_khz, tsc_khz); 1048 1049 if (ratio == 0 || ratio >= kvm_caps.max_tsc_scaling_ratio) { 1050 pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n", 1051 user_tsc_khz); 1052 return -1; 1053 } 1054 1055 kvm_vcpu_write_tsc_multiplier(vcpu, ratio); 1056 return 0; 1057 } 1058 1059 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz) 1060 { 1061 u32 thresh_lo, thresh_hi; 1062 int use_scaling = 0; 1063 1064 /* tsc_khz can be zero if TSC calibration fails */ 1065 if (user_tsc_khz == 0) { 1066 /* set tsc_scaling_ratio to a safe value */ 1067 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio); 1068 return -1; 1069 } 1070 1071 /* Compute a scale to convert nanoseconds in TSC cycles */ 1072 kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC, 1073 &vcpu->arch.virtual_tsc_shift, 1074 &vcpu->arch.virtual_tsc_mult); 1075 vcpu->arch.virtual_tsc_khz = user_tsc_khz; 1076 1077 /* 1078 * Compute the variation in TSC rate which is acceptable 1079 * within the range of tolerance and decide if the 1080 * rate being applied is within that bounds of the hardware 1081 * rate. If so, no scaling or compensation need be done. 1082 */ 1083 thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm); 1084 thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm); 1085 if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) { 1086 pr_debug("requested TSC rate %u falls outside tolerance [%u,%u]\n", 1087 user_tsc_khz, thresh_lo, thresh_hi); 1088 use_scaling = 1; 1089 } 1090 return set_tsc_khz(vcpu, user_tsc_khz, use_scaling); 1091 } 1092 1093 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) 1094 { 1095 s64 delta_ns = kernel_ns - vcpu->arch.this_tsc_nsec; 1096 u64 tsc; 1097 1098 /* Handle negative deltas gracefully (master clock ref may be earlier) */ 1099 tsc = pvclock_scale_delta(abs(delta_ns), 1100 vcpu->arch.virtual_tsc_mult, 1101 vcpu->arch.virtual_tsc_shift); 1102 1103 return vcpu->arch.this_tsc_write + (delta_ns >= 0 ? tsc : -tsc); 1104 } 1105 1106 #ifdef CONFIG_X86_64 1107 static inline bool gtod_is_based_on_tsc(int mode) 1108 { 1109 return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK; 1110 } 1111 #endif 1112 1113 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation) 1114 { 1115 #ifdef CONFIG_X86_64 1116 struct kvm_arch *ka = &vcpu->kvm->arch; 1117 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 1118 1119 /* 1120 * To use the masterclock, the host clocksource must be based on TSC 1121 * and all vCPUs must have matching TSCs. Note, the count for matching 1122 * vCPUs doesn't include the reference vCPU, hence "+1". 1123 */ 1124 bool use_master_clock = (ka->nr_vcpus_matched_tsc + 1 == 1125 atomic_read(&vcpu->kvm->online_vcpus)) && 1126 gtod_is_based_on_tsc(gtod->clock.vclock_mode); 1127 1128 /* 1129 * Request a masterclock update if the masterclock needs to be toggled 1130 * on/off, or when starting a new generation and the masterclock is 1131 * enabled (compute_guest_tsc() requires the masterclock snapshot to be 1132 * taken _after_ the new generation is created). 1133 */ 1134 if ((ka->use_master_clock && new_generation) || 1135 (ka->use_master_clock != use_master_clock)) 1136 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 1137 1138 trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc, 1139 atomic_read(&vcpu->kvm->online_vcpus), 1140 ka->use_master_clock, gtod->clock.vclock_mode); 1141 #endif 1142 } 1143 1144 /* 1145 * Multiply tsc by a fixed point number represented by ratio. 1146 * 1147 * The most significant 64-N bits (mult) of ratio represent the 1148 * integral part of the fixed point number; the remaining N bits 1149 * (frac) represent the fractional part, ie. ratio represents a fixed 1150 * point number (mult + frac * 2^(-N)). 1151 * 1152 * N equals to kvm_caps.tsc_scaling_ratio_frac_bits. 1153 */ 1154 static inline u64 __scale_tsc(u64 ratio, u64 tsc) 1155 { 1156 return mul_u64_u64_shr(tsc, ratio, kvm_caps.tsc_scaling_ratio_frac_bits); 1157 } 1158 1159 u64 kvm_scale_tsc(u64 tsc, u64 ratio) 1160 { 1161 u64 _tsc = tsc; 1162 1163 if (ratio != kvm_caps.default_tsc_scaling_ratio) 1164 _tsc = __scale_tsc(ratio, tsc); 1165 1166 return _tsc; 1167 } 1168 1169 u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, 1170 u64 target_tsc) 1171 { 1172 u64 tsc; 1173 1174 tsc = kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); 1175 1176 return target_tsc - tsc; 1177 } 1178 1179 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) 1180 { 1181 return vcpu->arch.l1_tsc_offset + 1182 kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); 1183 } 1184 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_l1_tsc); 1185 1186 u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier) 1187 { 1188 u64 nested_offset; 1189 1190 if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio) 1191 nested_offset = l1_offset; 1192 else 1193 nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier, 1194 kvm_caps.tsc_scaling_ratio_frac_bits); 1195 1196 nested_offset += l2_offset; 1197 return nested_offset; 1198 } 1199 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_offset); 1200 1201 u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) 1202 { 1203 u8 frac_bits = kvm_caps.tsc_scaling_ratio_frac_bits; 1204 u64 nested_multiplier; 1205 1206 if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio) 1207 return l1_multiplier; 1208 1209 /* 1210 * The shift is fixed on both AMD and Intel, and operates on a 64-bit 1211 * value. I.e. a shift greater than 63 is completely nonsensical. 1212 */ 1213 if (WARN_ON_ONCE(frac_bits > 63)) 1214 return l1_multiplier; 1215 1216 /* 1217 * If the resulting multiplier can't be programmed into hardware, run 1218 * L2 at the minimum/maximum frequency supported by hardware, i.e. 1219 * saturate L2's frequency on both sides. Because L2's frequency needs 1220 * to be distilled down to a single multiplier to get from: 1221 * 1222 * L2 = (((L0 * L1_mult) >> frac) * L2_mult) >> frac) 1223 * 1224 * to: 1225 * 1226 * L2 = (L0 * mult) >> frac 1227 * 1228 * very small/large L1 and L2 multipliers can underflow/overflow the 1229 * minimum/maximum multiplier supported by hardware when combined into 1230 * a single value. 1231 * 1232 * Manually check for the case where the result would overflow a u64, 1233 * i.e. if the multiplier would be silently truncated before the "too 1234 * large" check. Avoid doing the multiply twice in the common case 1235 * where the compiler natively supports 128-bit values. 1236 */ 1237 #ifdef CONFIG_ARCH_SUPPORTS_INT128 1238 unsigned __int128 m = (unsigned __int128)l1_multiplier * l2_multiplier; 1239 1240 if (m >> (64 + frac_bits)) 1241 return kvm_caps.max_tsc_scaling_ratio; 1242 1243 nested_multiplier = m >> frac_bits; 1244 #else 1245 if (mul_u64_u64_shr(l1_multiplier, l2_multiplier, 64 + frac_bits)) 1246 return kvm_caps.max_tsc_scaling_ratio; 1247 1248 nested_multiplier = mul_u64_u64_shr(l1_multiplier, l2_multiplier, frac_bits); 1249 #endif 1250 if (nested_multiplier > kvm_caps.max_tsc_scaling_ratio) 1251 return kvm_caps.max_tsc_scaling_ratio; 1252 1253 /* The minimum multiplier is '1' on both AMD and Intel. */ 1254 return nested_multiplier ?: 1; 1255 } 1256 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_multiplier); 1257 1258 void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset) 1259 { 1260 if (vcpu->arch.guest_tsc_protected) 1261 return; 1262 1263 trace_kvm_write_tsc_offset(vcpu->vcpu_id, 1264 vcpu->arch.l1_tsc_offset, 1265 l1_offset); 1266 1267 vcpu->arch.l1_tsc_offset = l1_offset; 1268 1269 /* 1270 * If we are here because L1 chose not to trap WRMSR to TSC then 1271 * according to the spec this should set L1's TSC (as opposed to 1272 * setting L1's offset for L2). 1273 */ 1274 if (is_guest_mode(vcpu)) 1275 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 1276 l1_offset, 1277 kvm_x86_call(get_l2_tsc_offset)(vcpu), 1278 kvm_x86_call(get_l2_tsc_multiplier)(vcpu)); 1279 else 1280 vcpu->arch.tsc_offset = l1_offset; 1281 1282 kvm_x86_call(write_tsc_offset)(vcpu); 1283 } 1284 1285 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier) 1286 { 1287 vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier; 1288 1289 /* Userspace is changing the multiplier while L2 is active */ 1290 if (is_guest_mode(vcpu)) 1291 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( 1292 l1_multiplier, 1293 kvm_x86_call(get_l2_tsc_multiplier)(vcpu)); 1294 else 1295 vcpu->arch.tsc_scaling_ratio = l1_multiplier; 1296 1297 if (kvm_caps.has_tsc_control) 1298 kvm_x86_call(write_tsc_multiplier)(vcpu); 1299 } 1300 1301 static inline bool kvm_check_tsc_unstable(void) 1302 { 1303 #ifdef CONFIG_X86_64 1304 /* 1305 * TSC is marked unstable when we're running on Hyper-V, 1306 * 'TSC page' clocksource is good. 1307 */ 1308 if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK) 1309 return false; 1310 #endif 1311 return check_tsc_unstable(); 1312 } 1313 1314 /* 1315 * Infers attempts to synchronize the guest's tsc from host writes. Sets the 1316 * offset for the vcpu and tracks the TSC matching generation that the vcpu 1317 * participates in. 1318 */ 1319 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, 1320 u64 ns, bool matched, bool user_set_tsc) 1321 { 1322 struct kvm *kvm = vcpu->kvm; 1323 1324 lockdep_assert_held(&kvm->arch.tsc_write_lock); 1325 1326 if (vcpu->arch.guest_tsc_protected) 1327 return; 1328 1329 if (user_set_tsc) 1330 vcpu->kvm->arch.user_set_tsc = true; 1331 1332 /* 1333 * We also track th most recent recorded KHZ, write and time to 1334 * allow the matching interval to be extended at each write. 1335 */ 1336 kvm->arch.last_tsc_nsec = ns; 1337 kvm->arch.last_tsc_write = tsc; 1338 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; 1339 kvm->arch.last_tsc_offset = offset; 1340 kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 1341 1342 vcpu->arch.last_guest_tsc = tsc; 1343 1344 kvm_vcpu_write_tsc_offset(vcpu, offset); 1345 1346 if (!matched) { 1347 /* 1348 * We split periods of matched TSC writes into generations. 1349 * For each generation, we track the original measured 1350 * nanosecond time, offset, and write, so if TSCs are in 1351 * sync, we can match exact offset, and if not, we can match 1352 * exact software computation in compute_guest_tsc() 1353 * 1354 * These values are tracked in kvm->arch.cur_xxx variables. 1355 */ 1356 kvm->arch.cur_tsc_generation++; 1357 kvm->arch.cur_tsc_nsec = ns; 1358 kvm->arch.cur_tsc_write = tsc; 1359 kvm->arch.cur_tsc_offset = offset; 1360 kvm->arch.nr_vcpus_matched_tsc = 0; 1361 } else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) { 1362 kvm->arch.nr_vcpus_matched_tsc++; 1363 } 1364 1365 /* Keep track of which generation this VCPU has synchronized to */ 1366 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation; 1367 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec; 1368 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write; 1369 1370 kvm_track_tsc_matching(vcpu, !matched); 1371 } 1372 1373 void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) 1374 { 1375 u64 data = user_value ? *user_value : 0; 1376 struct kvm *kvm = vcpu->kvm; 1377 u64 offset, ns, elapsed; 1378 unsigned long flags; 1379 bool matched = false; 1380 bool synchronizing = false; 1381 1382 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 1383 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); 1384 ns = get_kvmclock_base_ns(); 1385 elapsed = ns - kvm->arch.last_tsc_nsec; 1386 1387 if (vcpu->arch.virtual_tsc_khz) { 1388 if (data == 0) { 1389 /* 1390 * Force synchronization when creating a vCPU, or when 1391 * userspace explicitly writes a zero value. 1392 */ 1393 synchronizing = true; 1394 } else if (kvm->arch.user_set_tsc) { 1395 u64 tsc_exp = kvm->arch.last_tsc_write + 1396 nsec_to_cycles(vcpu, elapsed); 1397 u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL; 1398 /* 1399 * Here lies UAPI baggage: when a user-initiated TSC write has 1400 * a small delta (1 second) of virtual cycle time against the 1401 * previously set vCPU, we assume that they were intended to be 1402 * in sync and the delta was only due to the racy nature of the 1403 * legacy API. 1404 * 1405 * This trick falls down when restoring a guest which genuinely 1406 * has been running for less time than the 1 second of imprecision 1407 * which we allow for in the legacy API. In this case, the first 1408 * value written by userspace (on any vCPU) should not be subject 1409 * to this 'correction' to make it sync up with values that only 1410 * come from the kernel's default vCPU creation. Make the 1-second 1411 * slop hack only trigger if the user_set_tsc flag is already set. 1412 */ 1413 synchronizing = data < tsc_exp + tsc_hz && 1414 data + tsc_hz > tsc_exp; 1415 } 1416 } 1417 1418 1419 /* 1420 * For a reliable TSC, we can match TSC offsets, and for an unstable 1421 * TSC, we add elapsed time in this computation. We could let the 1422 * compensation code attempt to catch up if we fall behind, but 1423 * it's better to try to match offsets from the beginning. 1424 */ 1425 if (synchronizing && 1426 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) { 1427 if (!kvm_check_tsc_unstable()) { 1428 offset = kvm->arch.cur_tsc_offset; 1429 } else { 1430 u64 delta = nsec_to_cycles(vcpu, elapsed); 1431 data += delta; 1432 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); 1433 } 1434 matched = true; 1435 } 1436 1437 __kvm_synchronize_tsc(vcpu, offset, data, ns, matched, !!user_value); 1438 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 1439 } 1440 1441 #ifdef CONFIG_X86_64 1442 1443 static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap, 1444 u64 *tsc_timestamp) 1445 { 1446 /* 1447 * ktime_get_snapshot_id() cannot fail for standard clock IDs 1448 * (only for invalid/aux clocks or during suspend, with a WARN). 1449 */ 1450 if (!snap->valid) 1451 return false; 1452 1453 if (snap->cs_id == CSID_X86_TSC) { 1454 *tsc_timestamp = snap->cycles; 1455 return true; 1456 } 1457 1458 if (snap->hw_csid == CSID_X86_TSC && snap->hw_cycles) { 1459 *tsc_timestamp = snap->hw_cycles; 1460 return true; 1461 } 1462 1463 return false; 1464 } 1465 1466 /* 1467 * Calculates the kvmclock_base_ns (CLOCK_MONOTONIC_RAW + boot time) and 1468 * reports the TSC value from which it do so. Returns true if host is 1469 * using TSC based clocksource. 1470 */ 1471 static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) 1472 { 1473 struct system_time_snapshot snap = {}; 1474 1475 ktime_get_snapshot_id(CLOCK_MONOTONIC_RAW, &snap); 1476 if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) 1477 return false; 1478 1479 *kernel_ns = ktime_to_ns(ktime_mono_to_any(snap.systime, TK_OFFS_BOOT)); 1480 return true; 1481 } 1482 1483 /* 1484 * Calculates CLOCK_MONOTONIC and reports the TSC value from which it did 1485 * so. Returns true if host is using TSC based clocksource. 1486 */ 1487 bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) 1488 { 1489 struct system_time_snapshot snap = {}; 1490 1491 ktime_get_snapshot_id(CLOCK_MONOTONIC, &snap); 1492 if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) 1493 return false; 1494 1495 *kernel_ns = ktime_to_ns(snap.systime); 1496 return true; 1497 } 1498 1499 /* 1500 * Calculates CLOCK_REALTIME and reports the TSC value from which it did 1501 * so. Returns true if host is using TSC based clocksource. 1502 * 1503 * DO NOT USE this for anything related to migration. You want CLOCK_TAI 1504 * for that. 1505 */ 1506 static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, 1507 u64 *tsc_timestamp) 1508 { 1509 struct system_time_snapshot snap = {}; 1510 1511 ktime_get_snapshot_id(CLOCK_REALTIME, &snap); 1512 if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) 1513 return false; 1514 1515 *ts = ktime_to_timespec64(snap.systime); 1516 return true; 1517 } 1518 #endif 1519 1520 /* 1521 * 1522 * Assuming a stable TSC across physical CPUS, and a stable TSC 1523 * across virtual CPUs, the following condition is possible. 1524 * Each numbered line represents an event visible to both 1525 * CPUs at the next numbered event. 1526 * 1527 * "timespecX" represents host monotonic time. "tscX" represents 1528 * RDTSC value. 1529 * 1530 * VCPU0 on CPU0 | VCPU1 on CPU1 1531 * 1532 * 1. read timespec0,tsc0 1533 * 2. | timespec1 = timespec0 + N 1534 * | tsc1 = tsc0 + M 1535 * 3. transition to guest | transition to guest 1536 * 4. ret0 = timespec0 + (rdtsc - tsc0) | 1537 * 5. | ret1 = timespec1 + (rdtsc - tsc1) 1538 * | ret1 = timespec0 + N + (rdtsc - (tsc0 + M)) 1539 * 1540 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity: 1541 * 1542 * - ret0 < ret1 1543 * - timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M)) 1544 * ... 1545 * - 0 < N - M => M < N 1546 * 1547 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not 1548 * always the case (the difference between two distinct xtime instances 1549 * might be smaller then the difference between corresponding TSC reads, 1550 * when updating guest vcpus pvclock areas). 1551 * 1552 * To avoid that problem, do not allow visibility of distinct 1553 * system_timestamp/tsc_timestamp values simultaneously: use a master 1554 * copy of host monotonic time values. Update that master copy 1555 * in lockstep. 1556 * 1557 * Rely on synchronization of host TSCs and guest TSCs for monotonicity. 1558 * 1559 */ 1560 1561 static unsigned long get_cpu_tsc_khz(void); 1562 1563 static void pvclock_update_vm_gtod_copy(struct kvm *kvm) 1564 { 1565 #ifdef CONFIG_X86_64 1566 struct kvm_arch *ka = &kvm->arch; 1567 int vclock_mode; 1568 bool host_tsc_clocksource, vcpus_matched; 1569 1570 lockdep_assert_held(&kvm->arch.tsc_write_lock); 1571 vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == 1572 atomic_read(&kvm->online_vcpus)); 1573 1574 /* 1575 * If the host uses TSC clock, then passthrough TSC as stable 1576 * to the guest. 1577 */ 1578 host_tsc_clocksource = kvm_get_time_and_clockread( 1579 &ka->master_kernel_ns, 1580 &ka->master_cycle_now); 1581 1582 ka->use_master_clock = host_tsc_clocksource && vcpus_matched 1583 && !ka->backwards_tsc_observed 1584 && !ka->boot_vcpu_runs_old_kvmclock; 1585 1586 if (ka->use_master_clock) { 1587 u64 tsc_hz; 1588 1589 atomic_set(&kvm_guest_has_master_clock, 1); 1590 1591 /* 1592 * Copy the scaling ratio and precompute the mul/shift for 1593 * converting guest TSC to nanoseconds. These are used by 1594 * get_kvmclock() to compute kvmclock from the host TSC 1595 * without needing a vCPU reference. 1596 */ 1597 ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio; 1598 tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; 1599 if (tsc_hz && kvm_caps.has_tsc_control) 1600 tsc_hz = kvm_scale_tsc(tsc_hz, 1601 ka->master_tsc_scaling_ratio); 1602 if (tsc_hz) 1603 kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, 1604 &ka->master_tsc_shift, 1605 &ka->master_tsc_mul); 1606 else 1607 ka->use_master_clock = false; 1608 } 1609 1610 vclock_mode = pvclock_gtod_data.clock.vclock_mode; 1611 trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode, 1612 vcpus_matched); 1613 #endif 1614 } 1615 1616 static void kvm_make_mclock_inprogress_request(struct kvm *kvm) 1617 { 1618 kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS); 1619 } 1620 1621 static void __kvm_start_pvclock_update(struct kvm *kvm) 1622 { 1623 raw_spin_lock_irq(&kvm->arch.tsc_write_lock); 1624 write_seqcount_begin(&kvm->arch.pvclock_sc); 1625 } 1626 1627 static void kvm_start_pvclock_update(struct kvm *kvm) 1628 { 1629 kvm_make_mclock_inprogress_request(kvm); 1630 1631 /* no guest entries from this point */ 1632 __kvm_start_pvclock_update(kvm); 1633 } 1634 1635 static void kvm_end_pvclock_update(struct kvm *kvm) 1636 { 1637 struct kvm_arch *ka = &kvm->arch; 1638 struct kvm_vcpu *vcpu; 1639 unsigned long i; 1640 1641 write_seqcount_end(&ka->pvclock_sc); 1642 raw_spin_unlock_irq(&ka->tsc_write_lock); 1643 kvm_for_each_vcpu(i, vcpu, kvm) 1644 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 1645 1646 /* guest entries allowed */ 1647 kvm_for_each_vcpu(i, vcpu, kvm) 1648 kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu); 1649 } 1650 1651 static void kvm_update_masterclock(struct kvm *kvm) 1652 { 1653 kvm_hv_request_tsc_page_update(kvm); 1654 kvm_start_pvclock_update(kvm); 1655 pvclock_update_vm_gtod_copy(kvm); 1656 kvm_end_pvclock_update(kvm); 1657 } 1658 1659 /* 1660 * Use the kernel's tsc_khz directly if the TSC is constant, otherwise use KVM's 1661 * per-CPU value (which may be zero if a CPU is going offline). Note, tsc_khz 1662 * can change during boot even if the TSC is constant, as it's possible for KVM 1663 * to be loaded before TSC calibration completes. Ideally, KVM would get a 1664 * notification when calibration completes, but practically speaking calibration 1665 * will complete before userspace is alive enough to create VMs. 1666 */ 1667 static unsigned long get_cpu_tsc_khz(void) 1668 { 1669 if (cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC)) 1670 return tsc_khz; 1671 else 1672 return __this_cpu_read(cpu_tsc_khz); 1673 } 1674 1675 /* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */ 1676 static bool __get_kvmclock_master_clock(struct kvm *kvm, 1677 struct kvm_clock_data *data) 1678 { 1679 #ifdef CONFIG_X86_64 1680 struct kvm_arch *ka = &kvm->arch; 1681 struct pvclock_vcpu_time_info hv_clock; 1682 struct timespec64 ts; 1683 1684 if (!ka->use_master_clock) 1685 return false; 1686 1687 if (!kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) 1688 return false; 1689 1690 data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; 1691 data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC | 1692 KVM_CLOCK_TSC_STABLE; 1693 1694 hv_clock.tsc_timestamp = ka->master_cycle_now; 1695 hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; 1696 1697 /* 1698 * Use the precomputed guest-TSC-based mul/shift so that the kvmclock 1699 * value matches what the guest computes from its own TSC. 1700 */ 1701 hv_clock.tsc_shift = ka->master_tsc_shift; 1702 hv_clock.tsc_to_system_mul = ka->master_tsc_mul; 1703 1704 if (kvm_caps.has_tsc_control) { 1705 u64 tsc_delta = data->host_tsc - ka->master_cycle_now; 1706 1707 tsc_delta = kvm_scale_tsc(tsc_delta, ka->master_tsc_scaling_ratio); 1708 data->clock = hv_clock.system_time + 1709 pvclock_scale_delta(tsc_delta, 1710 hv_clock.tsc_to_system_mul, 1711 hv_clock.tsc_shift); 1712 } else { 1713 data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); 1714 } 1715 return true; 1716 #else 1717 return false; 1718 #endif 1719 } 1720 1721 static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) 1722 { 1723 struct kvm_arch *ka = &kvm->arch; 1724 unsigned seq; 1725 1726 do { 1727 data->flags = 0; 1728 1729 seq = read_seqcount_begin(&ka->pvclock_sc); 1730 if (!__get_kvmclock_master_clock(kvm, data)) 1731 data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; 1732 } while (read_seqcount_retry(&ka->pvclock_sc, seq)); 1733 } 1734 1735 u64 get_kvmclock_ns(struct kvm *kvm) 1736 { 1737 struct kvm_clock_data data; 1738 1739 get_kvmclock(kvm, &data); 1740 return data.clock; 1741 } 1742 1743 static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock, 1744 struct kvm_vcpu *vcpu, 1745 struct gfn_to_pfn_cache *gpc, 1746 unsigned int offset) 1747 { 1748 struct pvclock_vcpu_time_info *guest_hv_clock; 1749 struct pvclock_vcpu_time_info hv_clock; 1750 unsigned long flags; 1751 1752 memcpy(&hv_clock, ref_hv_clock, sizeof(hv_clock)); 1753 1754 read_lock_irqsave(&gpc->lock, flags); 1755 while (!kvm_gpc_check(gpc, offset + sizeof(*guest_hv_clock))) { 1756 read_unlock_irqrestore(&gpc->lock, flags); 1757 1758 if (kvm_gpc_refresh(gpc, offset + sizeof(*guest_hv_clock))) 1759 return; 1760 1761 read_lock_irqsave(&gpc->lock, flags); 1762 } 1763 1764 guest_hv_clock = (void *)(gpc->khva + offset); 1765 1766 /* 1767 * This VCPU is paused, but it's legal for a guest to read another 1768 * VCPU's kvmclock, so we really have to follow the specification where 1769 * it says that version is odd if data is being modified, and even after 1770 * it is consistent. 1771 */ 1772 1773 guest_hv_clock->version = hv_clock.version = (guest_hv_clock->version + 1) | 1; 1774 smp_wmb(); 1775 1776 /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */ 1777 hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED); 1778 1779 memcpy(guest_hv_clock, &hv_clock, sizeof(*guest_hv_clock)); 1780 1781 smp_wmb(); 1782 1783 guest_hv_clock->version = ++hv_clock.version; 1784 1785 kvm_gpc_mark_dirty_in_slot(gpc); 1786 read_unlock_irqrestore(&gpc->lock, flags); 1787 1788 trace_kvm_pvclock_update(vcpu->vcpu_id, &hv_clock); 1789 } 1790 1791 int kvm_guest_time_update(struct kvm_vcpu *v) 1792 { 1793 u64 tgt_tsc_hz, tsc_timestamp, host_tsc, master_tsc, master_ns; 1794 struct kvm_arch *ka __maybe_unused = &v->kvm->arch; 1795 struct pvclock_vcpu_time_info hv_clock = {}; 1796 struct kvm_vcpu_arch *vcpu = &v->arch; 1797 s64 kernel_ns; 1798 1799 /* 1800 * If the host uses TSC clock, then passthrough TSC as stable 1801 * to the guest. 1802 */ 1803 #ifdef CONFIG_X86_64 1804 bool use_master_clock; 1805 unsigned int seq; 1806 1807 do { 1808 seq = read_seqcount_begin(&ka->pvclock_sc); 1809 use_master_clock = ka->use_master_clock; 1810 if (!use_master_clock) 1811 continue; 1812 1813 if (!kvm_get_time_and_clockread(&kernel_ns, &host_tsc)) { 1814 use_master_clock = false; 1815 continue; 1816 } 1817 1818 master_tsc = ka->master_cycle_now; 1819 master_ns = ka->master_kernel_ns; 1820 } while (read_seqcount_retry(&ka->pvclock_sc, seq)); 1821 #else 1822 const bool use_master_clock = false; 1823 #endif 1824 /* 1825 * Ensure reading the TSC+frequency pair is done on the same CPU. When 1826 * NOT using the master clock, the TSC frequency may vary between CPUs. 1827 */ 1828 preempt_disable(); 1829 tgt_tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; 1830 if (unlikely(tgt_tsc_hz == 0)) { 1831 preempt_enable(); 1832 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); 1833 return 1; 1834 } 1835 if (!use_master_clock) { 1836 host_tsc = rdtsc(); 1837 kernel_ns = get_kvmclock_base_ns(); 1838 } 1839 1840 tsc_timestamp = kvm_read_l1_tsc(v, host_tsc); 1841 1842 /* 1843 * We may have to catch up the TSC to match elapsed wall clock 1844 * time for two reasons, even if kvmclock is used. 1845 * 1) CPU could have been running below the maximum TSC rate 1846 * 2) Broken TSC compensation resets the base at each VCPU 1847 * entry to avoid unknown leaps of TSC even when running 1848 * again on the same CPU. This may cause apparent elapsed 1849 * time to disappear, and the guest to stand still or run 1850 * very slowly. 1851 */ 1852 if (vcpu->tsc_catchup) { 1853 u64 tsc = compute_guest_tsc(v, kernel_ns); 1854 if (tsc > tsc_timestamp) { 1855 adjust_tsc_offset_guest(v, tsc - tsc_timestamp); 1856 tsc_timestamp = tsc; 1857 } 1858 } 1859 1860 /* 1861 * Refresh L1's last "observed" TSC to match the PV clock's timestamp, 1862 * e.g. so that the guest can't see a TSC that's behind the reference. 1863 */ 1864 vcpu->last_guest_tsc = tsc_timestamp; 1865 1866 preempt_enable(); 1867 1868 /* With all the info we got, fill in the values */ 1869 1870 if (kvm_caps.has_tsc_control) { 1871 tgt_tsc_hz = kvm_scale_tsc(tgt_tsc_hz, 1872 v->arch.l1_tsc_scaling_ratio); 1873 tgt_tsc_hz = tgt_tsc_hz ? : 1; 1874 } 1875 1876 if (unlikely(vcpu->hw_tsc_hz != tgt_tsc_hz)) { 1877 kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_hz, 1878 &vcpu->pvclock_tsc_shift, 1879 &vcpu->pvclock_tsc_mul); 1880 vcpu->hw_tsc_hz = tgt_tsc_hz; 1881 } 1882 1883 hv_clock.tsc_shift = vcpu->pvclock_tsc_shift; 1884 hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul; 1885 /* 1886 * If the master clock is NOT in use, the reference time placed in the 1887 * hv_clock is "now". If master clock is in use, the reference time is 1888 * the master clock's snapshot from some time in the past, not "now". 1889 */ 1890 if (use_master_clock) { 1891 hv_clock.tsc_timestamp = kvm_read_l1_tsc(v, master_tsc); 1892 hv_clock.system_time = master_ns + v->kvm->arch.kvmclock_offset; 1893 } else { 1894 hv_clock.tsc_timestamp = tsc_timestamp; 1895 hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; 1896 } 1897 1898 /* If the host uses TSC clocksource, then it is stable */ 1899 hv_clock.flags = 0; 1900 if (use_master_clock) 1901 hv_clock.flags |= PVCLOCK_TSC_STABLE_BIT; 1902 1903 if (vcpu->pv_time.active) { 1904 /* 1905 * GUEST_STOPPED is only supported by kvmclock, and KVM's 1906 * historic behavior is to only process the request if kvmclock 1907 * is active/enabled. 1908 */ 1909 if (vcpu->pvclock_set_guest_stopped_request) { 1910 hv_clock.flags |= PVCLOCK_GUEST_STOPPED; 1911 vcpu->pvclock_set_guest_stopped_request = false; 1912 } 1913 kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->pv_time, 0); 1914 1915 hv_clock.flags &= ~PVCLOCK_GUEST_STOPPED; 1916 } 1917 1918 kvm_hv_setup_tsc_page(v->kvm, &hv_clock); 1919 1920 #ifdef CONFIG_KVM_XEN 1921 /* 1922 * For Xen guests we may need to override PVCLOCK_TSC_STABLE_BIT as unless 1923 * explicitly told to use TSC as its clocksource Xen will not set this bit. 1924 * This default behaviour led to bugs in some guest kernels which cause 1925 * problems if they observe PVCLOCK_TSC_STABLE_BIT in the pvclock flags. 1926 * 1927 * Note! Clear TSC_STABLE only for Xen clocks, i.e. the order matters! 1928 */ 1929 if (ka->xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE) 1930 hv_clock.flags &= ~PVCLOCK_TSC_STABLE_BIT; 1931 1932 if (vcpu->xen.vcpu_info_cache.active) 1933 kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->xen.vcpu_info_cache, 1934 offsetof(struct compat_vcpu_info, time)); 1935 if (vcpu->xen.vcpu_time_info_cache.active) 1936 kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->xen.vcpu_time_info_cache, 0); 1937 #endif 1938 return 0; 1939 } 1940 1941 /* 1942 * The pvclock_wall_clock ABI tells the guest the wall clock time at 1943 * which it started (i.e. its epoch, when its kvmclock was zero). 1944 * 1945 * In fact those clocks are subtly different; wall clock frequency is 1946 * adjusted by NTP and has leap seconds, while the kvmclock is a 1947 * simple function of the TSC without any such adjustment. 1948 * 1949 * Perhaps the ABI should have exposed CLOCK_TAI and a ratio between 1950 * that and kvmclock, but even that would be subject to change over 1951 * time. 1952 * 1953 * Attempt to calculate the epoch at a given moment using the *same* 1954 * TSC reading via kvm_get_walltime_and_clockread() to obtain both 1955 * wallclock and kvmclock times, and subtracting one from the other. 1956 * 1957 * Fall back to using their values at slightly different moments by 1958 * calling ktime_get_real_ns() and get_kvmclock() separately. 1959 */ 1960 uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm) 1961 { 1962 struct kvm_clock_data data; 1963 1964 get_kvmclock(kvm, &data); 1965 1966 /* 1967 * If get_kvmclock() captured both wallclock and kvmclock from the 1968 * same TSC reading, use them for a precise epoch calculation. 1969 */ 1970 if (data.flags & KVM_CLOCK_REALTIME) 1971 return data.realtime - data.clock; 1972 1973 return ktime_get_real_ns() - data.clock; 1974 } 1975 1976 /* 1977 * kvmclock updates which are isolated to a given vcpu, such as 1978 * vcpu->cpu migration, should not allow system_timestamp from 1979 * the rest of the vcpus to remain static. 1980 * 1981 * So in those cases, request a kvmclock update for all vcpus. 1982 * The worst case for a remote vcpu to update its kvmclock 1983 * is then bounded by maximum nohz sleep latency. 1984 */ 1985 static void kvm_gen_kvmclock_update(struct kvm_vcpu *v) 1986 { 1987 unsigned long i; 1988 struct kvm_vcpu *vcpu; 1989 struct kvm *kvm = v->kvm; 1990 1991 kvm_for_each_vcpu(i, vcpu, kvm) { 1992 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 1993 kvm_vcpu_kick(vcpu); 1994 } 1995 } 1996 1997 static void kvmclock_reset(struct kvm_vcpu *vcpu) 1998 { 1999 kvm_gpc_deactivate(&vcpu->arch.pv_time); 2000 vcpu->arch.time = 0; 2001 } 2002 2003 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu) 2004 { 2005 ++vcpu->stat.tlb_flush; 2006 kvm_x86_call(flush_tlb_all)(vcpu); 2007 2008 /* Flushing all ASIDs flushes the current ASID... */ 2009 kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 2010 } 2011 2012 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu) 2013 { 2014 ++vcpu->stat.tlb_flush; 2015 2016 if (!tdp_enabled) { 2017 /* 2018 * A TLB flush on behalf of the guest is equivalent to 2019 * INVPCID(all), toggling CR4.PGE, etc., which requires 2020 * a forced sync of the shadow page tables. Ensure all the 2021 * roots are synced and the guest TLB in hardware is clean. 2022 */ 2023 kvm_mmu_sync_roots(vcpu); 2024 kvm_mmu_sync_prev_roots(vcpu); 2025 } 2026 2027 kvm_x86_call(flush_tlb_guest)(vcpu); 2028 2029 /* 2030 * Flushing all "guest" TLB is always a superset of Hyper-V's fine 2031 * grained flushing. 2032 */ 2033 kvm_hv_vcpu_purge_flush_tlb(vcpu); 2034 } 2035 2036 2037 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu) 2038 { 2039 ++vcpu->stat.tlb_flush; 2040 kvm_x86_call(flush_tlb_current)(vcpu); 2041 } 2042 2043 /* 2044 * Service "local" TLB flush requests, which are specific to the current MMU 2045 * context. In addition to the generic event handling in vcpu_enter_guest(), 2046 * TLB flushes that are targeted at an MMU context also need to be serviced 2047 * prior before nested VM-Enter/VM-Exit. 2048 */ 2049 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu) 2050 { 2051 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) 2052 kvm_vcpu_flush_tlb_current(vcpu); 2053 2054 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu)) 2055 kvm_vcpu_flush_tlb_guest(vcpu); 2056 } 2057 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_service_local_tlb_flush_requests); 2058 2059 static void record_steal_time(struct kvm_vcpu *vcpu) 2060 { 2061 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache; 2062 struct kvm_steal_time __user *st; 2063 struct kvm_memslots *slots; 2064 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS; 2065 u64 steal; 2066 u32 version; 2067 2068 if (kvm_xen_msr_enabled(vcpu->kvm)) { 2069 kvm_xen_runstate_set_running(vcpu); 2070 return; 2071 } 2072 2073 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 2074 return; 2075 2076 if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm)) 2077 return; 2078 2079 slots = kvm_memslots(vcpu->kvm); 2080 2081 if (unlikely(slots->generation != ghc->generation || 2082 gpa != ghc->gpa || 2083 kvm_is_error_hva(ghc->hva) || !ghc->memslot)) { 2084 /* We rely on the fact that it fits in a single page. */ 2085 BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS); 2086 2087 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) || 2088 kvm_is_error_hva(ghc->hva) || !ghc->memslot) 2089 return; 2090 } 2091 2092 st = (struct kvm_steal_time __user *)ghc->hva; 2093 /* 2094 * Doing a TLB flush here, on the guest's behalf, can avoid 2095 * expensive IPIs. 2096 */ 2097 if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) { 2098 u8 st_preempted = 0; 2099 int err = -EFAULT; 2100 2101 if (!user_access_begin(st, sizeof(*st))) 2102 return; 2103 2104 asm volatile("1: xchgb %0, %2\n" 2105 "xor %1, %1\n" 2106 "2:\n" 2107 _ASM_EXTABLE_UA(1b, 2b) 2108 : "+q" (st_preempted), 2109 "+&r" (err), 2110 "+m" (st->preempted)); 2111 if (err) 2112 goto out; 2113 2114 user_access_end(); 2115 2116 vcpu->arch.st.preempted = 0; 2117 2118 trace_kvm_pv_tlb_flush(vcpu->vcpu_id, 2119 st_preempted & KVM_VCPU_FLUSH_TLB); 2120 if (st_preempted & KVM_VCPU_FLUSH_TLB) 2121 kvm_vcpu_flush_tlb_guest(vcpu); 2122 2123 if (!user_access_begin(st, sizeof(*st))) 2124 goto dirty; 2125 } else { 2126 if (!user_access_begin(st, sizeof(*st))) 2127 return; 2128 2129 unsafe_put_user(0, &st->preempted, out); 2130 vcpu->arch.st.preempted = 0; 2131 } 2132 2133 unsafe_get_user(version, &st->version, out); 2134 if (version & 1) 2135 version += 1; /* first time write, random junk */ 2136 2137 version += 1; 2138 unsafe_put_user(version, &st->version, out); 2139 2140 smp_wmb(); 2141 2142 unsafe_get_user(steal, &st->steal, out); 2143 steal += current->sched_info.run_delay - 2144 vcpu->arch.st.last_steal; 2145 vcpu->arch.st.last_steal = current->sched_info.run_delay; 2146 unsafe_put_user(steal, &st->steal, out); 2147 2148 version += 1; 2149 unsafe_put_user(version, &st->version, out); 2150 2151 out: 2152 user_access_end(); 2153 dirty: 2154 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 2155 } 2156 2157 static inline bool kvm_can_mwait_in_guest(void) 2158 { 2159 return boot_cpu_has(X86_FEATURE_MWAIT) && 2160 !boot_cpu_has_bug(X86_BUG_MONITOR) && 2161 boot_cpu_has(X86_FEATURE_ARAT); 2162 } 2163 2164 static u64 kvm_get_allowed_disable_exits(void) 2165 { 2166 u64 r = KVM_X86_DISABLE_EXITS_PAUSE; 2167 2168 if (boot_cpu_has(X86_FEATURE_APERFMPERF)) 2169 r |= KVM_X86_DISABLE_EXITS_APERFMPERF; 2170 2171 if (!mitigate_smt_rsb) { 2172 r |= KVM_X86_DISABLE_EXITS_HLT | 2173 KVM_X86_DISABLE_EXITS_CSTATE; 2174 2175 if (kvm_can_mwait_in_guest()) 2176 r |= KVM_X86_DISABLE_EXITS_MWAIT; 2177 } 2178 return r; 2179 } 2180 2181 #ifdef CONFIG_KVM_HYPERV 2182 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu, 2183 struct kvm_cpuid2 __user *cpuid_arg) 2184 { 2185 struct kvm_cpuid2 cpuid; 2186 int r; 2187 2188 r = -EFAULT; 2189 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 2190 return r; 2191 2192 r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries); 2193 if (r) 2194 return r; 2195 2196 r = -EFAULT; 2197 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid))) 2198 return r; 2199 2200 return 0; 2201 } 2202 #endif 2203 2204 static bool kvm_is_vm_type_supported(unsigned long type) 2205 { 2206 return type < 32 && (kvm_caps.supported_vm_types & BIT(type)); 2207 } 2208 2209 static inline u64 kvm_sync_valid_fields(struct kvm *kvm) 2210 { 2211 return kvm && kvm->arch.has_protected_state ? 0 : KVM_SYNC_X86_VALID_FIELDS; 2212 } 2213 2214 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 2215 { 2216 int r = 0; 2217 2218 switch (ext) { 2219 case KVM_CAP_IRQCHIP: 2220 case KVM_CAP_HLT: 2221 case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: 2222 case KVM_CAP_SET_TSS_ADDR: 2223 case KVM_CAP_EXT_CPUID: 2224 case KVM_CAP_EXT_EMUL_CPUID: 2225 case KVM_CAP_CLOCKSOURCE: 2226 #ifdef CONFIG_KVM_IOAPIC 2227 case KVM_CAP_PIT: 2228 case KVM_CAP_PIT2: 2229 case KVM_CAP_PIT_STATE2: 2230 case KVM_CAP_REINJECT_CONTROL: 2231 #endif 2232 case KVM_CAP_NOP_IO_DELAY: 2233 case KVM_CAP_MP_STATE: 2234 case KVM_CAP_USER_NMI: 2235 case KVM_CAP_IRQ_INJECT_STATUS: 2236 case KVM_CAP_IOEVENTFD: 2237 case KVM_CAP_IOEVENTFD_NO_LENGTH: 2238 2239 case KVM_CAP_SET_IDENTITY_MAP_ADDR: 2240 case KVM_CAP_VCPU_EVENTS: 2241 #ifdef CONFIG_KVM_HYPERV 2242 case KVM_CAP_HYPERV: 2243 case KVM_CAP_HYPERV_VAPIC: 2244 case KVM_CAP_HYPERV_SPIN: 2245 case KVM_CAP_HYPERV_TIME: 2246 case KVM_CAP_HYPERV_SYNIC: 2247 case KVM_CAP_HYPERV_SYNIC2: 2248 case KVM_CAP_HYPERV_VP_INDEX: 2249 case KVM_CAP_HYPERV_EVENTFD: 2250 case KVM_CAP_HYPERV_TLBFLUSH: 2251 case KVM_CAP_HYPERV_SEND_IPI: 2252 case KVM_CAP_HYPERV_CPUID: 2253 case KVM_CAP_HYPERV_ENFORCE_CPUID: 2254 case KVM_CAP_SYS_HYPERV_CPUID: 2255 #endif 2256 case KVM_CAP_PCI_SEGMENT: 2257 case KVM_CAP_DEBUGREGS: 2258 case KVM_CAP_X86_ROBUST_SINGLESTEP: 2259 case KVM_CAP_XSAVE: 2260 case KVM_CAP_ASYNC_PF: 2261 case KVM_CAP_ASYNC_PF_INT: 2262 case KVM_CAP_GET_TSC_KHZ: 2263 case KVM_CAP_KVMCLOCK_CTRL: 2264 case KVM_CAP_IOAPIC_POLARITY_IGNORED: 2265 case KVM_CAP_TSC_DEADLINE_TIMER: 2266 case KVM_CAP_DISABLE_QUIRKS: 2267 case KVM_CAP_SET_BOOT_CPU_ID: 2268 case KVM_CAP_SPLIT_IRQCHIP: 2269 case KVM_CAP_IMMEDIATE_EXIT: 2270 case KVM_CAP_PMU_EVENT_FILTER: 2271 case KVM_CAP_PMU_EVENT_MASKED_EVENTS: 2272 case KVM_CAP_GET_MSR_FEATURES: 2273 case KVM_CAP_MSR_PLATFORM_INFO: 2274 case KVM_CAP_EXCEPTION_PAYLOAD: 2275 case KVM_CAP_X86_TRIPLE_FAULT_EVENT: 2276 case KVM_CAP_SET_GUEST_DEBUG: 2277 case KVM_CAP_LAST_CPU: 2278 case KVM_CAP_X86_USER_SPACE_MSR: 2279 case KVM_CAP_X86_MSR_FILTER: 2280 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID: 2281 #ifdef CONFIG_X86_SGX_KVM 2282 case KVM_CAP_SGX_ATTRIBUTE: 2283 #endif 2284 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: 2285 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM: 2286 case KVM_CAP_SREGS2: 2287 case KVM_CAP_EXIT_ON_EMULATION_FAILURE: 2288 case KVM_CAP_VCPU_ATTRIBUTES: 2289 case KVM_CAP_SYS_ATTRIBUTES: 2290 case KVM_CAP_VAPIC: 2291 case KVM_CAP_ENABLE_CAP: 2292 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES: 2293 case KVM_CAP_IRQFD_RESAMPLE: 2294 case KVM_CAP_MEMORY_FAULT_INFO: 2295 case KVM_CAP_X86_GUEST_MODE: 2296 case KVM_CAP_ONE_REG: 2297 r = 1; 2298 break; 2299 case KVM_CAP_PRE_FAULT_MEMORY: 2300 r = tdp_enabled; 2301 break; 2302 case KVM_CAP_X86_APIC_BUS_CYCLES_NS: 2303 r = kvm ? kvm->arch.apic_bus_cycle_ns : APIC_BUS_CYCLE_NS_DEFAULT; 2304 break; 2305 case KVM_CAP_EXIT_HYPERCALL: 2306 r = KVM_EXIT_HYPERCALL_VALID_MASK; 2307 break; 2308 case KVM_CAP_SET_GUEST_DEBUG2: 2309 return KVM_GUESTDBG_VALID_MASK; 2310 #ifdef CONFIG_KVM_XEN 2311 case KVM_CAP_XEN_HVM: 2312 r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR | 2313 KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL | 2314 KVM_XEN_HVM_CONFIG_SHARED_INFO | 2315 KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL | 2316 KVM_XEN_HVM_CONFIG_EVTCHN_SEND | 2317 KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE | 2318 KVM_XEN_HVM_CONFIG_SHARED_INFO_HVA; 2319 if (sched_info_on()) 2320 r |= KVM_XEN_HVM_CONFIG_RUNSTATE | 2321 KVM_XEN_HVM_CONFIG_RUNSTATE_UPDATE_FLAG; 2322 break; 2323 #endif 2324 case KVM_CAP_SYNC_REGS: 2325 r = kvm_sync_valid_fields(kvm); 2326 break; 2327 case KVM_CAP_ADJUST_CLOCK: 2328 r = KVM_CLOCK_VALID_FLAGS; 2329 break; 2330 case KVM_CAP_X86_DISABLE_EXITS: 2331 r = kvm_get_allowed_disable_exits(); 2332 break; 2333 case KVM_CAP_X86_SMM: 2334 if (!IS_ENABLED(CONFIG_KVM_SMM)) 2335 break; 2336 2337 /* SMBASE is usually relocated above 1M on modern chipsets, 2338 * and SMM handlers might indeed rely on 4G segment limits, 2339 * so do not report SMM to be available if real mode is 2340 * emulated via vm86 mode. Still, do not go to great lengths 2341 * to avoid userspace's usage of the feature, because it is a 2342 * fringe case that is not enabled except via specific settings 2343 * of the module parameters. 2344 */ 2345 r = kvm_x86_call(has_emulated_msr)(kvm, MSR_IA32_SMBASE); 2346 break; 2347 case KVM_CAP_NR_VCPUS: 2348 r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS); 2349 break; 2350 case KVM_CAP_MAX_VCPUS: 2351 r = KVM_MAX_VCPUS; 2352 if (kvm) 2353 r = kvm->max_vcpus; 2354 break; 2355 case KVM_CAP_MAX_VCPU_ID: 2356 r = KVM_MAX_VCPU_IDS; 2357 break; 2358 case KVM_CAP_PV_MMU: /* obsolete */ 2359 r = 0; 2360 break; 2361 case KVM_CAP_MCE: 2362 r = KVM_MAX_MCE_BANKS; 2363 break; 2364 case KVM_CAP_XCRS: 2365 r = boot_cpu_has(X86_FEATURE_XSAVE); 2366 break; 2367 case KVM_CAP_TSC_CONTROL: 2368 case KVM_CAP_VM_TSC_CONTROL: 2369 r = kvm_caps.has_tsc_control; 2370 break; 2371 case KVM_CAP_X2APIC_API: 2372 r = KVM_X2APIC_API_VALID_FLAGS; 2373 if (kvm && !irqchip_split(kvm)) 2374 r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST; 2375 break; 2376 case KVM_CAP_NESTED_STATE: 2377 r = kvm_nested_ops.enabled ? kvm_nested_call(get_state)(NULL, NULL, 0) : 0; 2378 break; 2379 #ifdef CONFIG_KVM_HYPERV 2380 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: 2381 r = kvm_x86_ops.enable_l2_tlb_flush != NULL; 2382 break; 2383 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: 2384 r = kvm_nested_ops.enabled && kvm_nested_ops.enable_evmcs != NULL; 2385 break; 2386 #endif 2387 case KVM_CAP_SMALLER_MAXPHYADDR: 2388 r = (int) allow_smaller_maxphyaddr; 2389 break; 2390 case KVM_CAP_STEAL_TIME: 2391 r = sched_info_on(); 2392 break; 2393 case KVM_CAP_X86_BUS_LOCK_EXIT: 2394 if (kvm_caps.has_bus_lock_exit) 2395 r = KVM_BUS_LOCK_DETECTION_OFF | 2396 KVM_BUS_LOCK_DETECTION_EXIT; 2397 else 2398 r = 0; 2399 break; 2400 case KVM_CAP_XSAVE2: { 2401 r = xstate_required_size(kvm_get_filtered_xcr0(), false); 2402 if (r < sizeof(struct kvm_xsave)) 2403 r = sizeof(struct kvm_xsave); 2404 break; 2405 } 2406 case KVM_CAP_PMU_CAPABILITY: 2407 r = enable_pmu ? KVM_CAP_PMU_VALID_MASK : 0; 2408 break; 2409 case KVM_CAP_DISABLE_QUIRKS2: 2410 r = kvm_caps.supported_quirks; 2411 break; 2412 case KVM_CAP_X86_NOTIFY_VMEXIT: 2413 r = kvm_caps.has_notify_vmexit; 2414 break; 2415 case KVM_CAP_VM_TYPES: 2416 r = kvm_caps.supported_vm_types; 2417 break; 2418 case KVM_CAP_READONLY_MEM: 2419 r = kvm ? kvm_arch_has_readonly_mem(kvm) : 1; 2420 break; 2421 default: 2422 break; 2423 } 2424 return r; 2425 } 2426 2427 static int __kvm_x86_dev_get_attr(struct kvm_device_attr *attr, u64 *val) 2428 { 2429 if (attr->group) { 2430 if (kvm_x86_ops.dev_get_attr) 2431 return kvm_x86_call(dev_get_attr)(attr->group, attr->attr, val); 2432 return -ENXIO; 2433 } 2434 2435 switch (attr->attr) { 2436 case KVM_X86_XCOMP_GUEST_SUPP: 2437 *val = kvm_caps.supported_xcr0; 2438 return 0; 2439 default: 2440 return -ENXIO; 2441 } 2442 } 2443 2444 static int kvm_x86_dev_get_attr(struct kvm_device_attr *attr) 2445 { 2446 u64 __user *uaddr = u64_to_user_ptr(attr->addr); 2447 int r; 2448 u64 val; 2449 2450 r = __kvm_x86_dev_get_attr(attr, &val); 2451 if (r < 0) 2452 return r; 2453 2454 if (put_user(val, uaddr)) 2455 return -EFAULT; 2456 2457 return 0; 2458 } 2459 2460 static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr) 2461 { 2462 u64 val; 2463 2464 return __kvm_x86_dev_get_attr(attr, &val); 2465 } 2466 2467 long kvm_arch_dev_ioctl(struct file *filp, 2468 unsigned int ioctl, unsigned long arg) 2469 { 2470 void __user *argp = (void __user *)arg; 2471 long r; 2472 2473 switch (ioctl) { 2474 case KVM_GET_MSR_INDEX_LIST: 2475 r = kvm_get_msr_index_list(argp); 2476 break; 2477 case KVM_GET_SUPPORTED_CPUID: 2478 case KVM_GET_EMULATED_CPUID: { 2479 struct kvm_cpuid2 __user *cpuid_arg = argp; 2480 struct kvm_cpuid2 cpuid; 2481 2482 r = -EFAULT; 2483 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 2484 goto out; 2485 2486 r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries, 2487 ioctl); 2488 if (r) 2489 goto out; 2490 2491 r = -EFAULT; 2492 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid))) 2493 goto out; 2494 r = 0; 2495 break; 2496 } 2497 case KVM_X86_GET_MCE_CAP_SUPPORTED: 2498 r = -EFAULT; 2499 if (copy_to_user(argp, &kvm_caps.supported_mce_cap, 2500 sizeof(kvm_caps.supported_mce_cap))) 2501 goto out; 2502 r = 0; 2503 break; 2504 case KVM_GET_MSR_FEATURE_INDEX_LIST: 2505 r = kvm_get_feature_msr_index_list(argp); 2506 break; 2507 case KVM_GET_MSRS: 2508 r = kvm_get_feature_msrs(argp); 2509 break; 2510 #ifdef CONFIG_KVM_HYPERV 2511 case KVM_GET_SUPPORTED_HV_CPUID: 2512 r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp); 2513 break; 2514 #endif 2515 case KVM_GET_DEVICE_ATTR: { 2516 struct kvm_device_attr attr; 2517 r = -EFAULT; 2518 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2519 break; 2520 r = kvm_x86_dev_get_attr(&attr); 2521 break; 2522 } 2523 case KVM_HAS_DEVICE_ATTR: { 2524 struct kvm_device_attr attr; 2525 r = -EFAULT; 2526 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2527 break; 2528 r = kvm_x86_dev_has_attr(&attr); 2529 break; 2530 } 2531 default: 2532 r = -EINVAL; 2533 break; 2534 } 2535 out: 2536 return r; 2537 } 2538 2539 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu) 2540 { 2541 return kvm_arch_has_noncoherent_dma(vcpu->kvm); 2542 } 2543 2544 static DEFINE_PER_CPU(struct kvm_vcpu *, last_vcpu); 2545 2546 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 2547 { 2548 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 2549 2550 kvm_request_l1tf_flush_l1d(); 2551 2552 if (vcpu->scheduled_out && pmu->version && pmu->event_count) { 2553 pmu->need_cleanup = true; 2554 kvm_make_request(KVM_REQ_PMU, vcpu); 2555 } 2556 2557 /* Address WBINVD may be executed by guest */ 2558 if (need_emulate_wbinvd(vcpu)) { 2559 if (kvm_x86_call(has_wbinvd_exit)()) 2560 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 2561 else if (vcpu->cpu != -1 && vcpu->cpu != cpu) 2562 wbinvd_on_cpu(vcpu->cpu); 2563 } 2564 2565 kvm_x86_call(vcpu_load)(vcpu, cpu); 2566 2567 if (vcpu != per_cpu(last_vcpu, cpu)) { 2568 /* 2569 * Flush the branch predictor when switching vCPUs on the same 2570 * physical CPU, as each vCPU needs its own branch prediction 2571 * domain. No IBPB is needed when switching between L1 and L2 2572 * on the same vCPU unless IBRS is advertised to the vCPU; that 2573 * is handled on the nested VM-Exit path. 2574 */ 2575 if (static_branch_likely(&switch_vcpu_ibpb)) 2576 indirect_branch_prediction_barrier(); 2577 per_cpu(last_vcpu, cpu) = vcpu; 2578 } 2579 2580 /* Save host pkru register if supported */ 2581 vcpu->arch.host_pkru = read_pkru(); 2582 2583 /* Apply any externally detected TSC adjustments (due to suspend) */ 2584 if (unlikely(vcpu->arch.tsc_offset_adjustment)) { 2585 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment); 2586 vcpu->arch.tsc_offset_adjustment = 0; 2587 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 2588 } 2589 2590 if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) { 2591 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 : 2592 rdtsc() - vcpu->arch.last_host_tsc; 2593 if (tsc_delta < 0) 2594 mark_tsc_unstable("KVM discovered backwards TSC"); 2595 2596 if (kvm_check_tsc_unstable()) { 2597 u64 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), 2598 vcpu->arch.last_guest_tsc); 2599 kvm_vcpu_write_tsc_offset(vcpu, offset); 2600 if (!vcpu->arch.guest_tsc_protected) 2601 vcpu->arch.tsc_catchup = 1; 2602 } 2603 2604 if (kvm_lapic_hv_timer_in_use(vcpu)) 2605 kvm_lapic_restart_hv_timer(vcpu); 2606 2607 /* 2608 * On a host with synchronized TSC, there is no need to update 2609 * kvmclock on vcpu->cpu migration 2610 */ 2611 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1) { 2612 if (__ratelimit(&vcpu->kvm->arch.kvmclock_update_rs)) 2613 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); 2614 else 2615 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 2616 } 2617 2618 if (vcpu->cpu != cpu) 2619 kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu); 2620 vcpu->cpu = cpu; 2621 } 2622 2623 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 2624 } 2625 2626 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) 2627 { 2628 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache; 2629 struct kvm_steal_time __user *st; 2630 struct kvm_memslots *slots; 2631 static const u8 preempted = KVM_VCPU_PREEMPTED; 2632 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS; 2633 2634 /* 2635 * The vCPU can be marked preempted if and only if the VM-Exit was on 2636 * an instruction boundary and will not trigger guest emulation of any 2637 * kind (see vcpu_run). Vendor specific code controls (conservatively) 2638 * when this is true, for example allowing the vCPU to be marked 2639 * preempted if and only if the VM-Exit was due to a host interrupt. 2640 */ 2641 if (!vcpu->arch.at_instruction_boundary) { 2642 vcpu->stat.preemption_other++; 2643 return; 2644 } 2645 2646 vcpu->stat.preemption_reported++; 2647 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 2648 return; 2649 2650 if (vcpu->arch.st.preempted) 2651 return; 2652 2653 /* This happens on process exit */ 2654 if (unlikely(current->mm != vcpu->kvm->mm)) 2655 return; 2656 2657 slots = kvm_memslots(vcpu->kvm); 2658 2659 if (unlikely(slots->generation != ghc->generation || 2660 gpa != ghc->gpa || 2661 kvm_is_error_hva(ghc->hva) || !ghc->memslot)) 2662 return; 2663 2664 st = (struct kvm_steal_time __user *)ghc->hva; 2665 BUILD_BUG_ON(sizeof(st->preempted) != sizeof(preempted)); 2666 2667 if (!copy_to_user_nofault(&st->preempted, &preempted, sizeof(preempted))) 2668 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED; 2669 2670 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 2671 } 2672 2673 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 2674 { 2675 int idx; 2676 2677 if (vcpu->preempted) { 2678 /* 2679 * Assume protected guests are in-kernel. Inefficient yielding 2680 * due to false positives is preferable to never yielding due 2681 * to false negatives. 2682 */ 2683 vcpu->arch.preempted_in_kernel = vcpu->arch.guest_state_protected || 2684 !kvm_x86_call(get_cpl_no_cache)(vcpu); 2685 2686 /* 2687 * Take the srcu lock as memslots will be accessed to check the gfn 2688 * cache generation against the memslots generation. 2689 */ 2690 idx = srcu_read_lock(&vcpu->kvm->srcu); 2691 if (kvm_xen_msr_enabled(vcpu->kvm)) 2692 kvm_xen_runstate_set_preempted(vcpu); 2693 else 2694 kvm_steal_time_set_preempted(vcpu); 2695 srcu_read_unlock(&vcpu->kvm->srcu, idx); 2696 } 2697 2698 kvm_x86_call(vcpu_put)(vcpu); 2699 vcpu->arch.last_host_tsc = rdtsc(); 2700 } 2701 2702 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, 2703 struct kvm_lapic_state *s) 2704 { 2705 if (vcpu->arch.apic->guest_apic_protected) 2706 return -EINVAL; 2707 2708 kvm_x86_call(sync_pir_to_irr)(vcpu); 2709 2710 return kvm_apic_get_state(vcpu, s); 2711 } 2712 2713 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, 2714 struct kvm_lapic_state *s) 2715 { 2716 int r; 2717 2718 if (vcpu->arch.apic->guest_apic_protected) 2719 return -EINVAL; 2720 2721 r = kvm_apic_set_state(vcpu, s); 2722 if (r) 2723 return r; 2724 2725 return 0; 2726 } 2727 2728 static bool kvm_is_interrupt_allowed(struct kvm_vcpu *vcpu) 2729 { 2730 /* 2731 * Note, .interrupt_allowed() returns -EBUSY if interrupts are allowed 2732 * based on CPU state, but can't be immediately delivered due to a 2733 * pending nested VM-Enter. Treat that case as "allowed", because 2734 * the goal here is just to check if interrupts are architecturally 2735 * allowed, not to check if they can be injected. 2736 */ 2737 return kvm_x86_call(interrupt_allowed)(vcpu, false); 2738 } 2739 2740 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu) 2741 { 2742 /* 2743 * We can accept userspace's request for interrupt injection 2744 * as long as we have a place to store the interrupt number. 2745 * The actual injection will happen when the CPU is able to 2746 * deliver the interrupt. 2747 */ 2748 if (kvm_cpu_has_extint(vcpu)) 2749 return false; 2750 2751 /* Acknowledging ExtINT does not happen if LINT0 is masked. */ 2752 return (!lapic_in_kernel(vcpu) || 2753 kvm_apic_accept_pic_intr(vcpu)); 2754 } 2755 2756 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu) 2757 { 2758 /* 2759 * Do not cause an interrupt window exit if an exception 2760 * is pending or an event needs reinjection; userspace 2761 * might want to inject the interrupt manually using KVM_SET_REGS 2762 * or KVM_SET_SREGS. For that to work, we must be at an 2763 * instruction boundary and with no events half-injected. 2764 */ 2765 return (kvm_is_interrupt_allowed(vcpu) && 2766 kvm_cpu_accept_dm_intr(vcpu) && 2767 !kvm_event_needs_reinjection(vcpu) && 2768 !kvm_is_exception_pending(vcpu)); 2769 } 2770 2771 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, 2772 struct kvm_interrupt *irq) 2773 { 2774 if (irq->irq >= KVM_NR_INTERRUPTS) 2775 return -EINVAL; 2776 2777 if (!irqchip_in_kernel(vcpu->kvm)) { 2778 kvm_queue_interrupt(vcpu, irq->irq, false); 2779 kvm_make_request(KVM_REQ_EVENT, vcpu); 2780 return 0; 2781 } 2782 2783 /* 2784 * With in-kernel LAPIC, we only use this to inject EXTINT, so 2785 * fail for in-kernel 8259. 2786 */ 2787 if (pic_in_kernel(vcpu->kvm)) 2788 return -ENXIO; 2789 2790 if (vcpu->arch.pending_external_vector != -1) 2791 return -EEXIST; 2792 2793 vcpu->arch.pending_external_vector = irq->irq; 2794 kvm_make_request(KVM_REQ_EVENT, vcpu); 2795 return 0; 2796 } 2797 2798 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu) 2799 { 2800 kvm_inject_nmi(vcpu); 2801 2802 return 0; 2803 } 2804 2805 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu, 2806 struct kvm_tpr_access_ctl *tac) 2807 { 2808 if (tac->flags) 2809 return -EINVAL; 2810 vcpu->arch.tpr_access_reporting = !!tac->enabled; 2811 return 0; 2812 } 2813 2814 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, 2815 u64 mcg_cap) 2816 { 2817 int r; 2818 unsigned bank_num = mcg_cap & 0xff, bank; 2819 2820 r = -EINVAL; 2821 if (!bank_num || bank_num > KVM_MAX_MCE_BANKS) 2822 goto out; 2823 if (mcg_cap & ~(kvm_caps.supported_mce_cap | 0xff | 0xff0000)) 2824 goto out; 2825 r = 0; 2826 vcpu->arch.mcg_cap = mcg_cap; 2827 /* Init IA32_MCG_CTL to all 1s */ 2828 if (mcg_cap & MCG_CTL_P) 2829 vcpu->arch.mcg_ctl = ~(u64)0; 2830 /* Init IA32_MCi_CTL to all 1s, IA32_MCi_CTL2 to all 0s */ 2831 for (bank = 0; bank < bank_num; bank++) { 2832 vcpu->arch.mce_banks[bank*4] = ~(u64)0; 2833 if (mcg_cap & MCG_CMCI_P) 2834 vcpu->arch.mci_ctl2_banks[bank] = 0; 2835 } 2836 2837 kvm_apic_after_set_mcg_cap(vcpu); 2838 2839 kvm_x86_call(setup_mce)(vcpu); 2840 out: 2841 return r; 2842 } 2843 2844 /* 2845 * Validate this is an UCNA (uncorrectable no action) error by checking the 2846 * MCG_STATUS and MCi_STATUS registers: 2847 * - none of the bits for Machine Check Exceptions are set 2848 * - both the VAL (valid) and UC (uncorrectable) bits are set 2849 * MCI_STATUS_PCC - Processor Context Corrupted 2850 * MCI_STATUS_S - Signaled as a Machine Check Exception 2851 * MCI_STATUS_AR - Software recoverable Action Required 2852 */ 2853 static bool is_ucna(struct kvm_x86_mce *mce) 2854 { 2855 return !mce->mcg_status && 2856 !(mce->status & (MCI_STATUS_PCC | MCI_STATUS_S | MCI_STATUS_AR)) && 2857 (mce->status & MCI_STATUS_VAL) && 2858 (mce->status & MCI_STATUS_UC); 2859 } 2860 2861 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks) 2862 { 2863 u64 mcg_cap = vcpu->arch.mcg_cap; 2864 2865 banks[1] = mce->status; 2866 banks[2] = mce->addr; 2867 banks[3] = mce->misc; 2868 vcpu->arch.mcg_status = mce->mcg_status; 2869 2870 if (!(mcg_cap & MCG_CMCI_P) || 2871 !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN)) 2872 return 0; 2873 2874 if (lapic_in_kernel(vcpu)) 2875 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI); 2876 2877 return 0; 2878 } 2879 2880 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, 2881 struct kvm_x86_mce *mce) 2882 { 2883 u64 mcg_cap = vcpu->arch.mcg_cap; 2884 unsigned bank_num = mcg_cap & 0xff; 2885 u64 *banks = vcpu->arch.mce_banks; 2886 2887 if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL)) 2888 return -EINVAL; 2889 2890 mce->bank = array_index_nospec(mce->bank, bank_num); 2891 banks += 4 * mce->bank; 2892 2893 if (is_ucna(mce)) 2894 return kvm_vcpu_x86_set_ucna(vcpu, mce, banks); 2895 2896 /* 2897 * if IA32_MCG_CTL is not all 1s, the uncorrected error 2898 * reporting is disabled 2899 */ 2900 if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) && 2901 vcpu->arch.mcg_ctl != ~(u64)0) 2902 return 0; 2903 /* 2904 * if IA32_MCi_CTL is not all 1s, the uncorrected error 2905 * reporting is disabled for the bank 2906 */ 2907 if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0) 2908 return 0; 2909 if (mce->status & MCI_STATUS_UC) { 2910 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) || 2911 !kvm_is_cr4_bit_set(vcpu, X86_CR4_MCE)) { 2912 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 2913 return 0; 2914 } 2915 if (banks[1] & MCI_STATUS_VAL) 2916 mce->status |= MCI_STATUS_OVER; 2917 banks[2] = mce->addr; 2918 banks[3] = mce->misc; 2919 vcpu->arch.mcg_status = mce->mcg_status; 2920 banks[1] = mce->status; 2921 kvm_queue_exception(vcpu, MC_VECTOR); 2922 } else if (!(banks[1] & MCI_STATUS_VAL) 2923 || !(banks[1] & MCI_STATUS_UC)) { 2924 if (banks[1] & MCI_STATUS_VAL) 2925 mce->status |= MCI_STATUS_OVER; 2926 banks[2] = mce->addr; 2927 banks[3] = mce->misc; 2928 banks[1] = mce->status; 2929 } else 2930 banks[1] |= MCI_STATUS_OVER; 2931 return 0; 2932 } 2933 2934 static struct kvm_queued_exception *kvm_get_exception_to_save(struct kvm_vcpu *vcpu) 2935 { 2936 /* 2937 * KVM's ABI only allows for one exception to be migrated. Luckily, 2938 * the only time there can be two queued exceptions is if there's a 2939 * non-exiting _injected_ exception, and a pending exiting exception. 2940 * In that case, ignore the VM-Exiting exception as it's an extension 2941 * of the injected exception. 2942 */ 2943 if (vcpu->arch.exception_vmexit.pending && 2944 !vcpu->arch.exception.pending && 2945 !vcpu->arch.exception.injected) 2946 return &vcpu->arch.exception_vmexit; 2947 2948 return &vcpu->arch.exception; 2949 } 2950 2951 void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu) 2952 { 2953 struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu); 2954 2955 /* 2956 * If KVM_CAP_EXCEPTION_PAYLOAD is disabled, then (prematurely) deliver 2957 * the pending exception payload when userspace saves *any* vCPU state 2958 * that interacts with exception payloads to avoid breaking userspace. 2959 * 2960 * Architecturally, KVM must not deliver an exception payload until the 2961 * exception is actually injected, e.g. to avoid losing pending #DB 2962 * information (which VMX tracks in the VMCS), and to avoid clobbering 2963 * state if the exception is never injected for whatever reason. But 2964 * if KVM_CAP_EXCEPTION_PAYLOAD isn't enabled, then userspace may or 2965 * may not propagate the payload across save+restore, and so KVM can't 2966 * safely defer delivery of the payload. 2967 */ 2968 if (!vcpu->kvm->arch.exception_payload_enabled && 2969 ex->pending && ex->has_payload) 2970 kvm_deliver_exception_payload(vcpu, ex); 2971 } 2972 2973 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, 2974 struct kvm_vcpu_events *events) 2975 { 2976 struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu); 2977 2978 process_nmi(vcpu); 2979 2980 #ifdef CONFIG_KVM_SMM 2981 if (kvm_check_request(KVM_REQ_SMI, vcpu)) 2982 process_smi(vcpu); 2983 #endif 2984 2985 kvm_handle_exception_payload_quirk(vcpu); 2986 2987 memset(events, 0, sizeof(*events)); 2988 2989 /* 2990 * The API doesn't provide the instruction length for software 2991 * exceptions, so don't report them. As long as the guest RIP 2992 * isn't advanced, we should expect to encounter the exception 2993 * again. 2994 */ 2995 if (!kvm_exception_is_soft(ex->vector)) { 2996 events->exception.injected = ex->injected; 2997 events->exception.pending = ex->pending; 2998 /* 2999 * For ABI compatibility, deliberately conflate 3000 * pending and injected exceptions when 3001 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled. 3002 */ 3003 if (!vcpu->kvm->arch.exception_payload_enabled) 3004 events->exception.injected |= ex->pending; 3005 } 3006 events->exception.nr = ex->vector; 3007 events->exception.has_error_code = ex->has_error_code; 3008 events->exception.error_code = ex->error_code; 3009 events->exception_has_payload = ex->has_payload; 3010 events->exception_payload = ex->payload; 3011 3012 events->interrupt.injected = 3013 vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft; 3014 events->interrupt.nr = vcpu->arch.interrupt.nr; 3015 events->interrupt.shadow = kvm_x86_call(get_interrupt_shadow)(vcpu); 3016 3017 events->nmi.injected = vcpu->arch.nmi_injected; 3018 events->nmi.pending = kvm_get_nr_pending_nmis(vcpu); 3019 events->nmi.masked = kvm_x86_call(get_nmi_mask)(vcpu); 3020 3021 /* events->sipi_vector is never valid when reporting to user space */ 3022 3023 #ifdef CONFIG_KVM_SMM 3024 events->smi.smm = is_smm(vcpu); 3025 events->smi.pending = vcpu->arch.smi_pending; 3026 events->smi.smm_inside_nmi = 3027 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK); 3028 #endif 3029 events->smi.latched_init = kvm_lapic_latched_init(vcpu); 3030 3031 events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING 3032 | KVM_VCPUEVENT_VALID_SHADOW 3033 | KVM_VCPUEVENT_VALID_SMM); 3034 if (vcpu->kvm->arch.exception_payload_enabled) 3035 events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD; 3036 if (vcpu->kvm->arch.triple_fault_event) { 3037 events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu); 3038 events->flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT; 3039 } 3040 } 3041 3042 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, 3043 struct kvm_vcpu_events *events) 3044 { 3045 if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING 3046 | KVM_VCPUEVENT_VALID_SIPI_VECTOR 3047 | KVM_VCPUEVENT_VALID_SHADOW 3048 | KVM_VCPUEVENT_VALID_SMM 3049 | KVM_VCPUEVENT_VALID_PAYLOAD 3050 | KVM_VCPUEVENT_VALID_TRIPLE_FAULT)) 3051 return -EINVAL; 3052 3053 if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) { 3054 if (!vcpu->kvm->arch.exception_payload_enabled) 3055 return -EINVAL; 3056 if (events->exception.pending) 3057 events->exception.injected = 0; 3058 else 3059 events->exception_has_payload = 0; 3060 } else { 3061 events->exception.pending = 0; 3062 events->exception_has_payload = 0; 3063 } 3064 3065 if ((events->exception.injected || events->exception.pending) && 3066 (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR)) 3067 return -EINVAL; 3068 3069 process_nmi(vcpu); 3070 3071 /* 3072 * Flag that userspace is stuffing an exception, the next KVM_RUN will 3073 * morph the exception to a VM-Exit if appropriate. Do this only for 3074 * pending exceptions, already-injected exceptions are not subject to 3075 * intercpetion. Note, userspace that conflates pending and injected 3076 * is hosed, and will incorrectly convert an injected exception into a 3077 * pending exception, which in turn may cause a spurious VM-Exit. 3078 */ 3079 vcpu->arch.exception_from_userspace = events->exception.pending; 3080 3081 vcpu->arch.exception_vmexit.pending = false; 3082 3083 vcpu->arch.exception.injected = events->exception.injected; 3084 vcpu->arch.exception.pending = events->exception.pending; 3085 vcpu->arch.exception.vector = events->exception.nr; 3086 vcpu->arch.exception.has_error_code = events->exception.has_error_code; 3087 vcpu->arch.exception.error_code = events->exception.error_code; 3088 vcpu->arch.exception.has_payload = events->exception_has_payload; 3089 vcpu->arch.exception.payload = events->exception_payload; 3090 3091 vcpu->arch.interrupt.injected = events->interrupt.injected; 3092 vcpu->arch.interrupt.nr = events->interrupt.nr; 3093 vcpu->arch.interrupt.soft = events->interrupt.soft; 3094 if (events->flags & KVM_VCPUEVENT_VALID_SHADOW) 3095 kvm_x86_call(set_interrupt_shadow)(vcpu, 3096 events->interrupt.shadow); 3097 3098 vcpu->arch.nmi_injected = events->nmi.injected; 3099 if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING) { 3100 vcpu->arch.nmi_pending = 0; 3101 atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending); 3102 if (events->nmi.pending) 3103 kvm_make_request(KVM_REQ_NMI, vcpu); 3104 } 3105 kvm_x86_call(set_nmi_mask)(vcpu, events->nmi.masked); 3106 3107 if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR && 3108 lapic_in_kernel(vcpu)) 3109 vcpu->arch.apic->sipi_vector = events->sipi_vector; 3110 3111 if (events->flags & KVM_VCPUEVENT_VALID_SMM) { 3112 #ifdef CONFIG_KVM_SMM 3113 if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) { 3114 kvm_leave_nested(vcpu); 3115 kvm_smm_changed(vcpu, events->smi.smm); 3116 } 3117 3118 vcpu->arch.smi_pending = events->smi.pending; 3119 3120 if (events->smi.smm) { 3121 if (events->smi.smm_inside_nmi) 3122 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK; 3123 else 3124 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK; 3125 } 3126 3127 #else 3128 if (events->smi.smm || events->smi.pending || 3129 events->smi.smm_inside_nmi) 3130 return -EINVAL; 3131 #endif 3132 3133 if (lapic_in_kernel(vcpu)) { 3134 if (events->smi.latched_init) 3135 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); 3136 else 3137 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); 3138 } 3139 } 3140 3141 if (events->flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) { 3142 if (!vcpu->kvm->arch.triple_fault_event) 3143 return -EINVAL; 3144 if (events->triple_fault.pending) 3145 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 3146 else 3147 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 3148 } 3149 3150 kvm_make_request(KVM_REQ_EVENT, vcpu); 3151 3152 return 0; 3153 } 3154 3155 static int kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu, 3156 u8 *state, unsigned int size) 3157 { 3158 /* 3159 * Only copy state for features that are enabled for the guest. The 3160 * state itself isn't problematic, but setting bits in the header for 3161 * features that are supported in *this* host but not exposed to the 3162 * guest can result in KVM_SET_XSAVE failing when live migrating to a 3163 * compatible host without the features that are NOT exposed to the 3164 * guest. 3165 * 3166 * FP+SSE can always be saved/restored via KVM_{G,S}ET_XSAVE, even if 3167 * XSAVE/XCRO are not exposed to the guest, and even if XSAVE isn't 3168 * supported by the host. 3169 */ 3170 u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 | 3171 XFEATURE_MASK_FPSSE; 3172 3173 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 3174 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 3175 3176 fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size, 3177 supported_xcr0, vcpu->arch.pkru); 3178 return 0; 3179 } 3180 3181 static int kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, 3182 struct kvm_xsave *guest_xsave) 3183 { 3184 return kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region, 3185 sizeof(guest_xsave->region)); 3186 } 3187 3188 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, 3189 struct kvm_xsave *guest_xsave) 3190 { 3191 union fpregs_state *xstate = (union fpregs_state *)guest_xsave->region; 3192 3193 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 3194 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 3195 3196 /* 3197 * For backwards compatibility, do not expect disabled features to be in 3198 * their initial state. XSTATE_BV[i] must still be cleared whenever 3199 * XFD[i]=1, or XRSTOR would cause a #NM. 3200 */ 3201 xstate->xsave.header.xfeatures &= ~vcpu->arch.guest_fpu.fpstate->xfd; 3202 3203 return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu, 3204 guest_xsave->region, 3205 kvm_caps.supported_xcr0, 3206 &vcpu->arch.pkru); 3207 } 3208 3209 static int kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu, 3210 struct kvm_xcrs *guest_xcrs) 3211 { 3212 if (vcpu->kvm->arch.has_protected_state && 3213 vcpu->arch.guest_state_protected) 3214 return -EINVAL; 3215 3216 if (!boot_cpu_has(X86_FEATURE_XSAVE)) { 3217 guest_xcrs->nr_xcrs = 0; 3218 return 0; 3219 } 3220 3221 guest_xcrs->nr_xcrs = 1; 3222 guest_xcrs->flags = 0; 3223 guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK; 3224 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0; 3225 return 0; 3226 } 3227 3228 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu, 3229 struct kvm_xcrs *guest_xcrs) 3230 { 3231 int i, r = 0; 3232 3233 if (vcpu->kvm->arch.has_protected_state && 3234 vcpu->arch.guest_state_protected) 3235 return -EINVAL; 3236 3237 if (!boot_cpu_has(X86_FEATURE_XSAVE)) 3238 return -EINVAL; 3239 3240 if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags) 3241 return -EINVAL; 3242 3243 for (i = 0; i < guest_xcrs->nr_xcrs; i++) 3244 /* Only support XCR0 currently */ 3245 if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) { 3246 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK, 3247 guest_xcrs->xcrs[i].value); 3248 break; 3249 } 3250 if (r) 3251 r = -EINVAL; 3252 return r; 3253 } 3254 3255 /* 3256 * kvm_set_guest_paused() indicates to the guest kernel that it has been 3257 * stopped by the hypervisor. This function will be called from the host only. 3258 * EINVAL is returned when the host attempts to set the flag for a guest that 3259 * does not support pv clocks. 3260 */ 3261 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) 3262 { 3263 if (!vcpu->arch.pv_time.active) 3264 return -EINVAL; 3265 vcpu->arch.pvclock_set_guest_stopped_request = true; 3266 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 3267 return 0; 3268 } 3269 3270 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu, 3271 struct kvm_device_attr *attr) 3272 { 3273 int r; 3274 3275 switch (attr->attr) { 3276 case KVM_VCPU_TSC_OFFSET: 3277 r = 0; 3278 break; 3279 default: 3280 r = -ENXIO; 3281 } 3282 3283 return r; 3284 } 3285 3286 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu, 3287 struct kvm_device_attr *attr) 3288 { 3289 u64 __user *uaddr = u64_to_user_ptr(attr->addr); 3290 int r; 3291 3292 switch (attr->attr) { 3293 case KVM_VCPU_TSC_OFFSET: 3294 r = -EFAULT; 3295 if (put_user(vcpu->arch.l1_tsc_offset, uaddr)) 3296 break; 3297 r = 0; 3298 break; 3299 default: 3300 r = -ENXIO; 3301 } 3302 3303 return r; 3304 } 3305 3306 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu, 3307 struct kvm_device_attr *attr) 3308 { 3309 u64 __user *uaddr = u64_to_user_ptr(attr->addr); 3310 struct kvm *kvm = vcpu->kvm; 3311 int r; 3312 3313 switch (attr->attr) { 3314 case KVM_VCPU_TSC_OFFSET: { 3315 u64 offset, tsc, ns; 3316 unsigned long flags; 3317 bool matched; 3318 3319 r = -EFAULT; 3320 if (get_user(offset, uaddr)) 3321 break; 3322 3323 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 3324 3325 matched = (vcpu->arch.virtual_tsc_khz && 3326 kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz && 3327 kvm->arch.last_tsc_offset == offset); 3328 3329 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset; 3330 ns = get_kvmclock_base_ns(); 3331 3332 __kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched, true); 3333 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 3334 3335 r = 0; 3336 break; 3337 } 3338 default: 3339 r = -ENXIO; 3340 } 3341 3342 return r; 3343 } 3344 3345 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu, 3346 unsigned int ioctl, 3347 void __user *argp) 3348 { 3349 struct kvm_device_attr attr; 3350 int r; 3351 3352 if (copy_from_user(&attr, argp, sizeof(attr))) 3353 return -EFAULT; 3354 3355 if (attr.group != KVM_VCPU_TSC_CTRL) 3356 return -ENXIO; 3357 3358 switch (ioctl) { 3359 case KVM_HAS_DEVICE_ATTR: 3360 r = kvm_arch_tsc_has_attr(vcpu, &attr); 3361 break; 3362 case KVM_GET_DEVICE_ATTR: 3363 r = kvm_arch_tsc_get_attr(vcpu, &attr); 3364 break; 3365 case KVM_SET_DEVICE_ATTR: 3366 r = kvm_arch_tsc_set_attr(vcpu, &attr); 3367 break; 3368 } 3369 3370 return r; 3371 } 3372 3373 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 3374 struct kvm_enable_cap *cap) 3375 { 3376 if (cap->flags) 3377 return -EINVAL; 3378 3379 switch (cap->cap) { 3380 #ifdef CONFIG_KVM_HYPERV 3381 case KVM_CAP_HYPERV_SYNIC2: 3382 if (cap->args[0]) 3383 return -EINVAL; 3384 fallthrough; 3385 3386 case KVM_CAP_HYPERV_SYNIC: 3387 if (!irqchip_in_kernel(vcpu->kvm)) 3388 return -EINVAL; 3389 return kvm_hv_activate_synic(vcpu, cap->cap == 3390 KVM_CAP_HYPERV_SYNIC2); 3391 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: 3392 { 3393 int r; 3394 uint16_t vmcs_version; 3395 void __user *user_ptr; 3396 3397 if (!kvm_nested_ops.enabled || 3398 !kvm_nested_ops.enable_evmcs) 3399 return -ENOTTY; 3400 r = kvm_nested_call(enable_evmcs)(vcpu, &vmcs_version); 3401 if (!r) { 3402 user_ptr = (void __user *)(uintptr_t)cap->args[0]; 3403 if (copy_to_user(user_ptr, &vmcs_version, 3404 sizeof(vmcs_version))) 3405 r = -EFAULT; 3406 } 3407 return r; 3408 } 3409 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: 3410 if (!kvm_x86_ops.enable_l2_tlb_flush) 3411 return -ENOTTY; 3412 3413 return kvm_x86_call(enable_l2_tlb_flush)(vcpu); 3414 3415 case KVM_CAP_HYPERV_ENFORCE_CPUID: 3416 return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]); 3417 #endif 3418 3419 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID: 3420 vcpu->arch.pv_cpuid.enforce = cap->args[0]; 3421 return 0; 3422 default: 3423 return -EINVAL; 3424 } 3425 } 3426 3427 long kvm_arch_vcpu_ioctl(struct file *filp, 3428 unsigned int ioctl, unsigned long arg) 3429 { 3430 struct kvm_vcpu *vcpu = filp->private_data; 3431 void __user *argp = (void __user *)arg; 3432 int r; 3433 union { 3434 struct kvm_sregs2 *sregs2; 3435 struct kvm_lapic_state *lapic; 3436 struct kvm_xsave *xsave; 3437 struct kvm_xcrs *xcrs; 3438 void *buffer; 3439 } u; 3440 3441 vcpu_load(vcpu); 3442 3443 u.buffer = NULL; 3444 switch (ioctl) { 3445 case KVM_GET_LAPIC: { 3446 r = -EINVAL; 3447 if (!lapic_in_kernel(vcpu)) 3448 goto out; 3449 u.lapic = kzalloc_obj(struct kvm_lapic_state); 3450 3451 r = -ENOMEM; 3452 if (!u.lapic) 3453 goto out; 3454 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic); 3455 if (r) 3456 goto out; 3457 r = -EFAULT; 3458 if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state))) 3459 goto out; 3460 r = 0; 3461 break; 3462 } 3463 case KVM_SET_LAPIC: { 3464 r = -EINVAL; 3465 if (!lapic_in_kernel(vcpu)) 3466 goto out; 3467 u.lapic = memdup_user(argp, sizeof(*u.lapic)); 3468 if (IS_ERR(u.lapic)) { 3469 r = PTR_ERR(u.lapic); 3470 goto out_nofree; 3471 } 3472 3473 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic); 3474 break; 3475 } 3476 case KVM_INTERRUPT: { 3477 struct kvm_interrupt irq; 3478 3479 r = -EFAULT; 3480 if (copy_from_user(&irq, argp, sizeof(irq))) 3481 goto out; 3482 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); 3483 break; 3484 } 3485 case KVM_NMI: { 3486 r = kvm_vcpu_ioctl_nmi(vcpu); 3487 break; 3488 } 3489 case KVM_SMI: { 3490 r = kvm_inject_smi(vcpu); 3491 break; 3492 } 3493 case KVM_SET_CPUID: { 3494 struct kvm_cpuid __user *cpuid_arg = argp; 3495 struct kvm_cpuid cpuid; 3496 3497 r = -EFAULT; 3498 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 3499 goto out; 3500 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); 3501 break; 3502 } 3503 case KVM_SET_CPUID2: { 3504 struct kvm_cpuid2 __user *cpuid_arg = argp; 3505 struct kvm_cpuid2 cpuid; 3506 3507 r = -EFAULT; 3508 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 3509 goto out; 3510 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, 3511 cpuid_arg->entries); 3512 break; 3513 } 3514 case KVM_GET_CPUID2: { 3515 struct kvm_cpuid2 __user *cpuid_arg = argp; 3516 struct kvm_cpuid2 cpuid; 3517 3518 r = -EFAULT; 3519 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 3520 goto out; 3521 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, 3522 cpuid_arg->entries); 3523 if (r) 3524 goto out; 3525 r = -EFAULT; 3526 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid))) 3527 goto out; 3528 r = 0; 3529 break; 3530 } 3531 case KVM_GET_MSRS: 3532 r = kvm_get_msrs(vcpu, argp); 3533 break; 3534 case KVM_SET_MSRS: 3535 r = kvm_set_msrs(vcpu, argp); 3536 break; 3537 case KVM_GET_ONE_REG: 3538 case KVM_SET_ONE_REG: 3539 r = kvm_get_set_one_reg(vcpu, ioctl, argp); 3540 break; 3541 case KVM_GET_REG_LIST: 3542 r = kvm_get_reg_list(vcpu, argp); 3543 break; 3544 case KVM_TPR_ACCESS_REPORTING: { 3545 struct kvm_tpr_access_ctl tac; 3546 3547 r = -EFAULT; 3548 if (copy_from_user(&tac, argp, sizeof(tac))) 3549 goto out; 3550 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac); 3551 if (r) 3552 goto out; 3553 r = -EFAULT; 3554 if (copy_to_user(argp, &tac, sizeof(tac))) 3555 goto out; 3556 r = 0; 3557 break; 3558 }; 3559 case KVM_SET_VAPIC_ADDR: { 3560 struct kvm_vapic_addr va; 3561 int idx; 3562 3563 r = -EINVAL; 3564 if (!lapic_in_kernel(vcpu)) 3565 goto out; 3566 r = -EFAULT; 3567 if (copy_from_user(&va, argp, sizeof(va))) 3568 goto out; 3569 idx = srcu_read_lock(&vcpu->kvm->srcu); 3570 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr); 3571 srcu_read_unlock(&vcpu->kvm->srcu, idx); 3572 break; 3573 } 3574 case KVM_X86_SETUP_MCE: { 3575 u64 mcg_cap; 3576 3577 r = -EFAULT; 3578 if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap))) 3579 goto out; 3580 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap); 3581 break; 3582 } 3583 case KVM_X86_SET_MCE: { 3584 struct kvm_x86_mce mce; 3585 3586 r = -EFAULT; 3587 if (copy_from_user(&mce, argp, sizeof(mce))) 3588 goto out; 3589 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce); 3590 break; 3591 } 3592 case KVM_GET_VCPU_EVENTS: { 3593 struct kvm_vcpu_events events; 3594 3595 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events); 3596 3597 r = -EFAULT; 3598 if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events))) 3599 break; 3600 r = 0; 3601 break; 3602 } 3603 case KVM_SET_VCPU_EVENTS: { 3604 struct kvm_vcpu_events events; 3605 3606 r = -EFAULT; 3607 if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events))) 3608 break; 3609 3610 kvm_vcpu_srcu_read_lock(vcpu); 3611 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events); 3612 kvm_vcpu_srcu_read_unlock(vcpu); 3613 break; 3614 } 3615 case KVM_GET_DEBUGREGS: { 3616 struct kvm_debugregs dbgregs; 3617 3618 r = kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs); 3619 if (r < 0) 3620 break; 3621 3622 r = -EFAULT; 3623 if (copy_to_user(argp, &dbgregs, 3624 sizeof(struct kvm_debugregs))) 3625 break; 3626 r = 0; 3627 break; 3628 } 3629 case KVM_SET_DEBUGREGS: { 3630 struct kvm_debugregs dbgregs; 3631 3632 r = -EFAULT; 3633 if (copy_from_user(&dbgregs, argp, 3634 sizeof(struct kvm_debugregs))) 3635 break; 3636 3637 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs); 3638 break; 3639 } 3640 case KVM_GET_XSAVE: { 3641 r = -EINVAL; 3642 if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave)) 3643 break; 3644 3645 u.xsave = kzalloc_obj(struct kvm_xsave); 3646 r = -ENOMEM; 3647 if (!u.xsave) 3648 break; 3649 3650 r = kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave); 3651 if (r < 0) 3652 break; 3653 3654 r = -EFAULT; 3655 if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave))) 3656 break; 3657 r = 0; 3658 break; 3659 } 3660 case KVM_SET_XSAVE: { 3661 int size = vcpu->arch.guest_fpu.uabi_size; 3662 3663 u.xsave = memdup_user(argp, size); 3664 if (IS_ERR(u.xsave)) { 3665 r = PTR_ERR(u.xsave); 3666 goto out_nofree; 3667 } 3668 3669 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave); 3670 break; 3671 } 3672 3673 case KVM_GET_XSAVE2: { 3674 int size = vcpu->arch.guest_fpu.uabi_size; 3675 3676 u.xsave = kzalloc(size, GFP_KERNEL); 3677 r = -ENOMEM; 3678 if (!u.xsave) 3679 break; 3680 3681 r = kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size); 3682 if (r < 0) 3683 break; 3684 3685 r = -EFAULT; 3686 if (copy_to_user(argp, u.xsave, size)) 3687 break; 3688 3689 r = 0; 3690 break; 3691 } 3692 3693 case KVM_GET_XCRS: { 3694 u.xcrs = kzalloc_obj(struct kvm_xcrs); 3695 r = -ENOMEM; 3696 if (!u.xcrs) 3697 break; 3698 3699 r = kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs); 3700 if (r < 0) 3701 break; 3702 3703 r = -EFAULT; 3704 if (copy_to_user(argp, u.xcrs, 3705 sizeof(struct kvm_xcrs))) 3706 break; 3707 r = 0; 3708 break; 3709 } 3710 case KVM_SET_XCRS: { 3711 u.xcrs = memdup_user(argp, sizeof(*u.xcrs)); 3712 if (IS_ERR(u.xcrs)) { 3713 r = PTR_ERR(u.xcrs); 3714 goto out_nofree; 3715 } 3716 3717 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs); 3718 break; 3719 } 3720 case KVM_SET_TSC_KHZ: { 3721 u32 user_tsc_khz; 3722 3723 r = -EINVAL; 3724 3725 if (vcpu->arch.guest_tsc_protected) 3726 goto out; 3727 3728 user_tsc_khz = (u32)arg; 3729 3730 if (kvm_caps.has_tsc_control && 3731 user_tsc_khz > kvm_caps.max_guest_tsc_khz) 3732 goto out; 3733 3734 if (user_tsc_khz == 0) 3735 user_tsc_khz = tsc_khz; 3736 3737 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz)) 3738 r = 0; 3739 3740 goto out; 3741 } 3742 case KVM_GET_TSC_KHZ: { 3743 r = vcpu->arch.virtual_tsc_khz; 3744 goto out; 3745 } 3746 case KVM_KVMCLOCK_CTRL: { 3747 r = kvm_set_guest_paused(vcpu); 3748 goto out; 3749 } 3750 case KVM_ENABLE_CAP: { 3751 struct kvm_enable_cap cap; 3752 3753 r = -EFAULT; 3754 if (copy_from_user(&cap, argp, sizeof(cap))) 3755 goto out; 3756 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 3757 break; 3758 } 3759 case KVM_GET_NESTED_STATE: { 3760 struct kvm_nested_state __user *user_kvm_nested_state = argp; 3761 u32 user_data_size; 3762 3763 r = -EINVAL; 3764 if (!kvm_nested_ops.enabled) 3765 break; 3766 3767 BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size)); 3768 r = -EFAULT; 3769 if (get_user(user_data_size, &user_kvm_nested_state->size)) 3770 break; 3771 3772 r = kvm_nested_call(get_state)(vcpu, user_kvm_nested_state, user_data_size); 3773 if (r < 0) 3774 break; 3775 3776 if (r > user_data_size) { 3777 if (put_user(r, &user_kvm_nested_state->size)) 3778 r = -EFAULT; 3779 else 3780 r = -E2BIG; 3781 break; 3782 } 3783 3784 r = 0; 3785 break; 3786 } 3787 case KVM_SET_NESTED_STATE: { 3788 struct kvm_nested_state __user *user_kvm_nested_state = argp; 3789 struct kvm_nested_state kvm_state; 3790 int idx; 3791 3792 r = -EINVAL; 3793 if (!kvm_nested_ops.enabled) 3794 break; 3795 3796 r = -EFAULT; 3797 if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state))) 3798 break; 3799 3800 r = -EINVAL; 3801 if (kvm_state.size < sizeof(kvm_state)) 3802 break; 3803 3804 if (kvm_state.flags & 3805 ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE 3806 | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING 3807 | KVM_STATE_NESTED_GIF_SET)) 3808 break; 3809 3810 /* nested_run_pending implies guest_mode. */ 3811 if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING) 3812 && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE)) 3813 break; 3814 3815 idx = srcu_read_lock(&vcpu->kvm->srcu); 3816 r = kvm_nested_call(set_state)(vcpu, user_kvm_nested_state, &kvm_state); 3817 srcu_read_unlock(&vcpu->kvm->srcu, idx); 3818 break; 3819 } 3820 #ifdef CONFIG_KVM_HYPERV 3821 case KVM_GET_SUPPORTED_HV_CPUID: 3822 r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp); 3823 break; 3824 #endif 3825 #ifdef CONFIG_KVM_XEN 3826 case KVM_XEN_VCPU_GET_ATTR: { 3827 struct kvm_xen_vcpu_attr xva; 3828 3829 r = -EFAULT; 3830 if (copy_from_user(&xva, argp, sizeof(xva))) 3831 goto out; 3832 r = kvm_xen_vcpu_get_attr(vcpu, &xva); 3833 if (!r && copy_to_user(argp, &xva, sizeof(xva))) 3834 r = -EFAULT; 3835 break; 3836 } 3837 case KVM_XEN_VCPU_SET_ATTR: { 3838 struct kvm_xen_vcpu_attr xva; 3839 3840 r = -EFAULT; 3841 if (copy_from_user(&xva, argp, sizeof(xva))) 3842 goto out; 3843 r = kvm_xen_vcpu_set_attr(vcpu, &xva); 3844 break; 3845 } 3846 #endif 3847 case KVM_GET_SREGS2: { 3848 r = -EINVAL; 3849 if (vcpu->kvm->arch.has_protected_state && 3850 vcpu->arch.guest_state_protected) 3851 goto out; 3852 3853 u.sregs2 = kzalloc_obj(struct kvm_sregs2); 3854 r = -ENOMEM; 3855 if (!u.sregs2) 3856 goto out; 3857 kvm_vcpu_ioctl_x86_get_sregs2(vcpu, u.sregs2); 3858 r = -EFAULT; 3859 if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2))) 3860 goto out; 3861 r = 0; 3862 break; 3863 } 3864 case KVM_SET_SREGS2: { 3865 r = -EINVAL; 3866 if (vcpu->kvm->arch.has_protected_state && 3867 vcpu->arch.guest_state_protected) 3868 goto out; 3869 3870 u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2)); 3871 if (IS_ERR(u.sregs2)) { 3872 r = PTR_ERR(u.sregs2); 3873 u.sregs2 = NULL; 3874 goto out; 3875 } 3876 r = kvm_vcpu_ioctl_x86_set_sregs2(vcpu, u.sregs2); 3877 break; 3878 } 3879 case KVM_HAS_DEVICE_ATTR: 3880 case KVM_GET_DEVICE_ATTR: 3881 case KVM_SET_DEVICE_ATTR: 3882 r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp); 3883 break; 3884 case KVM_MEMORY_ENCRYPT_OP: 3885 r = -ENOTTY; 3886 if (!kvm_x86_ops.vcpu_mem_enc_ioctl) 3887 goto out; 3888 r = kvm_x86_ops.vcpu_mem_enc_ioctl(vcpu, argp); 3889 break; 3890 default: 3891 r = -EINVAL; 3892 } 3893 out: 3894 kfree(u.buffer); 3895 out_nofree: 3896 vcpu_put(vcpu); 3897 return r; 3898 } 3899 3900 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 3901 { 3902 return VM_FAULT_SIGBUS; 3903 } 3904 3905 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) 3906 { 3907 int ret; 3908 3909 if (addr > (unsigned int)(-3 * PAGE_SIZE)) 3910 return -EINVAL; 3911 ret = kvm_x86_call(set_tss_addr)(kvm, addr); 3912 return ret; 3913 } 3914 3915 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm, 3916 u64 ident_addr) 3917 { 3918 return kvm_x86_call(set_identity_map_addr)(kvm, ident_addr); 3919 } 3920 3921 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, 3922 unsigned long kvm_nr_mmu_pages) 3923 { 3924 if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) 3925 return -EINVAL; 3926 3927 mutex_lock(&kvm->slots_lock); 3928 3929 kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); 3930 kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; 3931 3932 mutex_unlock(&kvm->slots_lock); 3933 return 0; 3934 } 3935 3936 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 3937 { 3938 3939 /* 3940 * Flush all CPUs' dirty log buffers to the dirty_bitmap. Called 3941 * before reporting dirty_bitmap to userspace. KVM flushes the buffers 3942 * on all VM-Exits, thus we only need to kick running vCPUs to force a 3943 * VM-Exit. 3944 */ 3945 struct kvm_vcpu *vcpu; 3946 unsigned long i; 3947 3948 if (!kvm->arch.cpu_dirty_log_size) 3949 return; 3950 3951 kvm_for_each_vcpu(i, vcpu, kvm) 3952 kvm_vcpu_kick(vcpu); 3953 } 3954 3955 void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx) 3956 { 3957 u16 pml_tail_index; 3958 u64 *pml_buf; 3959 int i; 3960 3961 /* 3962 * PML index always points to the next available PML buffer entity 3963 * unless PML log has just overflowed. 3964 */ 3965 pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; 3966 3967 /* 3968 * PML log is written backwards: the CPU first writes the entry 511 3969 * then the entry 510, and so on. 3970 * 3971 * Read the entries in the same order they were written, to ensure that 3972 * the dirty ring is filled in the same order the CPU wrote them. 3973 */ 3974 pml_buf = page_address(vcpu->arch.pml_page); 3975 3976 for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { 3977 u64 gpa; 3978 3979 gpa = pml_buf[i]; 3980 WARN_ON_ONCE(gpa & (PAGE_SIZE - 1)); 3981 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); 3982 } 3983 } 3984 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_flush_pml_buffer); 3985 3986 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 3987 struct kvm_enable_cap *cap) 3988 { 3989 int r; 3990 3991 if (cap->flags) 3992 return -EINVAL; 3993 3994 switch (cap->cap) { 3995 case KVM_CAP_DISABLE_QUIRKS2: 3996 r = -EINVAL; 3997 if (cap->args[0] & ~kvm_caps.supported_quirks) 3998 break; 3999 fallthrough; 4000 case KVM_CAP_DISABLE_QUIRKS: 4001 mutex_lock(&kvm->lock); 4002 WRITE_ONCE(kvm->arch.disabled_quirks, 4003 kvm->arch.disabled_quirks | (cap->args[0] & kvm_caps.supported_quirks)); 4004 mutex_unlock(&kvm->lock); 4005 r = 0; 4006 break; 4007 case KVM_CAP_SPLIT_IRQCHIP: { 4008 mutex_lock(&kvm->lock); 4009 r = -EINVAL; 4010 if (cap->args[0] > KVM_MAX_IRQ_ROUTES) 4011 goto split_irqchip_unlock; 4012 r = -EEXIST; 4013 if (irqchip_in_kernel(kvm)) 4014 goto split_irqchip_unlock; 4015 if (kvm->created_vcpus) 4016 goto split_irqchip_unlock; 4017 /* Pairs with irqchip_in_kernel. */ 4018 smp_wmb(); 4019 kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT; 4020 kvm->arch.nr_reserved_ioapic_pins = cap->args[0]; 4021 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT); 4022 r = 0; 4023 split_irqchip_unlock: 4024 mutex_unlock(&kvm->lock); 4025 break; 4026 } 4027 case KVM_CAP_X2APIC_API: 4028 r = -EINVAL; 4029 if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS) 4030 break; 4031 4032 if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && 4033 (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)) 4034 break; 4035 4036 if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && 4037 !irqchip_split(kvm)) 4038 break; 4039 4040 if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS) 4041 kvm->arch.x2apic_format = true; 4042 if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) 4043 kvm->arch.x2apic_broadcast_quirk_disabled = true; 4044 4045 if (cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) 4046 kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_ENABLED; 4047 if (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) 4048 kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_DISABLED; 4049 4050 r = 0; 4051 break; 4052 case KVM_CAP_X86_DISABLE_EXITS: 4053 r = -EINVAL; 4054 if (cap->args[0] & ~kvm_get_allowed_disable_exits()) 4055 break; 4056 4057 mutex_lock(&kvm->lock); 4058 if (kvm->created_vcpus) 4059 goto disable_exits_unlock; 4060 4061 #define SMT_RSB_MSG "This processor is affected by the Cross-Thread Return Predictions vulnerability. " \ 4062 "KVM_CAP_X86_DISABLE_EXITS should only be used with SMT disabled or trusted guests." 4063 4064 if (!mitigate_smt_rsb && boot_cpu_has_bug(X86_BUG_SMT_RSB) && 4065 cpu_smt_possible() && 4066 (cap->args[0] & ~(KVM_X86_DISABLE_EXITS_PAUSE | 4067 KVM_X86_DISABLE_EXITS_APERFMPERF))) 4068 pr_warn_once(SMT_RSB_MSG); 4069 4070 kvm_disable_exits(kvm, cap->args[0]); 4071 r = 0; 4072 disable_exits_unlock: 4073 mutex_unlock(&kvm->lock); 4074 break; 4075 case KVM_CAP_MSR_PLATFORM_INFO: 4076 kvm->arch.guest_can_read_msr_platform_info = cap->args[0]; 4077 r = 0; 4078 break; 4079 case KVM_CAP_EXCEPTION_PAYLOAD: 4080 kvm->arch.exception_payload_enabled = cap->args[0]; 4081 r = 0; 4082 break; 4083 case KVM_CAP_X86_TRIPLE_FAULT_EVENT: 4084 kvm->arch.triple_fault_event = cap->args[0]; 4085 r = 0; 4086 break; 4087 case KVM_CAP_X86_USER_SPACE_MSR: 4088 r = -EINVAL; 4089 if (cap->args[0] & ~KVM_MSR_EXIT_REASON_VALID_MASK) 4090 break; 4091 kvm->arch.user_space_msr_mask = cap->args[0]; 4092 r = 0; 4093 break; 4094 case KVM_CAP_X86_BUS_LOCK_EXIT: 4095 r = -EINVAL; 4096 if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE) 4097 break; 4098 4099 if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) && 4100 (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)) 4101 break; 4102 4103 if (kvm_caps.has_bus_lock_exit && 4104 cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT) 4105 kvm->arch.bus_lock_detection_enabled = true; 4106 r = 0; 4107 break; 4108 #ifdef CONFIG_X86_SGX_KVM 4109 case KVM_CAP_SGX_ATTRIBUTE: { 4110 unsigned long allowed_attributes = 0; 4111 4112 r = sgx_set_attribute(&allowed_attributes, cap->args[0]); 4113 if (r) 4114 break; 4115 4116 /* KVM only supports the PROVISIONKEY privileged attribute. */ 4117 if ((allowed_attributes & SGX_ATTR_PROVISIONKEY) && 4118 !(allowed_attributes & ~SGX_ATTR_PROVISIONKEY)) 4119 kvm->arch.sgx_provisioning_allowed = true; 4120 else 4121 r = -EINVAL; 4122 break; 4123 } 4124 #endif 4125 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: 4126 r = -EINVAL; 4127 if (!kvm_x86_ops.vm_copy_enc_context_from) 4128 break; 4129 4130 r = kvm_x86_call(vm_copy_enc_context_from)(kvm, cap->args[0]); 4131 break; 4132 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM: 4133 r = -EINVAL; 4134 if (!kvm_x86_ops.vm_move_enc_context_from) 4135 break; 4136 4137 r = kvm_x86_call(vm_move_enc_context_from)(kvm, cap->args[0]); 4138 break; 4139 case KVM_CAP_EXIT_HYPERCALL: 4140 if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) { 4141 r = -EINVAL; 4142 break; 4143 } 4144 kvm->arch.hypercall_exit_enabled = cap->args[0]; 4145 r = 0; 4146 break; 4147 case KVM_CAP_EXIT_ON_EMULATION_FAILURE: 4148 r = -EINVAL; 4149 if (cap->args[0] & ~1) 4150 break; 4151 kvm->arch.exit_on_emulation_error = cap->args[0]; 4152 r = 0; 4153 break; 4154 case KVM_CAP_PMU_CAPABILITY: 4155 r = -EINVAL; 4156 if (!enable_pmu || (cap->args[0] & ~KVM_CAP_PMU_VALID_MASK)) 4157 break; 4158 4159 if (kvm->arch.has_protected_pmu && 4160 cap->args[0] != KVM_PMU_CAP_DISABLE) 4161 break; 4162 4163 mutex_lock(&kvm->lock); 4164 if (!kvm->created_vcpus && !kvm->arch.created_mediated_pmu) { 4165 kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE); 4166 r = 0; 4167 } 4168 mutex_unlock(&kvm->lock); 4169 break; 4170 case KVM_CAP_MAX_VCPU_ID: 4171 r = -EINVAL; 4172 if (cap->args[0] > KVM_MAX_VCPU_IDS) 4173 break; 4174 4175 mutex_lock(&kvm->lock); 4176 if (kvm->arch.bsp_vcpu_id > cap->args[0]) { 4177 ; 4178 } else if (kvm->arch.max_vcpu_ids == cap->args[0]) { 4179 r = 0; 4180 } else if (!kvm->arch.max_vcpu_ids) { 4181 kvm->arch.max_vcpu_ids = cap->args[0]; 4182 r = 0; 4183 } 4184 mutex_unlock(&kvm->lock); 4185 break; 4186 case KVM_CAP_X86_NOTIFY_VMEXIT: 4187 r = -EINVAL; 4188 if ((u32)cap->args[0] & ~KVM_X86_NOTIFY_VMEXIT_VALID_BITS) 4189 break; 4190 if (!kvm_caps.has_notify_vmexit) 4191 break; 4192 if (!((u32)cap->args[0] & KVM_X86_NOTIFY_VMEXIT_ENABLED)) 4193 break; 4194 mutex_lock(&kvm->lock); 4195 if (!kvm->created_vcpus) { 4196 kvm->arch.notify_window = cap->args[0] >> 32; 4197 kvm->arch.notify_vmexit_flags = (u32)cap->args[0]; 4198 r = 0; 4199 } 4200 mutex_unlock(&kvm->lock); 4201 break; 4202 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES: 4203 r = -EINVAL; 4204 4205 /* 4206 * Since the risk of disabling NX hugepages is a guest crashing 4207 * the system, ensure the userspace process has permission to 4208 * reboot the system. 4209 * 4210 * Note that unlike the reboot() syscall, the process must have 4211 * this capability in the root namespace because exposing 4212 * /dev/kvm into a container does not limit the scope of the 4213 * iTLB multihit bug to that container. In other words, 4214 * this must use capable(), not ns_capable(). 4215 */ 4216 if (!capable(CAP_SYS_BOOT)) { 4217 r = -EPERM; 4218 break; 4219 } 4220 4221 if (cap->args[0]) 4222 break; 4223 4224 mutex_lock(&kvm->lock); 4225 if (!kvm->created_vcpus) { 4226 kvm->arch.disable_nx_huge_pages = true; 4227 r = 0; 4228 } 4229 mutex_unlock(&kvm->lock); 4230 break; 4231 case KVM_CAP_X86_APIC_BUS_CYCLES_NS: { 4232 u64 bus_cycle_ns = cap->args[0]; 4233 u64 unused; 4234 4235 /* 4236 * Guard against overflow in tmict_to_ns(). 128 is the highest 4237 * divide value that can be programmed in APIC_TDCR. 4238 */ 4239 r = -EINVAL; 4240 if (!bus_cycle_ns || 4241 check_mul_overflow((u64)U32_MAX * 128, bus_cycle_ns, &unused)) 4242 break; 4243 4244 r = 0; 4245 mutex_lock(&kvm->lock); 4246 if (!irqchip_in_kernel(kvm)) 4247 r = -ENXIO; 4248 else if (kvm->created_vcpus) 4249 r = -EINVAL; 4250 else 4251 kvm->arch.apic_bus_cycle_ns = bus_cycle_ns; 4252 mutex_unlock(&kvm->lock); 4253 break; 4254 } 4255 default: 4256 r = -EINVAL; 4257 break; 4258 } 4259 return r; 4260 } 4261 4262 #ifdef CONFIG_KVM_COMPAT 4263 /* for KVM_X86_SET_MSR_FILTER */ 4264 struct kvm_msr_filter_range_compat { 4265 __u32 flags; 4266 __u32 nmsrs; 4267 __u32 base; 4268 __u32 bitmap; 4269 }; 4270 4271 struct kvm_msr_filter_compat { 4272 __u32 flags; 4273 struct kvm_msr_filter_range_compat ranges[KVM_MSR_FILTER_MAX_RANGES]; 4274 }; 4275 4276 #define KVM_X86_SET_MSR_FILTER_COMPAT _IOW(KVMIO, 0xc6, struct kvm_msr_filter_compat) 4277 4278 long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl, 4279 unsigned long arg) 4280 { 4281 void __user *argp = (void __user *)arg; 4282 struct kvm *kvm = filp->private_data; 4283 long r = -ENOTTY; 4284 4285 switch (ioctl) { 4286 case KVM_X86_SET_MSR_FILTER_COMPAT: { 4287 struct kvm_msr_filter __user *user_msr_filter = argp; 4288 struct kvm_msr_filter_compat filter_compat; 4289 struct kvm_msr_filter filter; 4290 int i; 4291 4292 if (copy_from_user(&filter_compat, user_msr_filter, 4293 sizeof(filter_compat))) 4294 return -EFAULT; 4295 4296 filter.flags = filter_compat.flags; 4297 for (i = 0; i < ARRAY_SIZE(filter.ranges); i++) { 4298 struct kvm_msr_filter_range_compat *cr; 4299 4300 cr = &filter_compat.ranges[i]; 4301 filter.ranges[i] = (struct kvm_msr_filter_range) { 4302 .flags = cr->flags, 4303 .nmsrs = cr->nmsrs, 4304 .base = cr->base, 4305 .bitmap = (__u8 *)(ulong)cr->bitmap, 4306 }; 4307 } 4308 4309 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter); 4310 break; 4311 } 4312 } 4313 4314 return r; 4315 } 4316 #endif 4317 4318 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER 4319 static int kvm_arch_suspend_notifier(struct kvm *kvm) 4320 { 4321 struct kvm_vcpu *vcpu; 4322 unsigned long i; 4323 4324 /* 4325 * Ignore the return, marking the guest paused only "fails" if the vCPU 4326 * isn't using kvmclock; continuing on is correct and desirable. 4327 */ 4328 kvm_for_each_vcpu(i, vcpu, kvm) 4329 (void)kvm_set_guest_paused(vcpu); 4330 4331 return NOTIFY_DONE; 4332 } 4333 4334 int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state) 4335 { 4336 switch (state) { 4337 case PM_HIBERNATION_PREPARE: 4338 case PM_SUSPEND_PREPARE: 4339 return kvm_arch_suspend_notifier(kvm); 4340 } 4341 4342 return NOTIFY_DONE; 4343 } 4344 #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */ 4345 4346 static int kvm_vm_ioctl_get_clock(struct kvm *kvm, void __user *argp) 4347 { 4348 struct kvm_clock_data data = { 0 }; 4349 4350 get_kvmclock(kvm, &data); 4351 if (copy_to_user(argp, &data, sizeof(data))) 4352 return -EFAULT; 4353 4354 return 0; 4355 } 4356 4357 static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp) 4358 { 4359 struct kvm_arch *ka = &kvm->arch; 4360 struct kvm_clock_data data; 4361 u64 now_raw_ns; 4362 4363 if (copy_from_user(&data, argp, sizeof(data))) 4364 return -EFAULT; 4365 4366 /* 4367 * Only KVM_CLOCK_REALTIME is used, but allow passing the 4368 * result of KVM_GET_CLOCK back to KVM_SET_CLOCK. 4369 */ 4370 if (data.flags & ~KVM_CLOCK_VALID_FLAGS) 4371 return -EINVAL; 4372 4373 kvm_hv_request_tsc_page_update(kvm); 4374 kvm_start_pvclock_update(kvm); 4375 pvclock_update_vm_gtod_copy(kvm); 4376 4377 /* 4378 * This pairs with kvm_guest_time_update(): when masterclock is 4379 * in use, we use master_kernel_ns + kvmclock_offset to set 4380 * unsigned 'system_time' so if we use get_kvmclock_ns() (which 4381 * is slightly ahead) here we risk going negative on unsigned 4382 * 'system_time' when 'data.clock' is very small. 4383 */ 4384 if (data.flags & KVM_CLOCK_REALTIME) { 4385 u64 now_real_ns = ktime_get_real_ns(); 4386 4387 /* 4388 * Avoid stepping the kvmclock backwards. 4389 */ 4390 if (now_real_ns > data.realtime) 4391 data.clock += now_real_ns - data.realtime; 4392 } 4393 4394 if (ka->use_master_clock) 4395 now_raw_ns = ka->master_kernel_ns; 4396 else 4397 now_raw_ns = get_kvmclock_base_ns(); 4398 ka->kvmclock_offset = data.clock - now_raw_ns; 4399 kvm_end_pvclock_update(kvm); 4400 return 0; 4401 } 4402 4403 long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, 4404 unsigned long arg) 4405 { 4406 struct kvm_vcpu *vcpu = filp->private_data; 4407 void __user *argp = (void __user *)arg; 4408 4409 if (ioctl == KVM_MEMORY_ENCRYPT_OP && 4410 kvm_x86_ops.vcpu_mem_enc_unlocked_ioctl) 4411 return kvm_x86_call(vcpu_mem_enc_unlocked_ioctl)(vcpu, argp); 4412 4413 return -ENOIOCTLCMD; 4414 } 4415 4416 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) 4417 { 4418 struct kvm *kvm = filp->private_data; 4419 void __user *argp = (void __user *)arg; 4420 int r = -ENOTTY; 4421 4422 #ifdef CONFIG_KVM_IOAPIC 4423 /* 4424 * This union makes it completely explicit to gcc-3.x 4425 * that these three variables' stack usage should be 4426 * combined, not added together. 4427 */ 4428 union { 4429 struct kvm_pit_state ps; 4430 struct kvm_pit_state2 ps2; 4431 struct kvm_pit_config pit_config; 4432 } u; 4433 #endif 4434 4435 switch (ioctl) { 4436 case KVM_SET_TSS_ADDR: 4437 r = kvm_vm_ioctl_set_tss_addr(kvm, arg); 4438 break; 4439 case KVM_SET_IDENTITY_MAP_ADDR: { 4440 u64 ident_addr; 4441 4442 mutex_lock(&kvm->lock); 4443 r = -EINVAL; 4444 if (kvm->created_vcpus) 4445 goto set_identity_unlock; 4446 r = -EFAULT; 4447 if (copy_from_user(&ident_addr, argp, sizeof(ident_addr))) 4448 goto set_identity_unlock; 4449 r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr); 4450 set_identity_unlock: 4451 mutex_unlock(&kvm->lock); 4452 break; 4453 } 4454 case KVM_SET_NR_MMU_PAGES: 4455 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); 4456 break; 4457 #ifdef CONFIG_KVM_IOAPIC 4458 case KVM_CREATE_IRQCHIP: { 4459 mutex_lock(&kvm->lock); 4460 4461 r = -EEXIST; 4462 if (irqchip_in_kernel(kvm)) 4463 goto create_irqchip_unlock; 4464 4465 /* 4466 * Disallow an in-kernel I/O APIC if the VM has protected EOIs, 4467 * i.e. if KVM can't intercept EOIs and thus can't properly 4468 * emulate level-triggered interrupts. 4469 */ 4470 r = -ENOTTY; 4471 if (kvm->arch.has_protected_eoi) 4472 goto create_irqchip_unlock; 4473 4474 r = -EINVAL; 4475 if (kvm->created_vcpus) 4476 goto create_irqchip_unlock; 4477 4478 r = kvm_pic_init(kvm); 4479 if (r) 4480 goto create_irqchip_unlock; 4481 4482 r = kvm_ioapic_init(kvm); 4483 if (r) { 4484 kvm_pic_destroy(kvm); 4485 goto create_irqchip_unlock; 4486 } 4487 4488 r = kvm_setup_default_ioapic_and_pic_routing(kvm); 4489 if (r) { 4490 kvm_ioapic_destroy(kvm); 4491 kvm_pic_destroy(kvm); 4492 goto create_irqchip_unlock; 4493 } 4494 /* Write kvm->irq_routing before enabling irqchip_in_kernel. */ 4495 smp_wmb(); 4496 kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL; 4497 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT); 4498 create_irqchip_unlock: 4499 mutex_unlock(&kvm->lock); 4500 break; 4501 } 4502 case KVM_CREATE_PIT: 4503 u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY; 4504 goto create_pit; 4505 case KVM_CREATE_PIT2: 4506 r = -EFAULT; 4507 if (copy_from_user(&u.pit_config, argp, 4508 sizeof(struct kvm_pit_config))) 4509 goto out; 4510 create_pit: 4511 mutex_lock(&kvm->lock); 4512 r = -EEXIST; 4513 if (kvm->arch.vpit) 4514 goto create_pit_unlock; 4515 r = -ENOENT; 4516 if (!pic_in_kernel(kvm)) 4517 goto create_pit_unlock; 4518 r = -ENOMEM; 4519 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags); 4520 if (kvm->arch.vpit) 4521 r = 0; 4522 create_pit_unlock: 4523 mutex_unlock(&kvm->lock); 4524 break; 4525 case KVM_GET_IRQCHIP: { 4526 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 4527 struct kvm_irqchip *chip; 4528 4529 chip = memdup_user(argp, sizeof(*chip)); 4530 if (IS_ERR(chip)) { 4531 r = PTR_ERR(chip); 4532 goto out; 4533 } 4534 4535 r = -ENXIO; 4536 if (!irqchip_full(kvm)) 4537 goto get_irqchip_out; 4538 r = kvm_vm_ioctl_get_irqchip(kvm, chip); 4539 if (r) 4540 goto get_irqchip_out; 4541 r = -EFAULT; 4542 if (copy_to_user(argp, chip, sizeof(*chip))) 4543 goto get_irqchip_out; 4544 r = 0; 4545 get_irqchip_out: 4546 kfree(chip); 4547 break; 4548 } 4549 case KVM_SET_IRQCHIP: { 4550 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 4551 struct kvm_irqchip *chip; 4552 4553 chip = memdup_user(argp, sizeof(*chip)); 4554 if (IS_ERR(chip)) { 4555 r = PTR_ERR(chip); 4556 goto out; 4557 } 4558 4559 r = -ENXIO; 4560 if (!irqchip_full(kvm)) 4561 goto set_irqchip_out; 4562 r = kvm_vm_ioctl_set_irqchip(kvm, chip); 4563 set_irqchip_out: 4564 kfree(chip); 4565 break; 4566 } 4567 case KVM_GET_PIT: { 4568 r = -EFAULT; 4569 if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state))) 4570 goto out; 4571 r = -ENXIO; 4572 if (!kvm->arch.vpit) 4573 goto out; 4574 r = kvm_vm_ioctl_get_pit(kvm, &u.ps); 4575 if (r) 4576 goto out; 4577 r = -EFAULT; 4578 if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state))) 4579 goto out; 4580 r = 0; 4581 break; 4582 } 4583 case KVM_SET_PIT: { 4584 r = -EFAULT; 4585 if (copy_from_user(&u.ps, argp, sizeof(u.ps))) 4586 goto out; 4587 mutex_lock(&kvm->lock); 4588 r = -ENXIO; 4589 if (!kvm->arch.vpit) 4590 goto set_pit_out; 4591 r = kvm_vm_ioctl_set_pit(kvm, &u.ps); 4592 set_pit_out: 4593 mutex_unlock(&kvm->lock); 4594 break; 4595 } 4596 case KVM_GET_PIT2: { 4597 r = -ENXIO; 4598 if (!kvm->arch.vpit) 4599 goto out; 4600 r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2); 4601 if (r) 4602 goto out; 4603 r = -EFAULT; 4604 if (copy_to_user(argp, &u.ps2, sizeof(u.ps2))) 4605 goto out; 4606 r = 0; 4607 break; 4608 } 4609 case KVM_SET_PIT2: { 4610 r = -EFAULT; 4611 if (copy_from_user(&u.ps2, argp, sizeof(u.ps2))) 4612 goto out; 4613 mutex_lock(&kvm->lock); 4614 r = -ENXIO; 4615 if (!kvm->arch.vpit) 4616 goto set_pit2_out; 4617 r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2); 4618 set_pit2_out: 4619 mutex_unlock(&kvm->lock); 4620 break; 4621 } 4622 case KVM_REINJECT_CONTROL: { 4623 struct kvm_reinject_control control; 4624 r = -EFAULT; 4625 if (copy_from_user(&control, argp, sizeof(control))) 4626 goto out; 4627 r = -ENXIO; 4628 if (!kvm->arch.vpit) 4629 goto out; 4630 r = kvm_vm_ioctl_reinject(kvm, &control); 4631 break; 4632 } 4633 #endif 4634 case KVM_SET_BOOT_CPU_ID: 4635 r = 0; 4636 mutex_lock(&kvm->lock); 4637 if (kvm->created_vcpus) 4638 r = -EBUSY; 4639 else if (arg > KVM_MAX_VCPU_IDS || 4640 (kvm->arch.max_vcpu_ids && arg > kvm->arch.max_vcpu_ids)) 4641 r = -EINVAL; 4642 else 4643 kvm->arch.bsp_vcpu_id = arg; 4644 mutex_unlock(&kvm->lock); 4645 break; 4646 #ifdef CONFIG_KVM_XEN 4647 case KVM_XEN_HVM_CONFIG: { 4648 struct kvm_xen_hvm_config xhc; 4649 r = -EFAULT; 4650 if (copy_from_user(&xhc, argp, sizeof(xhc))) 4651 goto out; 4652 r = kvm_xen_hvm_config(kvm, &xhc); 4653 break; 4654 } 4655 case KVM_XEN_HVM_GET_ATTR: { 4656 struct kvm_xen_hvm_attr xha; 4657 4658 r = -EFAULT; 4659 if (copy_from_user(&xha, argp, sizeof(xha))) 4660 goto out; 4661 r = kvm_xen_hvm_get_attr(kvm, &xha); 4662 if (!r && copy_to_user(argp, &xha, sizeof(xha))) 4663 r = -EFAULT; 4664 break; 4665 } 4666 case KVM_XEN_HVM_SET_ATTR: { 4667 struct kvm_xen_hvm_attr xha; 4668 4669 r = -EFAULT; 4670 if (copy_from_user(&xha, argp, sizeof(xha))) 4671 goto out; 4672 r = kvm_xen_hvm_set_attr(kvm, &xha); 4673 break; 4674 } 4675 case KVM_XEN_HVM_EVTCHN_SEND: { 4676 struct kvm_irq_routing_xen_evtchn uxe; 4677 4678 r = -EFAULT; 4679 if (copy_from_user(&uxe, argp, sizeof(uxe))) 4680 goto out; 4681 r = kvm_xen_hvm_evtchn_send(kvm, &uxe); 4682 break; 4683 } 4684 #endif 4685 case KVM_SET_CLOCK: 4686 r = kvm_vm_ioctl_set_clock(kvm, argp); 4687 break; 4688 case KVM_GET_CLOCK: 4689 r = kvm_vm_ioctl_get_clock(kvm, argp); 4690 break; 4691 case KVM_SET_TSC_KHZ: { 4692 u32 user_tsc_khz; 4693 4694 r = -EINVAL; 4695 user_tsc_khz = (u32)arg; 4696 4697 if (kvm_caps.has_tsc_control && 4698 user_tsc_khz > kvm_caps.max_guest_tsc_khz) 4699 goto out; 4700 4701 if (user_tsc_khz == 0) 4702 user_tsc_khz = tsc_khz; 4703 4704 mutex_lock(&kvm->lock); 4705 if (!kvm->created_vcpus) { 4706 WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz); 4707 r = 0; 4708 } 4709 mutex_unlock(&kvm->lock); 4710 goto out; 4711 } 4712 case KVM_GET_TSC_KHZ: { 4713 r = READ_ONCE(kvm->arch.default_tsc_khz); 4714 goto out; 4715 } 4716 case KVM_MEMORY_ENCRYPT_OP: 4717 r = -ENOTTY; 4718 if (!kvm_x86_ops.mem_enc_ioctl) 4719 goto out; 4720 4721 r = kvm_x86_call(mem_enc_ioctl)(kvm, argp); 4722 break; 4723 case KVM_MEMORY_ENCRYPT_REG_REGION: { 4724 struct kvm_enc_region region; 4725 4726 r = -EFAULT; 4727 if (copy_from_user(®ion, argp, sizeof(region))) 4728 goto out; 4729 4730 r = -ENOTTY; 4731 if (!kvm_x86_ops.mem_enc_register_region) 4732 goto out; 4733 4734 r = kvm_x86_call(mem_enc_register_region)(kvm, ®ion); 4735 break; 4736 } 4737 case KVM_MEMORY_ENCRYPT_UNREG_REGION: { 4738 struct kvm_enc_region region; 4739 4740 r = -EFAULT; 4741 if (copy_from_user(®ion, argp, sizeof(region))) 4742 goto out; 4743 4744 r = -ENOTTY; 4745 if (!kvm_x86_ops.mem_enc_unregister_region) 4746 goto out; 4747 4748 r = kvm_x86_call(mem_enc_unregister_region)(kvm, ®ion); 4749 break; 4750 } 4751 #ifdef CONFIG_KVM_HYPERV 4752 case KVM_HYPERV_EVENTFD: { 4753 struct kvm_hyperv_eventfd hvevfd; 4754 4755 r = -EFAULT; 4756 if (copy_from_user(&hvevfd, argp, sizeof(hvevfd))) 4757 goto out; 4758 r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd); 4759 break; 4760 } 4761 #endif 4762 case KVM_SET_PMU_EVENT_FILTER: 4763 r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp); 4764 break; 4765 case KVM_X86_SET_MSR_FILTER: { 4766 struct kvm_msr_filter __user *user_msr_filter = argp; 4767 struct kvm_msr_filter filter; 4768 4769 if (copy_from_user(&filter, user_msr_filter, sizeof(filter))) 4770 return -EFAULT; 4771 4772 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter); 4773 break; 4774 } 4775 default: 4776 r = -ENOTTY; 4777 } 4778 out: 4779 return r; 4780 } 4781 4782 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len, 4783 void *__v) 4784 { 4785 const void *v = __v; 4786 int handled = 0; 4787 int n; 4788 4789 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, addr, __v); 4790 4791 do { 4792 n = min(len, 8); 4793 if (!(lapic_in_kernel(vcpu) && 4794 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v)) 4795 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v)) 4796 break; 4797 handled += n; 4798 addr += n; 4799 len -= n; 4800 v += n; 4801 } while (len); 4802 4803 return handled; 4804 } 4805 4806 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v) 4807 { 4808 int handled = 0; 4809 int n; 4810 4811 do { 4812 n = min(len, 8); 4813 if (!(lapic_in_kernel(vcpu) && 4814 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev, 4815 addr, n, v)) 4816 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v)) 4817 break; 4818 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v); 4819 handled += n; 4820 addr += n; 4821 len -= n; 4822 v += n; 4823 } while (len); 4824 4825 if (len) 4826 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len, addr, NULL); 4827 4828 return handled; 4829 } 4830 4831 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, 4832 struct x86_exception *exception) 4833 { 4834 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4835 4836 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 4837 return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception); 4838 } 4839 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_read); 4840 4841 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, 4842 struct x86_exception *exception) 4843 { 4844 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4845 4846 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 4847 access |= PFERR_WRITE_MASK; 4848 return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception); 4849 } 4850 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_write); 4851 4852 /* uses this to access any guest's mapped memory without checking CPL */ 4853 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, 4854 struct x86_exception *exception) 4855 { 4856 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4857 4858 return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, 0, exception); 4859 } 4860 4861 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, 4862 struct kvm_vcpu *vcpu, u64 access, 4863 struct x86_exception *exception) 4864 { 4865 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4866 void *data = val; 4867 int r = X86EMUL_CONTINUE; 4868 4869 while (bytes) { 4870 gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access, exception); 4871 unsigned offset = addr & (PAGE_SIZE-1); 4872 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset); 4873 int ret; 4874 4875 if (gpa == INVALID_GPA) 4876 return X86EMUL_PROPAGATE_FAULT; 4877 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data, 4878 offset, toread); 4879 if (ret < 0) { 4880 r = X86EMUL_IO_NEEDED; 4881 goto out; 4882 } 4883 4884 bytes -= toread; 4885 data += toread; 4886 addr += toread; 4887 } 4888 out: 4889 return r; 4890 } 4891 4892 /* used for instruction fetching */ 4893 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, 4894 gva_t addr, void *val, unsigned int bytes, 4895 struct x86_exception *exception) 4896 { 4897 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4898 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4899 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 4900 unsigned offset; 4901 int ret; 4902 4903 /* Inline kvm_read_guest_virt_helper for speed. */ 4904 gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access|PFERR_FETCH_MASK, 4905 exception); 4906 if (unlikely(gpa == INVALID_GPA)) 4907 return X86EMUL_PROPAGATE_FAULT; 4908 4909 offset = addr & (PAGE_SIZE-1); 4910 if (WARN_ON(offset + bytes > PAGE_SIZE)) 4911 bytes = (unsigned)PAGE_SIZE - offset; 4912 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val, 4913 offset, bytes); 4914 if (unlikely(ret < 0)) 4915 return X86EMUL_IO_NEEDED; 4916 4917 return X86EMUL_CONTINUE; 4918 } 4919 4920 int kvm_read_guest_virt(struct kvm_vcpu *vcpu, 4921 gva_t addr, void *val, unsigned int bytes, 4922 struct x86_exception *exception) 4923 { 4924 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 4925 4926 /* 4927 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED 4928 * is returned, but our callers are not ready for that and they blindly 4929 * call kvm_inject_page_fault. Ensure that they at least do not leak 4930 * uninitialized kernel stack memory into cr2 and error code. 4931 */ 4932 memset(exception, 0, sizeof(*exception)); 4933 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, 4934 exception); 4935 } 4936 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_guest_virt); 4937 4938 static int emulator_read_std(struct x86_emulate_ctxt *ctxt, 4939 gva_t addr, void *val, unsigned int bytes, 4940 struct x86_exception *exception, bool system) 4941 { 4942 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4943 u64 access = 0; 4944 4945 if (system) 4946 access |= PFERR_IMPLICIT_ACCESS; 4947 else if (kvm_x86_call(get_cpl)(vcpu) == 3) 4948 access |= PFERR_USER_MASK; 4949 4950 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception); 4951 } 4952 4953 static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, 4954 struct kvm_vcpu *vcpu, u64 access, 4955 struct x86_exception *exception) 4956 { 4957 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4958 void *data = val; 4959 int r = X86EMUL_CONTINUE; 4960 4961 while (bytes) { 4962 gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access, exception); 4963 unsigned offset = addr & (PAGE_SIZE-1); 4964 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset); 4965 int ret; 4966 4967 if (gpa == INVALID_GPA) 4968 return X86EMUL_PROPAGATE_FAULT; 4969 ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite); 4970 if (ret < 0) { 4971 r = X86EMUL_IO_NEEDED; 4972 goto out; 4973 } 4974 4975 bytes -= towrite; 4976 data += towrite; 4977 addr += towrite; 4978 } 4979 out: 4980 return r; 4981 } 4982 4983 static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val, 4984 unsigned int bytes, struct x86_exception *exception, 4985 bool system) 4986 { 4987 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4988 u64 access = PFERR_WRITE_MASK; 4989 4990 if (system) 4991 access |= PFERR_IMPLICIT_ACCESS; 4992 else if (kvm_x86_call(get_cpl)(vcpu) == 3) 4993 access |= PFERR_USER_MASK; 4994 4995 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, 4996 access, exception); 4997 } 4998 4999 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val, 5000 unsigned int bytes, struct x86_exception *exception) 5001 { 5002 /* kvm_write_guest_virt_system can pull in tons of pages. */ 5003 kvm_request_l1tf_flush_l1d(); 5004 5005 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, 5006 PFERR_WRITE_MASK, exception); 5007 } 5008 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_write_guest_virt_system); 5009 5010 static int kvm_check_emulate_insn(struct kvm_vcpu *vcpu, int emul_type, 5011 void *insn, int insn_len) 5012 { 5013 return kvm_x86_call(check_emulate_instruction)(vcpu, emul_type, 5014 insn, insn_len); 5015 } 5016 5017 int handle_ud(struct kvm_vcpu *vcpu) 5018 { 5019 static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX }; 5020 int fep_flags = READ_ONCE(force_emulation_prefix); 5021 int emul_type = EMULTYPE_TRAP_UD; 5022 char sig[5]; /* ud2; .ascii "kvm" */ 5023 struct x86_exception e; 5024 int r; 5025 5026 r = kvm_check_emulate_insn(vcpu, emul_type, NULL, 0); 5027 if (r != X86EMUL_CONTINUE) 5028 return 1; 5029 5030 if (fep_flags && 5031 kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu), 5032 sig, sizeof(sig), &e) == 0 && 5033 memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) { 5034 if (fep_flags & KVM_FEP_CLEAR_RFLAGS_RF) 5035 kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF); 5036 kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig)); 5037 emul_type = EMULTYPE_TRAP_UD_FORCED; 5038 } 5039 5040 return kvm_emulate_instruction(vcpu, emul_type); 5041 } 5042 EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_ud); 5043 5044 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 5045 gpa_t gpa, bool write) 5046 { 5047 /* For APIC access vmexit */ 5048 if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 5049 return 1; 5050 5051 if (vcpu_match_mmio_gpa(vcpu, gpa)) { 5052 trace_vcpu_match_mmio(gva, gpa, write, true); 5053 return 1; 5054 } 5055 5056 return 0; 5057 } 5058 5059 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 5060 gpa_t *gpa, struct x86_exception *exception, 5061 bool write) 5062 { 5063 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 5064 u64 access = ((kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0) 5065 | (write ? PFERR_WRITE_MASK : 0); 5066 5067 /* 5068 * currently PKRU is only applied to ept enabled guest so 5069 * there is no pkey in EPT page table for L1 guest or EPT 5070 * shadow page table for L2 guest. 5071 */ 5072 if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) || 5073 !permission_fault(vcpu, gva_walk, 5074 vcpu->arch.mmio_access, 0, access))) { 5075 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT | 5076 (gva & (PAGE_SIZE - 1)); 5077 trace_vcpu_match_mmio(gva, *gpa, write, false); 5078 return 1; 5079 } 5080 5081 *gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception); 5082 5083 if (*gpa == INVALID_GPA) 5084 return -1; 5085 5086 return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write); 5087 } 5088 5089 struct read_write_emulator_ops { 5090 int (*read_write_guest)(struct kvm_vcpu *vcpu, gpa_t gpa, 5091 void *val, int bytes); 5092 int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa, 5093 int bytes, void *val); 5094 bool write; 5095 }; 5096 5097 static int emulator_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, 5098 void *val, int bytes) 5099 { 5100 return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes); 5101 } 5102 5103 static int emulator_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, 5104 void *val, int bytes) 5105 { 5106 int ret; 5107 5108 ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes); 5109 if (ret < 0) 5110 return 0; 5111 kvm_page_track_write(vcpu, gpa, val, bytes); 5112 return 1; 5113 } 5114 5115 static int emulator_read_write_onepage(unsigned long addr, void *val, 5116 unsigned int bytes, 5117 struct x86_exception *exception, 5118 struct kvm_vcpu *vcpu, 5119 const struct read_write_emulator_ops *ops) 5120 { 5121 gpa_t gpa; 5122 int handled, ret; 5123 bool write = ops->write; 5124 struct kvm_mmio_fragment *frag; 5125 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 5126 5127 /* 5128 * If the exit was due to a NPF we may already have a GPA. 5129 * If the GPA is present, use it to avoid the GVA to GPA table walk. 5130 * Note, this cannot be used on string operations since string 5131 * operation using rep will only have the initial GPA from the NPF 5132 * occurred. 5133 */ 5134 if ((ctxt->gpa_access & (write ? ACC_WRITE_MASK : ACC_READ_MASK)) && 5135 emulator_can_use_gpa(ctxt) && 5136 (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) { 5137 gpa = ctxt->gpa_val; 5138 ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); 5139 } else { 5140 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write); 5141 if (ret < 0) 5142 return X86EMUL_PROPAGATE_FAULT; 5143 } 5144 5145 /* 5146 * If the memory is not _known_ to be emulated MMIO, attempt to access 5147 * guest memory. If accessing guest memory fails, e.g. because there's 5148 * no memslot, then handle the access as MMIO. Note, treating the 5149 * access as emulated MMIO is technically wrong if there is a memslot, 5150 * i.e. if accessing host user memory failed, but this has been KVM's 5151 * historical ABI for decades. 5152 */ 5153 if (!ret && ops->read_write_guest(vcpu, gpa, val, bytes)) 5154 return X86EMUL_CONTINUE; 5155 5156 /* 5157 * Attempt to handle emulated MMIO within the kernel, e.g. for accesses 5158 * to an in-kernel local or I/O APIC, or to an ioeventfd range attached 5159 * to MMIO bus. If the access isn't fully resolved, insert an MMIO 5160 * fragment with the relevant details. 5161 */ 5162 handled = ops->read_write_mmio(vcpu, gpa, bytes, val); 5163 if (handled == bytes) 5164 return X86EMUL_CONTINUE; 5165 5166 gpa += handled; 5167 bytes -= handled; 5168 val += handled; 5169 5170 WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS); 5171 frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++]; 5172 frag->gpa = gpa; 5173 if (write && bytes <= 8u) { 5174 frag->val = 0; 5175 frag->data = &frag->val; 5176 memcpy(&frag->val, val, bytes); 5177 } else { 5178 frag->data = val; 5179 } 5180 frag->len = bytes; 5181 5182 /* 5183 * Continue emulating, even though KVM needs to (eventually) do an MMIO 5184 * exit to userspace. If the access splits multiple pages, then KVM 5185 * needs to exit to userspace only after emulating both parts of the 5186 * access. 5187 */ 5188 return X86EMUL_CONTINUE; 5189 } 5190 5191 static int emulator_read_write(struct x86_emulate_ctxt *ctxt, 5192 unsigned long addr, 5193 void *val, unsigned int bytes, 5194 struct x86_exception *exception, 5195 const struct read_write_emulator_ops *ops) 5196 { 5197 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5198 int rc; 5199 5200 if (WARN_ON_ONCE((bytes > 8u || !ops->write) && object_is_on_stack(val))) 5201 return X86EMUL_UNHANDLEABLE; 5202 5203 /* 5204 * If the read was already completed via a userspace MMIO exit, there's 5205 * nothing left to do except trace the MMIO read. When completing MMIO 5206 * reads, KVM re-emulates the instruction to propagate the value into 5207 * the correct destination, e.g. into the correct register, but the 5208 * value itself has already been copied to the read cache. 5209 * 5210 * Note! This is *tightly* coupled to read_emulated() satisfying reads 5211 * from the emulator's mem_read cache, so that the MMIO fragment data 5212 * is copied to the correct chunk of the correct operand. 5213 */ 5214 if (!ops->write && vcpu->mmio_read_completed) { 5215 /* 5216 * For simplicity, trace the entire MMIO read in one shot, even 5217 * though the GPA might be incorrect if there are two fragments 5218 * that aren't contiguous in the GPA space. 5219 */ 5220 trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, 5221 vcpu->mmio_fragments[0].gpa, val); 5222 vcpu->mmio_read_completed = 0; 5223 return X86EMUL_CONTINUE; 5224 } 5225 5226 vcpu->mmio_nr_fragments = 0; 5227 5228 /* Crossing a page boundary? */ 5229 if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { 5230 int now; 5231 5232 now = -addr & ~PAGE_MASK; 5233 rc = emulator_read_write_onepage(addr, val, now, exception, 5234 vcpu, ops); 5235 5236 if (rc != X86EMUL_CONTINUE) 5237 return rc; 5238 addr += now; 5239 if (ctxt->mode != X86EMUL_MODE_PROT64) 5240 addr = (u32)addr; 5241 val += now; 5242 bytes -= now; 5243 } 5244 5245 rc = emulator_read_write_onepage(addr, val, bytes, exception, 5246 vcpu, ops); 5247 if (rc != X86EMUL_CONTINUE) 5248 return rc; 5249 5250 if (!vcpu->mmio_nr_fragments) 5251 return X86EMUL_CONTINUE; 5252 5253 vcpu->mmio_needed = 1; 5254 vcpu->mmio_cur_fragment = 0; 5255 vcpu->mmio_is_write = ops->write; 5256 5257 kvm_prepare_emulated_mmio_exit(vcpu, &vcpu->mmio_fragments[0]); 5258 5259 /* 5260 * For MMIO reads, stop emulating and immediately exit to userspace, as 5261 * KVM needs the value to correctly emulate the instruction. For MMIO 5262 * writes, continue emulating as the write to MMIO is a side effect for 5263 * all intents and purposes. KVM will still exit to userspace, but 5264 * after completing emulation (see the check on vcpu->mmio_needed in 5265 * x86_emulate_instruction()). 5266 */ 5267 return ops->write ? X86EMUL_CONTINUE : X86EMUL_IO_NEEDED; 5268 } 5269 5270 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt, 5271 unsigned long addr, 5272 void *val, 5273 unsigned int bytes, 5274 struct x86_exception *exception) 5275 { 5276 static const struct read_write_emulator_ops ops = { 5277 .read_write_guest = emulator_read_guest, 5278 .read_write_mmio = vcpu_mmio_read, 5279 .write = false, 5280 }; 5281 5282 return emulator_read_write(ctxt, addr, val, bytes, exception, &ops); 5283 } 5284 5285 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt, 5286 unsigned long addr, 5287 const void *val, 5288 unsigned int bytes, 5289 struct x86_exception *exception) 5290 { 5291 static const struct read_write_emulator_ops ops = { 5292 .read_write_guest = emulator_write_guest, 5293 .read_write_mmio = vcpu_mmio_write, 5294 .write = true, 5295 }; 5296 5297 return emulator_read_write(ctxt, addr, (void *)val, bytes, exception, &ops); 5298 } 5299 5300 #define emulator_try_cmpxchg_user(t, ptr, old, new) \ 5301 (__try_cmpxchg_user((t __user *)(ptr), (t *)(old), *(t *)(new), efault ## t)) 5302 5303 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, 5304 unsigned long addr, 5305 const void *old, 5306 const void *new, 5307 unsigned int bytes, 5308 struct x86_exception *exception) 5309 { 5310 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5311 u64 page_line_mask; 5312 unsigned long hva; 5313 gpa_t gpa; 5314 int r; 5315 5316 /* guests cmpxchg8b have to be emulated atomically */ 5317 if (bytes > 8 || (bytes & (bytes - 1))) 5318 goto emul_write; 5319 5320 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL); 5321 5322 if (gpa == INVALID_GPA || 5323 (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 5324 goto emul_write; 5325 5326 /* 5327 * Emulate the atomic as a straight write to avoid #AC if SLD is 5328 * enabled in the host and the access splits a cache line. 5329 */ 5330 if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 5331 page_line_mask = ~(cache_line_size() - 1); 5332 else 5333 page_line_mask = PAGE_MASK; 5334 5335 if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask)) 5336 goto emul_write; 5337 5338 hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa)); 5339 if (kvm_is_error_hva(hva)) 5340 goto emul_write; 5341 5342 hva += offset_in_page(gpa); 5343 5344 switch (bytes) { 5345 case 1: 5346 r = emulator_try_cmpxchg_user(u8, hva, old, new); 5347 break; 5348 case 2: 5349 r = emulator_try_cmpxchg_user(u16, hva, old, new); 5350 break; 5351 case 4: 5352 r = emulator_try_cmpxchg_user(u32, hva, old, new); 5353 break; 5354 case 8: 5355 r = emulator_try_cmpxchg_user(u64, hva, old, new); 5356 break; 5357 default: 5358 BUG(); 5359 } 5360 5361 if (r < 0) 5362 return X86EMUL_UNHANDLEABLE; 5363 5364 /* 5365 * Mark the page dirty _before_ checking whether or not the CMPXCHG was 5366 * successful, as the old value is written back on failure. Note, for 5367 * live migration, this is unnecessarily conservative as CMPXCHG writes 5368 * back the original value and the access is atomic, but KVM's ABI is 5369 * that all writes are dirty logged, regardless of the value written. 5370 */ 5371 kvm_vcpu_mark_page_dirty(vcpu, gpa_to_gfn(gpa)); 5372 5373 if (r) 5374 return X86EMUL_CMPXCHG_FAILED; 5375 5376 kvm_page_track_write(vcpu, gpa, new, bytes); 5377 5378 return X86EMUL_CONTINUE; 5379 5380 emul_write: 5381 pr_warn_once("emulating exchange as write\n"); 5382 5383 return emulator_write_emulated(ctxt, addr, new, bytes, exception); 5384 } 5385 5386 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size, 5387 unsigned short port, void *data, 5388 unsigned int count, bool in) 5389 { 5390 unsigned i; 5391 int r; 5392 5393 WARN_ON_ONCE(vcpu->arch.pio.count); 5394 for (i = 0; i < count; i++) { 5395 if (in) 5396 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data); 5397 else 5398 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data); 5399 5400 if (r) { 5401 if (i == 0) 5402 goto userspace_io; 5403 5404 /* 5405 * Userspace must have unregistered the device while PIO 5406 * was running. Drop writes / read as 0. 5407 */ 5408 if (in) 5409 memset(data, 0, size * (count - i)); 5410 break; 5411 } 5412 5413 data += size; 5414 } 5415 return 1; 5416 5417 userspace_io: 5418 vcpu->arch.pio.port = port; 5419 vcpu->arch.pio.in = in; 5420 vcpu->arch.pio.count = count; 5421 vcpu->arch.pio.size = size; 5422 5423 if (in) 5424 memset(vcpu->arch.pio_data, 0, size * count); 5425 else 5426 memcpy(vcpu->arch.pio_data, data, size * count); 5427 5428 vcpu->run->exit_reason = KVM_EXIT_IO; 5429 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; 5430 vcpu->run->io.size = size; 5431 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; 5432 vcpu->run->io.count = count; 5433 vcpu->run->io.port = port; 5434 return 0; 5435 } 5436 5437 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size, 5438 unsigned short port, void *val, unsigned int count) 5439 { 5440 int r = emulator_pio_in_out(vcpu, size, port, val, count, true); 5441 if (r) 5442 trace_kvm_pio(KVM_PIO_IN, port, size, count, val); 5443 5444 return r; 5445 } 5446 5447 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val) 5448 { 5449 int size = vcpu->arch.pio.size; 5450 unsigned int count = vcpu->arch.pio.count; 5451 memcpy(val, vcpu->arch.pio_data, size * count); 5452 trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data); 5453 vcpu->arch.pio.count = 0; 5454 } 5455 5456 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt, 5457 int size, unsigned short port, void *val, 5458 unsigned int count) 5459 { 5460 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5461 if (vcpu->arch.pio.count) { 5462 /* 5463 * Complete a previous iteration that required userspace I/O. 5464 * Note, @count isn't guaranteed to match pio.count as userspace 5465 * can modify ECX before rerunning the vCPU. Ignore any such 5466 * shenanigans as KVM doesn't support modifying the rep count, 5467 * and the emulator ensures @count doesn't overflow the buffer. 5468 */ 5469 complete_emulator_pio_in(vcpu, val); 5470 return 1; 5471 } 5472 5473 return emulator_pio_in(vcpu, size, port, val, count); 5474 } 5475 5476 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size, 5477 unsigned short port, const void *val, 5478 unsigned int count) 5479 { 5480 trace_kvm_pio(KVM_PIO_OUT, port, size, count, val); 5481 return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false); 5482 } 5483 5484 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, 5485 int size, unsigned short port, 5486 const void *val, unsigned int count) 5487 { 5488 return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count); 5489 } 5490 5491 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address) 5492 { 5493 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); 5494 } 5495 5496 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) 5497 { 5498 if (!need_emulate_wbinvd(vcpu)) 5499 return X86EMUL_CONTINUE; 5500 5501 if (kvm_x86_call(has_wbinvd_exit)()) { 5502 int cpu = get_cpu(); 5503 5504 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 5505 wbinvd_on_cpus_mask(vcpu->arch.wbinvd_dirty_mask); 5506 put_cpu(); 5507 cpumask_clear(vcpu->arch.wbinvd_dirty_mask); 5508 } else 5509 wbinvd(); 5510 return X86EMUL_CONTINUE; 5511 } 5512 5513 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) 5514 { 5515 kvm_emulate_wbinvd_noskip(vcpu); 5516 return kvm_skip_emulated_instruction(vcpu); 5517 } 5518 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wbinvd); 5519 5520 5521 5522 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) 5523 { 5524 kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt)); 5525 } 5526 5527 static unsigned long emulator_get_effective_dr7(struct x86_emulate_ctxt *ctxt) 5528 { 5529 return kvm_get_effective_dr7(emul_to_vcpu(ctxt)); 5530 } 5531 5532 static unsigned long emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr) 5533 { 5534 return kvm_get_dr(emul_to_vcpu(ctxt), dr); 5535 } 5536 5537 static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, 5538 unsigned long value) 5539 { 5540 5541 return kvm_set_dr(emul_to_vcpu(ctxt), dr, value); 5542 } 5543 5544 static u64 mk_cr_64(u64 curr_cr, u32 new_val) 5545 { 5546 return (curr_cr & ~((1ULL << 32) - 1)) | new_val; 5547 } 5548 5549 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr) 5550 { 5551 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5552 unsigned long value; 5553 5554 switch (cr) { 5555 case 0: 5556 value = kvm_read_cr0(vcpu); 5557 break; 5558 case 2: 5559 value = vcpu->arch.cr2; 5560 break; 5561 case 3: 5562 value = kvm_read_cr3(vcpu); 5563 break; 5564 case 4: 5565 value = kvm_read_cr4(vcpu); 5566 break; 5567 case 8: 5568 value = kvm_get_cr8(vcpu); 5569 break; 5570 default: 5571 kvm_err("%s: unexpected cr %u\n", __func__, cr); 5572 return 0; 5573 } 5574 5575 return value; 5576 } 5577 5578 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val) 5579 { 5580 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5581 int res = 0; 5582 5583 switch (cr) { 5584 case 0: 5585 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val)); 5586 break; 5587 case 2: 5588 vcpu->arch.cr2 = val; 5589 break; 5590 case 3: 5591 res = kvm_set_cr3(vcpu, val); 5592 break; 5593 case 4: 5594 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val)); 5595 break; 5596 case 8: 5597 res = kvm_set_cr8(vcpu, val); 5598 break; 5599 default: 5600 kvm_err("%s: unexpected cr %u\n", __func__, cr); 5601 res = -1; 5602 } 5603 5604 return res; 5605 } 5606 5607 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt) 5608 { 5609 return kvm_x86_call(get_cpl)(emul_to_vcpu(ctxt)); 5610 } 5611 5612 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 5613 { 5614 kvm_x86_call(get_gdt)(emul_to_vcpu(ctxt), dt); 5615 } 5616 5617 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 5618 { 5619 kvm_x86_call(get_idt)(emul_to_vcpu(ctxt), dt); 5620 } 5621 5622 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 5623 { 5624 kvm_x86_call(set_gdt)(emul_to_vcpu(ctxt), dt); 5625 } 5626 5627 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 5628 { 5629 kvm_x86_call(set_idt)(emul_to_vcpu(ctxt), dt); 5630 } 5631 5632 static unsigned long emulator_get_cached_segment_base( 5633 struct x86_emulate_ctxt *ctxt, int seg) 5634 { 5635 return kvm_get_segment_base(emul_to_vcpu(ctxt), seg); 5636 } 5637 5638 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector, 5639 struct desc_struct *desc, u32 *base3, 5640 int seg) 5641 { 5642 struct kvm_segment var; 5643 5644 kvm_get_segment(emul_to_vcpu(ctxt), &var, seg); 5645 *selector = var.selector; 5646 5647 if (var.unusable) { 5648 memset(desc, 0, sizeof(*desc)); 5649 if (base3) 5650 *base3 = 0; 5651 return false; 5652 } 5653 5654 if (var.g) 5655 var.limit >>= 12; 5656 set_desc_limit(desc, var.limit); 5657 set_desc_base(desc, (unsigned long)var.base); 5658 #ifdef CONFIG_X86_64 5659 if (base3) 5660 *base3 = var.base >> 32; 5661 #endif 5662 desc->type = var.type; 5663 desc->s = var.s; 5664 desc->dpl = var.dpl; 5665 desc->p = var.present; 5666 desc->avl = var.avl; 5667 desc->l = var.l; 5668 desc->d = var.db; 5669 desc->g = var.g; 5670 5671 return true; 5672 } 5673 5674 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector, 5675 struct desc_struct *desc, u32 base3, 5676 int seg) 5677 { 5678 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5679 struct kvm_segment var; 5680 5681 var.selector = selector; 5682 var.base = get_desc_base(desc); 5683 #ifdef CONFIG_X86_64 5684 var.base |= ((u64)base3) << 32; 5685 #endif 5686 var.limit = get_desc_limit(desc); 5687 if (desc->g) 5688 var.limit = (var.limit << 12) | 0xfff; 5689 var.type = desc->type; 5690 var.dpl = desc->dpl; 5691 var.db = desc->d; 5692 var.s = desc->s; 5693 var.l = desc->l; 5694 var.g = desc->g; 5695 var.avl = desc->avl; 5696 var.present = desc->p; 5697 var.unusable = !var.present; 5698 var.padding = 0; 5699 5700 kvm_set_segment(vcpu, &var, seg); 5701 return; 5702 } 5703 5704 static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt, 5705 u32 msr_index, u64 *pdata) 5706 { 5707 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5708 5709 return kvm_emulator_get_msr_with_filter(vcpu, msr_index, pdata); 5710 } 5711 5712 static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt, 5713 u32 msr_index, u64 data) 5714 { 5715 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5716 5717 return kvm_emulator_set_msr_with_filter(vcpu, msr_index, data); 5718 } 5719 5720 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, 5721 u32 msr_index, u64 *pdata) 5722 { 5723 return kvm_emulator_get_msr(emul_to_vcpu(ctxt), msr_index, pdata); 5724 } 5725 5726 static int emulator_check_rdpmc_early(struct x86_emulate_ctxt *ctxt, u32 pmc) 5727 { 5728 return kvm_pmu_check_rdpmc_early(emul_to_vcpu(ctxt), pmc); 5729 } 5730 5731 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt, 5732 u32 pmc, u64 *pdata) 5733 { 5734 return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata); 5735 } 5736 5737 static void emulator_halt(struct x86_emulate_ctxt *ctxt) 5738 { 5739 emul_to_vcpu(ctxt)->arch.halt_request = 1; 5740 } 5741 5742 static int emulator_intercept(struct x86_emulate_ctxt *ctxt, 5743 struct x86_instruction_info *info, 5744 enum x86_intercept_stage stage) 5745 { 5746 return kvm_x86_call(check_intercept)(emul_to_vcpu(ctxt), info, stage, 5747 &ctxt->exception); 5748 } 5749 5750 static bool emulator_is_cpuid_allowed(struct x86_emulate_ctxt *ctxt) 5751 { 5752 return kvm_is_cpuid_allowed(emul_to_vcpu(ctxt)); 5753 } 5754 5755 static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt, 5756 u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, 5757 bool exact_only) 5758 { 5759 return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only); 5760 } 5761 5762 static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt) 5763 { 5764 return guest_cpu_cap_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE); 5765 } 5766 5767 static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt) 5768 { 5769 return guest_cpu_cap_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR); 5770 } 5771 5772 static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt) 5773 { 5774 return guest_cpu_cap_has(emul_to_vcpu(ctxt), X86_FEATURE_RDPID); 5775 } 5776 5777 static bool emulator_guest_cpuid_is_intel_compatible(struct x86_emulate_ctxt *ctxt) 5778 { 5779 return guest_cpuid_is_intel_compatible(emul_to_vcpu(ctxt)); 5780 } 5781 5782 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg) 5783 { 5784 return kvm_register_read_raw(emul_to_vcpu(ctxt), reg); 5785 } 5786 5787 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val) 5788 { 5789 kvm_register_write_raw(emul_to_vcpu(ctxt), reg, val); 5790 } 5791 5792 static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked) 5793 { 5794 kvm_x86_call(set_nmi_mask)(emul_to_vcpu(ctxt), masked); 5795 } 5796 5797 static bool emulator_is_smm(struct x86_emulate_ctxt *ctxt) 5798 { 5799 return is_smm(emul_to_vcpu(ctxt)); 5800 } 5801 5802 #ifndef CONFIG_KVM_SMM 5803 static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt) 5804 { 5805 WARN_ON_ONCE(1); 5806 return X86EMUL_UNHANDLEABLE; 5807 } 5808 #endif 5809 5810 static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt) 5811 { 5812 kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt)); 5813 } 5814 5815 static int emulator_get_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 *xcr) 5816 { 5817 if (index != XCR_XFEATURE_ENABLED_MASK) 5818 return 1; 5819 *xcr = emul_to_vcpu(ctxt)->arch.xcr0; 5820 return 0; 5821 } 5822 5823 static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr) 5824 { 5825 return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr); 5826 } 5827 5828 static void emulator_vm_bugged(struct x86_emulate_ctxt *ctxt) 5829 { 5830 struct kvm *kvm = emul_to_vcpu(ctxt)->kvm; 5831 5832 if (!kvm->vm_bugged) 5833 kvm_vm_bugged(kvm); 5834 } 5835 5836 static gva_t emulator_get_untagged_addr(struct x86_emulate_ctxt *ctxt, 5837 gva_t addr, unsigned int flags) 5838 { 5839 if (!kvm_x86_ops.get_untagged_addr) 5840 return addr; 5841 5842 return kvm_x86_call(get_untagged_addr)(emul_to_vcpu(ctxt), 5843 addr, flags); 5844 } 5845 5846 static bool emulator_is_canonical_addr(struct x86_emulate_ctxt *ctxt, 5847 gva_t addr, unsigned int flags) 5848 { 5849 return !is_noncanonical_address(addr, emul_to_vcpu(ctxt), flags); 5850 } 5851 5852 static bool emulator_page_address_valid(struct x86_emulate_ctxt *ctxt, gpa_t gpa) 5853 { 5854 return page_address_valid(emul_to_vcpu(ctxt), gpa); 5855 } 5856 5857 static const struct x86_emulate_ops emulate_ops = { 5858 .vm_bugged = emulator_vm_bugged, 5859 .read_gpr = emulator_read_gpr, 5860 .write_gpr = emulator_write_gpr, 5861 .read_std = emulator_read_std, 5862 .write_std = emulator_write_std, 5863 .fetch = kvm_fetch_guest_virt, 5864 .read_emulated = emulator_read_emulated, 5865 .write_emulated = emulator_write_emulated, 5866 .cmpxchg_emulated = emulator_cmpxchg_emulated, 5867 .invlpg = emulator_invlpg, 5868 .pio_in_emulated = emulator_pio_in_emulated, 5869 .pio_out_emulated = emulator_pio_out_emulated, 5870 .get_segment = emulator_get_segment, 5871 .set_segment = emulator_set_segment, 5872 .get_cached_segment_base = emulator_get_cached_segment_base, 5873 .get_gdt = emulator_get_gdt, 5874 .get_idt = emulator_get_idt, 5875 .set_gdt = emulator_set_gdt, 5876 .set_idt = emulator_set_idt, 5877 .get_cr = emulator_get_cr, 5878 .set_cr = emulator_set_cr, 5879 .cpl = emulator_get_cpl, 5880 .get_effective_dr7 = emulator_get_effective_dr7, 5881 .get_dr = emulator_get_dr, 5882 .set_dr = emulator_set_dr, 5883 .set_msr_with_filter = emulator_set_msr_with_filter, 5884 .get_msr_with_filter = emulator_get_msr_with_filter, 5885 .get_msr = emulator_get_msr, 5886 .check_rdpmc_early = emulator_check_rdpmc_early, 5887 .read_pmc = emulator_read_pmc, 5888 .halt = emulator_halt, 5889 .wbinvd = emulator_wbinvd, 5890 .fix_hypercall = emulator_fix_hypercall, 5891 .intercept = emulator_intercept, 5892 .is_cpuid_allowed = emulator_is_cpuid_allowed, 5893 .get_cpuid = emulator_get_cpuid, 5894 .guest_has_movbe = emulator_guest_has_movbe, 5895 .guest_has_fxsr = emulator_guest_has_fxsr, 5896 .guest_has_rdpid = emulator_guest_has_rdpid, 5897 .guest_cpuid_is_intel_compatible = emulator_guest_cpuid_is_intel_compatible, 5898 .set_nmi_mask = emulator_set_nmi_mask, 5899 .is_smm = emulator_is_smm, 5900 .leave_smm = emulator_leave_smm, 5901 .triple_fault = emulator_triple_fault, 5902 .get_xcr = emulator_get_xcr, 5903 .set_xcr = emulator_set_xcr, 5904 .get_untagged_addr = emulator_get_untagged_addr, 5905 .is_canonical_addr = emulator_is_canonical_addr, 5906 .page_address_valid = emulator_page_address_valid, 5907 }; 5908 5909 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) 5910 { 5911 u32 int_shadow = kvm_x86_call(get_interrupt_shadow)(vcpu); 5912 /* 5913 * an sti; sti; sequence only disable interrupts for the first 5914 * instruction. So, if the last instruction, be it emulated or 5915 * not, left the system with the INT_STI flag enabled, it 5916 * means that the last instruction is an sti. We should not 5917 * leave the flag on in this case. The same goes for mov ss 5918 */ 5919 if (int_shadow & mask) 5920 mask = 0; 5921 if (unlikely(int_shadow || mask)) { 5922 kvm_x86_call(set_interrupt_shadow)(vcpu, mask); 5923 if (!mask) 5924 kvm_make_request(KVM_REQ_EVENT, vcpu); 5925 } 5926 } 5927 5928 static int kvm_inject_emulated_db(struct kvm_vcpu *vcpu, unsigned long dr6) 5929 { 5930 struct kvm_run *kvm_run = vcpu->run; 5931 5932 if (vcpu->guest_debug & (KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_SINGLESTEP)) { 5933 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; 5934 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5935 kvm_run->debug.arch.exception = DB_VECTOR; 5936 kvm_run->exit_reason = KVM_EXIT_DEBUG; 5937 return 0; 5938 } 5939 5940 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 5941 return 1; 5942 } 5943 5944 static int inject_emulated_exception(struct kvm_vcpu *vcpu) 5945 { 5946 struct x86_exception *ex = &vcpu->arch.emulate_ctxt->exception; 5947 5948 if (ex->vector == DB_VECTOR) 5949 return kvm_inject_emulated_db(vcpu, ex->dr6); 5950 5951 if (ex->vector == PF_VECTOR) 5952 kvm_inject_emulated_page_fault(vcpu, ex); 5953 else if (ex->error_code_valid) 5954 kvm_queue_exception_e(vcpu, ex->vector, ex->error_code); 5955 else 5956 kvm_queue_exception(vcpu, ex->vector); 5957 return 1; 5958 } 5959 5960 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu) 5961 { 5962 struct x86_emulate_ctxt *ctxt; 5963 5964 ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT); 5965 if (!ctxt) { 5966 pr_err("failed to allocate vcpu's emulator\n"); 5967 return NULL; 5968 } 5969 5970 ctxt->vcpu = vcpu; 5971 ctxt->ops = &emulate_ops; 5972 vcpu->arch.emulate_ctxt = ctxt; 5973 5974 return ctxt; 5975 } 5976 5977 static void init_emulate_ctxt(struct kvm_vcpu *vcpu) 5978 { 5979 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 5980 int cs_db, cs_l; 5981 5982 kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); 5983 5984 ctxt->gpa_access = 0; 5985 ctxt->eflags = kvm_get_rflags(vcpu); 5986 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; 5987 5988 ctxt->eip = kvm_rip_read(vcpu); 5989 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL : 5990 (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 : 5991 (cs_l && is_long_mode(vcpu)) ? X86EMUL_MODE_PROT64 : 5992 cs_db ? X86EMUL_MODE_PROT32 : 5993 X86EMUL_MODE_PROT16; 5994 ctxt->interruptibility = 0; 5995 ctxt->have_exception = false; 5996 ctxt->exception.vector = -1; 5997 ctxt->exception.payload = 0; 5998 ctxt->perm_ok = false; 5999 6000 init_decode_cache(ctxt); 6001 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 6002 } 6003 6004 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) 6005 { 6006 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 6007 int ret; 6008 6009 init_emulate_ctxt(vcpu); 6010 6011 ctxt->op_bytes = 2; 6012 ctxt->ad_bytes = 2; 6013 ctxt->_eip = ctxt->eip + inc_eip; 6014 ret = emulate_int_real(ctxt, irq); 6015 6016 if (ret != X86EMUL_CONTINUE) { 6017 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 6018 } else { 6019 ctxt->eip = ctxt->_eip; 6020 kvm_rip_write(vcpu, ctxt->eip); 6021 kvm_set_rflags(vcpu, ctxt->eflags); 6022 } 6023 } 6024 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_inject_realmode_interrupt); 6025 6026 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, 6027 u8 ndata, u8 *insn_bytes, u8 insn_size) 6028 { 6029 struct kvm_run *run = vcpu->run; 6030 u64 info[5]; 6031 u8 info_start; 6032 6033 /* 6034 * Zero the whole array used to retrieve the exit info, as casting to 6035 * u32 for select entries will leave some chunks uninitialized. 6036 */ 6037 memset(&info, 0, sizeof(info)); 6038 6039 kvm_x86_call(get_exit_info)(vcpu, (u32 *)&info[0], &info[1], &info[2], 6040 (u32 *)&info[3], (u32 *)&info[4]); 6041 6042 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6043 run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION; 6044 6045 /* 6046 * There's currently space for 13 entries, but 5 are used for the exit 6047 * reason and info. Restrict to 4 to reduce the maintenance burden 6048 * when expanding kvm_run.emulation_failure in the future. 6049 */ 6050 if (WARN_ON_ONCE(ndata > 4)) 6051 ndata = 4; 6052 6053 /* Always include the flags as a 'data' entry. */ 6054 info_start = 1; 6055 run->emulation_failure.flags = 0; 6056 6057 if (insn_size) { 6058 BUILD_BUG_ON((sizeof(run->emulation_failure.insn_size) + 6059 sizeof(run->emulation_failure.insn_bytes) != 16)); 6060 info_start += 2; 6061 run->emulation_failure.flags |= 6062 KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES; 6063 run->emulation_failure.insn_size = insn_size; 6064 memset(run->emulation_failure.insn_bytes, 0x90, 6065 sizeof(run->emulation_failure.insn_bytes)); 6066 memcpy(run->emulation_failure.insn_bytes, insn_bytes, insn_size); 6067 } 6068 6069 memcpy(&run->internal.data[info_start], info, sizeof(info)); 6070 memcpy(&run->internal.data[info_start + ARRAY_SIZE(info)], data, 6071 ndata * sizeof(data[0])); 6072 6073 run->emulation_failure.ndata = info_start + ARRAY_SIZE(info) + ndata; 6074 } 6075 6076 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu) 6077 { 6078 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 6079 6080 prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data, 6081 ctxt->fetch.end - ctxt->fetch.data); 6082 } 6083 6084 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, 6085 u8 ndata) 6086 { 6087 prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0); 6088 } 6089 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_prepare_emulation_failure_exit); 6090 6091 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu) 6092 { 6093 __kvm_prepare_emulation_failure_exit(vcpu, NULL, 0); 6094 } 6095 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prepare_emulation_failure_exit); 6096 6097 void kvm_prepare_event_vectoring_exit(struct kvm_vcpu *vcpu, gpa_t gpa) 6098 { 6099 u32 reason, intr_info, error_code; 6100 struct kvm_run *run = vcpu->run; 6101 u64 info1, info2; 6102 int ndata = 0; 6103 6104 kvm_x86_call(get_exit_info)(vcpu, &reason, &info1, &info2, 6105 &intr_info, &error_code); 6106 6107 run->internal.data[ndata++] = info2; 6108 run->internal.data[ndata++] = reason; 6109 run->internal.data[ndata++] = info1; 6110 run->internal.data[ndata++] = gpa; 6111 run->internal.data[ndata++] = vcpu->arch.last_vmentry_cpu; 6112 6113 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6114 run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV; 6115 run->internal.ndata = ndata; 6116 } 6117 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prepare_event_vectoring_exit); 6118 6119 void kvm_prepare_unexpected_reason_exit(struct kvm_vcpu *vcpu, u64 exit_reason) 6120 { 6121 vcpu_unimpl(vcpu, "unexpected exit reason 0x%llx\n", exit_reason); 6122 6123 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6124 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 6125 vcpu->run->internal.ndata = 2; 6126 vcpu->run->internal.data[0] = exit_reason; 6127 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 6128 } 6129 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prepare_unexpected_reason_exit); 6130 6131 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) 6132 { 6133 struct kvm *kvm = vcpu->kvm; 6134 6135 ++vcpu->stat.insn_emulation_fail; 6136 trace_kvm_emulate_insn_failed(vcpu); 6137 6138 if (emulation_type & EMULTYPE_VMWARE_GP) { 6139 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 6140 return 1; 6141 } 6142 6143 if (kvm->arch.exit_on_emulation_error || 6144 (emulation_type & EMULTYPE_SKIP)) { 6145 prepare_emulation_ctxt_failure_exit(vcpu); 6146 return 0; 6147 } 6148 6149 kvm_queue_exception(vcpu, UD_VECTOR); 6150 6151 if (!is_guest_mode(vcpu) && kvm_x86_call(get_cpl)(vcpu) == 0) { 6152 prepare_emulation_ctxt_failure_exit(vcpu); 6153 return 0; 6154 } 6155 6156 return 1; 6157 } 6158 6159 static bool kvm_unprotect_and_retry_on_failure(struct kvm_vcpu *vcpu, 6160 gpa_t cr2_or_gpa, 6161 int emulation_type) 6162 { 6163 if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF)) 6164 return false; 6165 6166 /* 6167 * If the failed instruction faulted on an access to page tables that 6168 * are used to translate any part of the instruction, KVM can't resolve 6169 * the issue by unprotecting the gfn, as zapping the shadow page will 6170 * result in the instruction taking a !PRESENT page fault and thus put 6171 * the vCPU into an infinite loop of page faults. E.g. KVM will create 6172 * a SPTE and write-protect the gfn to resolve the !PRESENT fault, and 6173 * then zap the SPTE to unprotect the gfn, and then do it all over 6174 * again. Report the error to userspace. 6175 */ 6176 if (emulation_type & EMULTYPE_WRITE_PF_TO_SP) 6177 return false; 6178 6179 /* 6180 * If emulation may have been triggered by a write to a shadowed page 6181 * table, unprotect the gfn (zap any relevant SPTEs) and re-enter the 6182 * guest to let the CPU re-execute the instruction in the hope that the 6183 * CPU can cleanly execute the instruction that KVM failed to emulate. 6184 */ 6185 __kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa, true); 6186 6187 /* 6188 * Retry even if _this_ vCPU didn't unprotect the gfn, as it's possible 6189 * all SPTEs were already zapped by a different task. The alternative 6190 * is to report the error to userspace and likely terminate the guest, 6191 * and the last_retry_{eip,addr} checks will prevent retrying the page 6192 * fault indefinitely, i.e. there's nothing to lose by retrying. 6193 */ 6194 return true; 6195 } 6196 6197 static int complete_emulated_mmio(struct kvm_vcpu *vcpu); 6198 static int complete_emulated_pio(struct kvm_vcpu *vcpu); 6199 6200 static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7, 6201 unsigned long *db) 6202 { 6203 u32 dr6 = 0; 6204 int i; 6205 u32 enable, rwlen; 6206 6207 enable = dr7; 6208 rwlen = dr7 >> 16; 6209 for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4) 6210 if ((enable & 3) && (rwlen & 15) == type && db[i] == addr) 6211 dr6 |= (1 << i); 6212 return dr6; 6213 } 6214 6215 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) 6216 { 6217 unsigned long rflags = kvm_x86_call(get_rflags)(vcpu); 6218 int r; 6219 6220 r = kvm_x86_call(skip_emulated_instruction)(vcpu); 6221 if (unlikely(!r)) 6222 return 0; 6223 6224 kvm_pmu_instruction_retired(vcpu); 6225 6226 /* 6227 * rflags is the old, "raw" value of the flags. The new value has 6228 * not been saved yet. 6229 * 6230 * This is correct even for TF set by the guest, because "the 6231 * processor will not generate this exception after the instruction 6232 * that sets the TF flag". 6233 */ 6234 if (unlikely(rflags & X86_EFLAGS_TF)) 6235 r = kvm_inject_emulated_db(vcpu, DR6_BS); 6236 return r; 6237 } 6238 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_skip_emulated_instruction); 6239 6240 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu) 6241 { 6242 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 6243 return false; 6244 6245 if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF) 6246 return true; 6247 6248 /* 6249 * Intel compatible CPUs inhibit code #DBs when MOV/POP SS blocking is 6250 * active, but AMD compatible CPUs do not. 6251 */ 6252 if (!guest_cpuid_is_intel_compatible(vcpu)) 6253 return false; 6254 6255 return kvm_x86_call(get_interrupt_shadow)(vcpu) & KVM_X86_SHADOW_INT_MOV_SS; 6256 } 6257 6258 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, 6259 int emulation_type, int *r) 6260 { 6261 unsigned long dr7 = kvm_get_effective_dr7(vcpu); 6262 6263 WARN_ON_ONCE(emulation_type & EMULTYPE_NO_DECODE); 6264 6265 /* 6266 * Do not check for code breakpoints if hardware has already done the 6267 * checks, as inferred from the emulation type. On NO_DECODE and SKIP, 6268 * the instruction has passed all exception checks, and all intercepted 6269 * exceptions that trigger emulation have lower priority than code 6270 * breakpoints, i.e. the fact that the intercepted exception occurred 6271 * means any code breakpoints have already been serviced. 6272 * 6273 * Note, KVM needs to check for code #DBs on EMULTYPE_TRAP_UD_FORCED as 6274 * hardware has checked the RIP of the magic prefix, but not the RIP of 6275 * the instruction being emulated. The intent of forced emulation is 6276 * to behave as if KVM intercepted the instruction without an exception 6277 * and without a prefix. 6278 */ 6279 if (emulation_type & (EMULTYPE_NO_DECODE | EMULTYPE_SKIP | 6280 EMULTYPE_TRAP_UD | EMULTYPE_VMWARE_GP | EMULTYPE_PF)) 6281 return false; 6282 6283 if (unlikely(dr7 & DR7_BP_EN_MASK) && 6284 !kvm_is_code_breakpoint_inhibited(vcpu)) { 6285 unsigned long eip = kvm_get_linear_rip(vcpu); 6286 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, dr7, 6287 vcpu->arch.eff_db); 6288 6289 if (dr6) { 6290 *r = kvm_inject_emulated_db(vcpu, dr6); 6291 return true; 6292 } 6293 } 6294 6295 return false; 6296 } 6297 6298 static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt) 6299 { 6300 switch (ctxt->opcode_len) { 6301 case 1: 6302 switch (ctxt->b) { 6303 case 0xe4: /* IN */ 6304 case 0xe5: 6305 case 0xec: 6306 case 0xed: 6307 case 0xe6: /* OUT */ 6308 case 0xe7: 6309 case 0xee: 6310 case 0xef: 6311 case 0x6c: /* INS */ 6312 case 0x6d: 6313 case 0x6e: /* OUTS */ 6314 case 0x6f: 6315 return true; 6316 } 6317 break; 6318 case 2: 6319 switch (ctxt->b) { 6320 case 0x33: /* RDPMC */ 6321 return true; 6322 } 6323 break; 6324 } 6325 6326 return false; 6327 } 6328 6329 static bool is_soft_int_instruction(struct x86_emulate_ctxt *ctxt, 6330 int emulation_type) 6331 { 6332 u8 vector = EMULTYPE_GET_SOFT_INT_VECTOR(emulation_type); 6333 6334 switch (ctxt->b) { 6335 case 0xcc: 6336 return vector == BP_VECTOR; 6337 case 0xcd: 6338 return vector == ctxt->src.val; 6339 case 0xce: 6340 return vector == OF_VECTOR; 6341 default: 6342 return false; 6343 } 6344 } 6345 6346 /* 6347 * Decode an instruction for emulation. The caller is responsible for handling 6348 * code breakpoints. Note, manually detecting code breakpoints is unnecessary 6349 * (and wrong) when emulating on an intercepted fault-like exception[*], as 6350 * code breakpoints have higher priority and thus have already been done by 6351 * hardware. 6352 * 6353 * [*] Except #MC, which is higher priority, but KVM should never emulate in 6354 * response to a machine check. 6355 */ 6356 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type, 6357 void *insn, int insn_len) 6358 { 6359 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 6360 int r; 6361 6362 init_emulate_ctxt(vcpu); 6363 6364 r = x86_decode_insn(ctxt, insn, insn_len, emulation_type); 6365 6366 trace_kvm_emulate_insn_start(vcpu); 6367 ++vcpu->stat.insn_emulation; 6368 6369 return r; 6370 } 6371 EXPORT_SYMBOL_FOR_KVM_INTERNAL(x86_decode_emulated_instruction); 6372 6373 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 6374 int emulation_type, void *insn, int insn_len) 6375 { 6376 int r; 6377 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 6378 bool writeback = true; 6379 6380 if ((emulation_type & EMULTYPE_ALLOW_RETRY_PF) && 6381 (WARN_ON_ONCE(is_guest_mode(vcpu)) || 6382 WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))) 6383 emulation_type &= ~EMULTYPE_ALLOW_RETRY_PF; 6384 6385 r = kvm_check_emulate_insn(vcpu, emulation_type, insn, insn_len); 6386 if (r != X86EMUL_CONTINUE) { 6387 if (r == X86EMUL_RETRY_INSTR || r == X86EMUL_PROPAGATE_FAULT) 6388 return 1; 6389 6390 if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa, 6391 emulation_type)) 6392 return 1; 6393 6394 if (r == X86EMUL_UNHANDLEABLE_VECTORING) { 6395 kvm_prepare_event_vectoring_exit(vcpu, cr2_or_gpa); 6396 return 0; 6397 } 6398 6399 WARN_ON_ONCE(r != X86EMUL_UNHANDLEABLE); 6400 return handle_emulation_failure(vcpu, emulation_type); 6401 } 6402 6403 kvm_request_l1tf_flush_l1d(); 6404 6405 if (!(emulation_type & EMULTYPE_NO_DECODE)) { 6406 kvm_clear_exception_queue(vcpu); 6407 6408 /* 6409 * Return immediately if RIP hits a code breakpoint, such #DBs 6410 * are fault-like and are higher priority than any faults on 6411 * the code fetch itself. 6412 */ 6413 if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r)) 6414 return r; 6415 6416 r = x86_decode_emulated_instruction(vcpu, emulation_type, 6417 insn, insn_len); 6418 if (r != EMULATION_OK) { 6419 if ((emulation_type & EMULTYPE_TRAP_UD) || 6420 (emulation_type & EMULTYPE_TRAP_UD_FORCED)) { 6421 kvm_queue_exception(vcpu, UD_VECTOR); 6422 return 1; 6423 } 6424 if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa, 6425 emulation_type)) 6426 return 1; 6427 6428 if (ctxt->have_exception && 6429 !(emulation_type & EMULTYPE_SKIP)) { 6430 /* 6431 * #UD should result in just EMULATION_FAILED, and trap-like 6432 * exception should not be encountered during decode. 6433 */ 6434 WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR || 6435 exception_type(ctxt->exception.vector) == EXCPT_TRAP); 6436 return inject_emulated_exception(vcpu); 6437 } 6438 return handle_emulation_failure(vcpu, emulation_type); 6439 } 6440 } 6441 6442 if ((emulation_type & EMULTYPE_VMWARE_GP) && 6443 !is_vmware_backdoor_opcode(ctxt)) { 6444 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 6445 return 1; 6446 } 6447 6448 /* 6449 * EMULTYPE_SKIP without EMULTYPE_COMPLETE_USER_EXIT is intended for 6450 * use *only* by vendor callbacks for kvm_skip_emulated_instruction(). 6451 * The caller is responsible for updating interruptibility state and 6452 * injecting single-step #DBs. 6453 */ 6454 if (emulation_type & EMULTYPE_SKIP) { 6455 if (emulation_type & EMULTYPE_SKIP_SOFT_INT && 6456 !is_soft_int_instruction(ctxt, emulation_type)) 6457 return 0; 6458 6459 if (ctxt->mode != X86EMUL_MODE_PROT64) 6460 ctxt->eip = (u32)ctxt->_eip; 6461 else 6462 ctxt->eip = ctxt->_eip; 6463 6464 if (emulation_type & EMULTYPE_COMPLETE_USER_EXIT) { 6465 r = 1; 6466 goto writeback; 6467 } 6468 6469 kvm_rip_write(vcpu, ctxt->eip); 6470 if (ctxt->eflags & X86_EFLAGS_RF) 6471 kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF); 6472 return 1; 6473 } 6474 6475 /* 6476 * If emulation was caused by a write-protection #PF on a non-page_table 6477 * writing instruction, try to unprotect the gfn, i.e. zap shadow pages, 6478 * and retry the instruction, as the vCPU is likely no longer using the 6479 * gfn as a page table. 6480 */ 6481 if ((emulation_type & EMULTYPE_ALLOW_RETRY_PF) && 6482 !x86_page_table_writing_insn(ctxt) && 6483 kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa)) 6484 return 1; 6485 6486 /* this is needed for vmware backdoor interface to work since it 6487 changes registers values during IO operation */ 6488 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) { 6489 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 6490 emulator_invalidate_register_cache(ctxt); 6491 } 6492 6493 restart: 6494 if (emulation_type & EMULTYPE_PF) { 6495 /* Save the faulting GPA (cr2) in the address field */ 6496 ctxt->exception.address = cr2_or_gpa; 6497 6498 /* With shadow page tables, cr2 contains a GVA or nGPA. */ 6499 if (vcpu->arch.mmu->root_role.direct) { 6500 ctxt->gpa_access = ACC_READ_MASK; 6501 /* 6502 * Always allow writes for guests with protected page 6503 * tables, as KVM can't walk the guest's page tables, 6504 * i.e. KVM can't get the RMW protections for a given 6505 * GVA to see if the write side of a RMW operation 6506 * should be allowed. 6507 */ 6508 if ((emulation_type & EMULTYPE_PF_WRITE) || 6509 vcpu->kvm->arch.has_protected_page_tables) 6510 ctxt->gpa_access |= ACC_WRITE_MASK; 6511 ctxt->gpa_val = cr2_or_gpa; 6512 } 6513 } else { 6514 /* Sanitize the address out of an abundance of paranoia. */ 6515 ctxt->exception.address = 0; 6516 } 6517 6518 /* 6519 * Check L1's instruction intercepts when emulating instructions for 6520 * L2, unless KVM is re-emulating a previously decoded instruction, 6521 * e.g. to complete userspace I/O, in which case KVM has already 6522 * checked the intercepts. 6523 */ 6524 r = x86_emulate_insn(ctxt, is_guest_mode(vcpu) && 6525 !(emulation_type & EMULTYPE_NO_DECODE)); 6526 6527 if (r == EMULATION_INTERCEPTED) 6528 return 1; 6529 6530 if (r == EMULATION_FAILED) { 6531 if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa, 6532 emulation_type)) 6533 return 1; 6534 6535 return handle_emulation_failure(vcpu, emulation_type); 6536 } 6537 6538 if (ctxt->have_exception) { 6539 WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write); 6540 vcpu->mmio_needed = false; 6541 r = inject_emulated_exception(vcpu); 6542 } else if (vcpu->arch.pio.count) { 6543 if (!vcpu->arch.pio.in) { 6544 /* FIXME: return into emulator if single-stepping. */ 6545 vcpu->arch.pio.count = 0; 6546 } else { 6547 writeback = false; 6548 vcpu->arch.complete_userspace_io = complete_emulated_pio; 6549 } 6550 r = 0; 6551 } else if (vcpu->mmio_needed) { 6552 ++vcpu->stat.mmio_exits; 6553 6554 if (!vcpu->mmio_is_write) 6555 writeback = false; 6556 r = 0; 6557 vcpu->arch.complete_userspace_io = complete_emulated_mmio; 6558 } else if (vcpu->arch.complete_userspace_io) { 6559 writeback = false; 6560 r = 0; 6561 } else if (r == EMULATION_RESTART) { 6562 cond_resched(); 6563 goto restart; 6564 } else { 6565 r = 1; 6566 } 6567 6568 writeback: 6569 if (writeback) { 6570 unsigned long rflags = kvm_x86_call(get_rflags)(vcpu); 6571 toggle_interruptibility(vcpu, ctxt->interruptibility); 6572 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 6573 6574 /* 6575 * Note, EXCPT_DB is assumed to be fault-like as the emulator 6576 * only supports code breakpoints and general detect #DB, both 6577 * of which are fault-like. 6578 */ 6579 if (!ctxt->have_exception || 6580 exception_type(ctxt->exception.vector) == EXCPT_TRAP) { 6581 kvm_pmu_instruction_retired(vcpu); 6582 if (ctxt->is_branch) 6583 kvm_pmu_branch_retired(vcpu); 6584 kvm_rip_write(vcpu, ctxt->eip); 6585 if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP))) 6586 r = kvm_inject_emulated_db(vcpu, DR6_BS); 6587 kvm_x86_call(update_emulated_instruction)(vcpu); 6588 __kvm_set_rflags(vcpu, ctxt->eflags); 6589 } 6590 6591 /* 6592 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will 6593 * do nothing, and it will be requested again as soon as 6594 * the shadow expires. But we still need to check here, 6595 * because POPF has no interrupt shadow. 6596 */ 6597 if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF)) 6598 kvm_make_request(KVM_REQ_EVENT, vcpu); 6599 } else 6600 vcpu->arch.emulate_regs_need_sync_to_vcpu = true; 6601 6602 return r; 6603 } 6604 6605 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type) 6606 { 6607 return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0); 6608 } 6609 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_instruction); 6610 6611 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, 6612 void *insn, int insn_len) 6613 { 6614 return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len); 6615 } 6616 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_instruction_from_buffer); 6617 6618 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu) 6619 { 6620 vcpu->arch.pio.count = 0; 6621 return 1; 6622 } 6623 6624 static int complete_fast_pio_out(struct kvm_vcpu *vcpu) 6625 { 6626 vcpu->arch.pio.count = 0; 6627 6628 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.cui_linear_rip))) 6629 return 1; 6630 6631 return kvm_skip_emulated_instruction(vcpu); 6632 } 6633 6634 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, 6635 unsigned short port) 6636 { 6637 unsigned long val = kvm_rax_read_raw(vcpu); 6638 int ret = emulator_pio_out(vcpu, size, port, &val, 1); 6639 6640 if (ret) 6641 return ret; 6642 6643 /* 6644 * Workaround userspace that relies on old KVM behavior of %rip being 6645 * incremented prior to exiting to userspace to handle "OUT 0x7e". 6646 */ 6647 if (port == 0x7e && 6648 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) { 6649 vcpu->arch.complete_userspace_io = 6650 complete_fast_pio_out_port_0x7e; 6651 kvm_skip_emulated_instruction(vcpu); 6652 } else { 6653 vcpu->arch.cui_linear_rip = kvm_get_linear_rip(vcpu); 6654 vcpu->arch.complete_userspace_io = complete_fast_pio_out; 6655 } 6656 return 0; 6657 } 6658 6659 static int complete_fast_pio_in(struct kvm_vcpu *vcpu) 6660 { 6661 unsigned long val; 6662 6663 /* We should only ever be called with arch.pio.count equal to 1 */ 6664 if (KVM_BUG_ON(vcpu->arch.pio.count != 1, vcpu->kvm)) 6665 return -EIO; 6666 6667 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.cui_linear_rip))) { 6668 vcpu->arch.pio.count = 0; 6669 return 1; 6670 } 6671 6672 /* For size less than 4 we merge, else we zero extend */ 6673 val = (vcpu->arch.pio.size < 4) ? kvm_rax_read_raw(vcpu) : 0; 6674 6675 complete_emulator_pio_in(vcpu, &val); 6676 kvm_rax_write_raw(vcpu, val); 6677 6678 return kvm_skip_emulated_instruction(vcpu); 6679 } 6680 6681 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, 6682 unsigned short port) 6683 { 6684 unsigned long val; 6685 int ret; 6686 6687 /* For size less than 4 we merge, else we zero extend */ 6688 val = (size < 4) ? kvm_rax_read_raw(vcpu) : 0; 6689 6690 ret = emulator_pio_in(vcpu, size, port, &val, 1); 6691 if (ret) { 6692 kvm_rax_write_raw(vcpu, val); 6693 return ret; 6694 } 6695 6696 vcpu->arch.cui_linear_rip = kvm_get_linear_rip(vcpu); 6697 vcpu->arch.complete_userspace_io = complete_fast_pio_in; 6698 6699 return 0; 6700 } 6701 6702 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in) 6703 { 6704 int ret; 6705 6706 if (in) 6707 ret = kvm_fast_pio_in(vcpu, size, port); 6708 else 6709 ret = kvm_fast_pio_out(vcpu, size, port); 6710 return ret && kvm_skip_emulated_instruction(vcpu); 6711 } 6712 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fast_pio); 6713 6714 static int kvmclock_cpu_down_prep(unsigned int cpu) 6715 { 6716 __this_cpu_write(cpu_tsc_khz, 0); 6717 return 0; 6718 } 6719 6720 static void tsc_khz_changed(void *data) 6721 { 6722 struct cpufreq_freqs *freq = data; 6723 unsigned long khz; 6724 6725 WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_CONSTANT_TSC)); 6726 6727 if (data) 6728 khz = freq->new; 6729 else 6730 khz = cpufreq_quick_get(raw_smp_processor_id()); 6731 if (!khz) 6732 khz = tsc_khz; 6733 __this_cpu_write(cpu_tsc_khz, khz); 6734 } 6735 6736 #ifdef CONFIG_X86_64 6737 static void kvm_hyperv_tsc_notifier(void) 6738 { 6739 struct kvm *kvm; 6740 int cpu; 6741 6742 mutex_lock(&kvm_lock); 6743 list_for_each_entry(kvm, &vm_list, vm_list) 6744 kvm_make_mclock_inprogress_request(kvm); 6745 6746 /* no guest entries from this point */ 6747 hyperv_stop_tsc_emulation(); 6748 6749 /* TSC frequency always matches when on Hyper-V */ 6750 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 6751 for_each_present_cpu(cpu) 6752 per_cpu(cpu_tsc_khz, cpu) = tsc_khz; 6753 } 6754 kvm_caps.max_guest_tsc_khz = tsc_khz; 6755 6756 list_for_each_entry(kvm, &vm_list, vm_list) { 6757 __kvm_start_pvclock_update(kvm); 6758 pvclock_update_vm_gtod_copy(kvm); 6759 kvm_end_pvclock_update(kvm); 6760 } 6761 6762 mutex_unlock(&kvm_lock); 6763 } 6764 #endif 6765 6766 static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu) 6767 { 6768 struct kvm *kvm; 6769 struct kvm_vcpu *vcpu; 6770 int send_ipi = 0; 6771 unsigned long i; 6772 6773 /* 6774 * We allow guests to temporarily run on slowing clocks, 6775 * provided we notify them after, or to run on accelerating 6776 * clocks, provided we notify them before. Thus time never 6777 * goes backwards. 6778 * 6779 * However, we have a problem. We can't atomically update 6780 * the frequency of a given CPU from this function; it is 6781 * merely a notifier, which can be called from any CPU. 6782 * Changing the TSC frequency at arbitrary points in time 6783 * requires a recomputation of local variables related to 6784 * the TSC for each VCPU. We must flag these local variables 6785 * to be updated and be sure the update takes place with the 6786 * new frequency before any guests proceed. 6787 * 6788 * Unfortunately, the combination of hotplug CPU and frequency 6789 * change creates an intractable locking scenario; the order 6790 * of when these callouts happen is undefined with respect to 6791 * CPU hotplug, and they can race with each other. As such, 6792 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is 6793 * undefined; you can actually have a CPU frequency change take 6794 * place in between the computation of X and the setting of the 6795 * variable. To protect against this problem, all updates of 6796 * the per_cpu tsc_khz variable are done in an interrupt 6797 * protected IPI, and all callers wishing to update the value 6798 * must wait for a synchronous IPI to complete (which is trivial 6799 * if the caller is on the CPU already). This establishes the 6800 * necessary total order on variable updates. 6801 * 6802 * Note that because a guest time update may take place 6803 * anytime after the setting of the VCPU's request bit, the 6804 * correct TSC value must be set before the request. However, 6805 * to ensure the update actually makes it to any guest which 6806 * starts running in hardware virtualization between the set 6807 * and the acquisition of the spinlock, we must also ping the 6808 * CPU after setting the request bit. 6809 * 6810 */ 6811 6812 smp_call_function_single(cpu, tsc_khz_changed, freq, 1); 6813 6814 mutex_lock(&kvm_lock); 6815 list_for_each_entry(kvm, &vm_list, vm_list) { 6816 kvm_for_each_vcpu(i, vcpu, kvm) { 6817 if (vcpu->cpu != cpu) 6818 continue; 6819 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 6820 if (vcpu->cpu != raw_smp_processor_id()) 6821 send_ipi = 1; 6822 } 6823 } 6824 mutex_unlock(&kvm_lock); 6825 6826 if (freq->old < freq->new && send_ipi) { 6827 /* 6828 * We upscale the frequency. Must make the guest 6829 * doesn't see old kvmclock values while running with 6830 * the new frequency, otherwise we risk the guest sees 6831 * time go backwards. 6832 * 6833 * In case we update the frequency for another cpu 6834 * (which might be in guest context) send an interrupt 6835 * to kick the cpu out of guest context. Next time 6836 * guest context is entered kvmclock will be updated, 6837 * so the guest will not see stale values. 6838 */ 6839 smp_call_function_single(cpu, tsc_khz_changed, freq, 1); 6840 } 6841 } 6842 6843 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 6844 void *data) 6845 { 6846 struct cpufreq_freqs *freq = data; 6847 int cpu; 6848 6849 if (val == CPUFREQ_PRECHANGE && freq->old > freq->new) 6850 return 0; 6851 if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new) 6852 return 0; 6853 6854 for_each_cpu(cpu, freq->policy->cpus) 6855 __kvmclock_cpufreq_notifier(freq, cpu); 6856 6857 return 0; 6858 } 6859 6860 static struct notifier_block kvmclock_cpufreq_notifier_block = { 6861 .notifier_call = kvmclock_cpufreq_notifier 6862 }; 6863 6864 static int kvmclock_cpu_online(unsigned int cpu) 6865 { 6866 tsc_khz_changed(NULL); 6867 return 0; 6868 } 6869 6870 static void kvm_timer_init(void) 6871 { 6872 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 6873 max_tsc_khz = tsc_khz; 6874 6875 if (IS_ENABLED(CONFIG_CPU_FREQ)) { 6876 struct cpufreq_policy *policy; 6877 int cpu; 6878 6879 cpu = get_cpu(); 6880 policy = cpufreq_cpu_get(cpu); 6881 if (policy) { 6882 if (policy->cpuinfo.max_freq) 6883 max_tsc_khz = policy->cpuinfo.max_freq; 6884 cpufreq_cpu_put(policy); 6885 } 6886 put_cpu(); 6887 } 6888 cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block, 6889 CPUFREQ_TRANSITION_NOTIFIER); 6890 6891 cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online", 6892 kvmclock_cpu_online, kvmclock_cpu_down_prep); 6893 } 6894 } 6895 6896 #ifdef CONFIG_X86_64 6897 static void pvclock_gtod_update_fn(struct work_struct *work) 6898 { 6899 struct kvm *kvm; 6900 struct kvm_vcpu *vcpu; 6901 unsigned long i; 6902 6903 mutex_lock(&kvm_lock); 6904 list_for_each_entry(kvm, &vm_list, vm_list) 6905 kvm_for_each_vcpu(i, vcpu, kvm) 6906 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 6907 atomic_set(&kvm_guest_has_master_clock, 0); 6908 mutex_unlock(&kvm_lock); 6909 } 6910 6911 static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn); 6912 6913 /* 6914 * Indirection to move queue_work() out of the tk_core.seq write held 6915 * region to prevent possible deadlocks against time accessors which 6916 * are invoked with work related locks held. 6917 */ 6918 static void pvclock_irq_work_fn(struct irq_work *w) 6919 { 6920 queue_work(system_long_wq, &pvclock_gtod_work); 6921 } 6922 6923 static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn); 6924 6925 /* 6926 * Notification about pvclock gtod data update. 6927 */ 6928 static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused, 6929 void *priv) 6930 { 6931 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 6932 struct timekeeper *tk = priv; 6933 6934 update_pvclock_gtod(tk); 6935 6936 /* 6937 * Disable master clock if host does not trust, or does not use, 6938 * TSC based clocksource. Delegate queue_work() to irq_work as 6939 * this is invoked with tk_core.seq write held. 6940 */ 6941 if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) && 6942 atomic_read(&kvm_guest_has_master_clock) != 0) 6943 irq_work_queue(&pvclock_irq_work); 6944 return 0; 6945 } 6946 6947 static struct notifier_block pvclock_gtod_notifier = { 6948 .notifier_call = pvclock_gtod_notify, 6949 }; 6950 #endif 6951 6952 void kvm_setup_xss_caps(void) 6953 { 6954 if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) 6955 kvm_caps.supported_xss = 0; 6956 6957 if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && 6958 !kvm_cpu_cap_has(X86_FEATURE_IBT)) 6959 kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; 6960 6961 if ((kvm_caps.supported_xss & XFEATURE_MASK_CET_ALL) != XFEATURE_MASK_CET_ALL) { 6962 kvm_cpu_cap_clear(X86_FEATURE_SHSTK); 6963 kvm_cpu_cap_clear(X86_FEATURE_IBT); 6964 kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; 6965 } 6966 } 6967 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_setup_xss_caps); 6968 6969 static void kvm_setup_efer_caps(void) 6970 { 6971 /* Enable syscall by default because its emulated by KVM */ 6972 kvm_caps.supported_efer_bits = (u64)EFER_SCE; 6973 6974 if (kvm_cpu_cap_has(X86_FEATURE_LM)) 6975 kvm_caps.supported_efer_bits |= (EFER_LME | EFER_LMA); 6976 6977 if (kvm_cpu_cap_has(X86_FEATURE_NX)) 6978 kvm_caps.supported_efer_bits |= EFER_NX; 6979 6980 if (kvm_cpu_cap_has(X86_FEATURE_FXSR_OPT)) 6981 kvm_caps.supported_efer_bits |= EFER_FFXSR; 6982 6983 if (kvm_cpu_cap_has(X86_FEATURE_AUTOIBRS)) 6984 kvm_caps.supported_efer_bits |= EFER_AUTOIBRS; 6985 6986 if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { 6987 kvm_caps.supported_efer_bits |= EFER_SVME; 6988 6989 /* 6990 * Enumerating EFER_LMSLE_MBZ and allowing EFER.LMSLE=1 6991 * would be nonsensical. Note, vendor code sets the defeature 6992 * if KVM can't support EFER.LMSLE for any reason, i.e. this 6993 * needs to consult KVM's capabilities, not just raw CPUID. 6994 */ 6995 if (!kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) 6996 kvm_caps.supported_efer_bits |= EFER_LMSLE; 6997 } 6998 } 6999 7000 static void kvm_nested_ops_update(const struct kvm_x86_nested_ops *nested_ops) 7001 { 7002 memcpy(&kvm_nested_ops, nested_ops, sizeof(kvm_nested_ops)); 7003 7004 #define __KVM_X86_NESTED_OP(func) \ 7005 static_call_update(kvm_x86_nested_##func, kvm_nested_ops.func); 7006 #define KVM_X86_NESTED_OP(func) \ 7007 WARN_ON(!kvm_nested_ops.func); __KVM_X86_NESTED_OP(func) 7008 #define KVM_X86_NESTED_OP_OPTIONAL __KVM_X86_NESTED_OP 7009 #define KVM_X86_NESTED_OP_OPTIONAL_RET0(func) \ 7010 static_call_update(kvm_x86_nested_##func, (void *)kvm_nested_ops.func ? : \ 7011 (void *)__static_call_return0); 7012 #include <asm/kvm-x86-nested-ops.h> 7013 #undef __KVM_X86_NESTED_OP 7014 } 7015 7016 static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) 7017 { 7018 memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); 7019 7020 #define __KVM_X86_OP(func) \ 7021 static_call_update(kvm_x86_##func, kvm_x86_ops.func); 7022 #define KVM_X86_OP(func) \ 7023 WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func) 7024 #define KVM_X86_OP_OPTIONAL __KVM_X86_OP 7025 #define KVM_X86_OP_OPTIONAL_RET0(func) \ 7026 static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \ 7027 (void *)__static_call_return0); 7028 #include <asm/kvm-x86-ops.h> 7029 #undef __KVM_X86_OP 7030 7031 kvm_nested_ops_update(ops->nested_ops); 7032 7033 kvm_pmu_ops_update(ops->pmu_ops); 7034 } 7035 7036 static int kvm_x86_check_processor_compatibility(void) 7037 { 7038 int cpu = smp_processor_id(); 7039 struct cpuinfo_x86 *c = &cpu_data(cpu); 7040 7041 /* 7042 * Compatibility checks are done when loading KVM and when enabling 7043 * hardware, e.g. during CPU hotplug, to ensure all online CPUs are 7044 * compatible, i.e. KVM should never perform a compatibility check on 7045 * an offline CPU. 7046 */ 7047 WARN_ON(!cpu_online(cpu)); 7048 7049 if (__cr4_reserved_bits(cpu_has, c) != 7050 __cr4_reserved_bits(cpu_has, &boot_cpu_data)) 7051 return -EIO; 7052 7053 return kvm_x86_call(check_processor_compatibility)(); 7054 } 7055 7056 static void kvm_x86_check_cpu_compat(void *ret) 7057 { 7058 *(int *)ret = kvm_x86_check_processor_compatibility(); 7059 } 7060 7061 int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) 7062 { 7063 u64 host_pat; 7064 int r, cpu; 7065 7066 guard(mutex)(&vendor_module_lock); 7067 7068 if (kvm_x86_ops.enable_virtualization_cpu) { 7069 pr_err("already loaded vendor module '%s'\n", kvm_x86_ops.name); 7070 return -EEXIST; 7071 } 7072 7073 /* 7074 * KVM explicitly assumes that the guest has an FPU and 7075 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the 7076 * vCPU's FPU state as a fxregs_state struct. 7077 */ 7078 if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) { 7079 pr_err("inadequate fpu\n"); 7080 return -EOPNOTSUPP; 7081 } 7082 7083 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 7084 pr_err("RT requires X86_FEATURE_CONSTANT_TSC\n"); 7085 return -EOPNOTSUPP; 7086 } 7087 7088 /* 7089 * KVM assumes that PAT entry '0' encodes WB memtype and simply zeroes 7090 * the PAT bits in SPTEs. Bail if PAT[0] is programmed to something 7091 * other than WB. Note, EPT doesn't utilize the PAT, but don't bother 7092 * with an exception. PAT[0] is set to WB on RESET and also by the 7093 * kernel, i.e. failure indicates a kernel bug or broken firmware. 7094 */ 7095 if (rdmsrq_safe(MSR_IA32_CR_PAT, &host_pat) || 7096 (host_pat & GENMASK(2, 0)) != 6) { 7097 pr_err("host PAT[0] is not WB\n"); 7098 return -EIO; 7099 } 7100 7101 if (boot_cpu_has(X86_FEATURE_SHSTK) || boot_cpu_has(X86_FEATURE_IBT)) { 7102 rdmsrq(MSR_IA32_S_CET, kvm_host.s_cet); 7103 /* 7104 * Linux doesn't yet support supervisor shadow stacks (SSS), so 7105 * KVM doesn't save/restore the associated MSRs, i.e. KVM may 7106 * clobber the host values. Yell and refuse to load if SSS is 7107 * unexpectedly enabled, e.g. to avoid crashing the host. 7108 */ 7109 if (WARN_ON_ONCE(kvm_host.s_cet & CET_SHSTK_EN)) 7110 return -EIO; 7111 } 7112 7113 memset(&kvm_caps, 0, sizeof(kvm_caps)); 7114 7115 x86_emulator_cache = kvm_alloc_emulator_cache(); 7116 if (!x86_emulator_cache) { 7117 pr_err("failed to allocate cache for x86 emulator\n"); 7118 return -ENOMEM; 7119 } 7120 7121 r = kvm_mmu_vendor_module_init(); 7122 if (r) 7123 goto out_free_x86_emulator_cache; 7124 7125 kvm_caps.supported_vm_types = BIT(KVM_X86_DEFAULT_VM); 7126 kvm_caps.supported_mce_cap = MCG_CTL_P | MCG_SER_P; 7127 7128 if (boot_cpu_has(X86_FEATURE_XSAVE)) { 7129 kvm_host.xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 7130 kvm_caps.supported_xcr0 = kvm_host.xcr0 & KVM_SUPPORTED_XCR0; 7131 } 7132 7133 if (boot_cpu_has(X86_FEATURE_XSAVES)) { 7134 rdmsrq(MSR_IA32_XSS, kvm_host.xss); 7135 kvm_caps.supported_xss = kvm_host.xss & KVM_SUPPORTED_XSS; 7136 } 7137 7138 kvm_caps.supported_quirks = KVM_X86_VALID_QUIRKS; 7139 kvm_caps.inapplicable_quirks = KVM_X86_CONDITIONAL_QUIRKS; 7140 7141 rdmsrq_safe(MSR_EFER, &kvm_host.efer); 7142 7143 kvm_init_pmu_capability(ops->pmu_ops); 7144 7145 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) 7146 rdmsrq(MSR_IA32_ARCH_CAPABILITIES, kvm_host.arch_capabilities); 7147 7148 WARN_ON_ONCE(kvm_nr_uret_msrs); 7149 7150 r = ops->hardware_setup(); 7151 if (r != 0) 7152 goto out_mmu_exit; 7153 7154 kvm_setup_efer_caps(); 7155 7156 enable_device_posted_irqs &= enable_apicv && 7157 irq_remapping_cap(IRQ_POSTING_CAP); 7158 7159 kvm_ops_update(ops); 7160 7161 for_each_online_cpu(cpu) { 7162 smp_call_function_single(cpu, kvm_x86_check_cpu_compat, &r, 1); 7163 if (r < 0) 7164 goto out_unwind_ops; 7165 } 7166 7167 /* 7168 * Point of no return! DO NOT add error paths below this point unless 7169 * absolutely necessary, as most operations from this point forward 7170 * require unwinding. 7171 */ 7172 kvm_timer_init(); 7173 7174 if (pi_inject_timer == -1) 7175 pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER); 7176 #ifdef CONFIG_X86_64 7177 pvclock_gtod_register_notifier(&pvclock_gtod_notifier); 7178 7179 if (hypervisor_is_type(X86_HYPER_MS_HYPERV)) 7180 set_hv_tscchange_cb(kvm_hyperv_tsc_notifier); 7181 #endif 7182 7183 __kvm_register_perf_callbacks(ops->handle_intel_pt_intr, 7184 enable_mediated_pmu ? kvm_handle_guest_mediated_pmi : NULL); 7185 7186 if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && tdp_mmu_enabled) 7187 kvm_caps.supported_vm_types |= BIT(KVM_X86_SW_PROTECTED_VM); 7188 7189 /* KVM always ignores guest PAT for shadow paging. */ 7190 if (!tdp_enabled) 7191 kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; 7192 7193 if (kvm_caps.has_tsc_control) { 7194 /* 7195 * Make sure the user can only configure tsc_khz values that 7196 * fit into a signed integer, otherwise KVM_GET_TSC_KHZ would 7197 * return a negative value and confuse userspace. 7198 * A min value is not calculated because it will always 7199 * be 1 on all machines. 7200 */ 7201 u64 max = min(0x7fffffffULL, 7202 __scale_tsc(kvm_caps.max_tsc_scaling_ratio, tsc_khz)); 7203 kvm_caps.max_guest_tsc_khz = max; 7204 } 7205 kvm_caps.default_tsc_scaling_ratio = 1ULL << kvm_caps.tsc_scaling_ratio_frac_bits; 7206 kvm_init_msr_lists(); 7207 return 0; 7208 7209 out_unwind_ops: 7210 kvm_x86_ops.enable_virtualization_cpu = NULL; 7211 kvm_x86_call(hardware_unsetup)(); 7212 out_mmu_exit: 7213 kvm_destroy_user_return_msrs(); 7214 kvm_mmu_vendor_module_exit(); 7215 out_free_x86_emulator_cache: 7216 kmem_cache_destroy(x86_emulator_cache); 7217 return r; 7218 } 7219 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_x86_vendor_init); 7220 7221 void kvm_x86_vendor_exit(void) 7222 { 7223 kvm_unregister_perf_callbacks(); 7224 7225 #ifdef CONFIG_X86_64 7226 if (hypervisor_is_type(X86_HYPER_MS_HYPERV)) 7227 clear_hv_tscchange_cb(); 7228 #endif 7229 kvm_lapic_exit(); 7230 7231 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 7232 cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block, 7233 CPUFREQ_TRANSITION_NOTIFIER); 7234 cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE); 7235 } 7236 #ifdef CONFIG_X86_64 7237 pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier); 7238 irq_work_sync(&pvclock_irq_work); 7239 cancel_work_sync(&pvclock_gtod_work); 7240 #endif 7241 kvm_x86_call(hardware_unsetup)(); 7242 kvm_destroy_user_return_msrs(); 7243 kvm_mmu_vendor_module_exit(); 7244 kmem_cache_destroy(x86_emulator_cache); 7245 #ifdef CONFIG_KVM_XEN 7246 static_key_deferred_flush(&kvm_xen_enabled); 7247 WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key)); 7248 #endif 7249 mutex_lock(&vendor_module_lock); 7250 kvm_x86_ops.enable_virtualization_cpu = NULL; 7251 mutex_unlock(&vendor_module_lock); 7252 } 7253 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_x86_vendor_exit); 7254 7255 #ifdef CONFIG_X86_64 7256 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr, 7257 unsigned long clock_type) 7258 { 7259 struct kvm_clock_pairing clock_pairing; 7260 struct timespec64 ts; 7261 u64 cycle; 7262 int ret; 7263 7264 if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK) 7265 return -KVM_EOPNOTSUPP; 7266 7267 /* 7268 * When tsc is in permanent catchup mode guests won't be able to use 7269 * pvclock_read_retry loop to get consistent view of pvclock 7270 */ 7271 if (vcpu->arch.tsc_always_catchup) 7272 return -KVM_EOPNOTSUPP; 7273 7274 if (!kvm_get_walltime_and_clockread(&ts, &cycle)) 7275 return -KVM_EOPNOTSUPP; 7276 7277 clock_pairing.sec = ts.tv_sec; 7278 clock_pairing.nsec = ts.tv_nsec; 7279 clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle); 7280 clock_pairing.flags = 0; 7281 memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad)); 7282 7283 ret = 0; 7284 if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing, 7285 sizeof(struct kvm_clock_pairing))) 7286 ret = -KVM_EFAULT; 7287 7288 return ret; 7289 } 7290 #endif 7291 7292 /* 7293 * kvm_pv_kick_cpu_op: Kick a vcpu. 7294 * 7295 * @apicid - apicid of vcpu to be kicked. 7296 */ 7297 static void kvm_pv_kick_cpu_op(struct kvm *kvm, int apicid) 7298 { 7299 /* 7300 * All other fields are unused for APIC_DM_REMRD, but may be consumed by 7301 * common code, e.g. for tracing. Defer initialization to the compiler. 7302 */ 7303 struct kvm_lapic_irq lapic_irq = { 7304 .delivery_mode = APIC_DM_REMRD, 7305 .dest_mode = APIC_DEST_PHYSICAL, 7306 .shorthand = APIC_DEST_NOSHORT, 7307 .dest_id = apicid, 7308 }; 7309 7310 kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq); 7311 } 7312 7313 bool kvm_apicv_activated(struct kvm *kvm) 7314 { 7315 return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0); 7316 } 7317 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_apicv_activated); 7318 7319 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu) 7320 { 7321 ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons); 7322 ulong vcpu_reasons = 7323 kvm_x86_call(vcpu_get_apicv_inhibit_reasons)(vcpu); 7324 7325 return (vm_reasons | vcpu_reasons) == 0; 7326 } 7327 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_apicv_activated); 7328 7329 static void set_or_clear_apicv_inhibit(unsigned long *inhibits, 7330 enum kvm_apicv_inhibit reason, bool set) 7331 { 7332 const struct trace_print_flags apicv_inhibits[] = { APICV_INHIBIT_REASONS }; 7333 7334 BUILD_BUG_ON(ARRAY_SIZE(apicv_inhibits) != NR_APICV_INHIBIT_REASONS); 7335 7336 if (set) 7337 __set_bit(reason, inhibits); 7338 else 7339 __clear_bit(reason, inhibits); 7340 7341 trace_kvm_apicv_inhibit_changed(reason, set, *inhibits); 7342 } 7343 7344 static void kvm_apicv_init(struct kvm *kvm) 7345 { 7346 enum kvm_apicv_inhibit reason = enable_apicv ? APICV_INHIBIT_REASON_ABSENT : 7347 APICV_INHIBIT_REASON_DISABLED; 7348 7349 set_or_clear_apicv_inhibit(&kvm->arch.apicv_inhibit_reasons, reason, true); 7350 7351 init_rwsem(&kvm->arch.apicv_update_lock); 7352 } 7353 7354 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id) 7355 { 7356 struct kvm_vcpu *target = NULL; 7357 struct kvm_apic_map *map; 7358 7359 vcpu->stat.directed_yield_attempted++; 7360 7361 if (single_task_running()) 7362 goto no_yield; 7363 7364 rcu_read_lock(); 7365 map = rcu_dereference(vcpu->kvm->arch.apic_map); 7366 7367 if (likely(map) && dest_id <= map->max_apic_id) { 7368 dest_id = array_index_nospec(dest_id, map->max_apic_id + 1); 7369 if (map->phys_map[dest_id]) 7370 target = map->phys_map[dest_id]->vcpu; 7371 } 7372 7373 rcu_read_unlock(); 7374 7375 if (!target || !READ_ONCE(target->ready)) 7376 goto no_yield; 7377 7378 /* Ignore requests to yield to self */ 7379 if (vcpu == target) 7380 goto no_yield; 7381 7382 if (kvm_vcpu_yield_to(target) <= 0) 7383 goto no_yield; 7384 7385 vcpu->stat.directed_yield_successful++; 7386 7387 no_yield: 7388 return; 7389 } 7390 7391 static int complete_hypercall_exit(struct kvm_vcpu *vcpu) 7392 { 7393 u64 ret = vcpu->run->hypercall.ret; 7394 7395 if (!is_64_bit_hypercall(vcpu)) 7396 ret = (u32)ret; 7397 kvm_rax_write_raw(vcpu, ret); 7398 return kvm_skip_emulated_instruction(vcpu); 7399 } 7400 7401 int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl, 7402 int (*complete_hypercall)(struct kvm_vcpu *)) 7403 { 7404 int op_64_bit = is_64_bit_hypercall(vcpu); 7405 unsigned long ret, nr, a0, a1, a2, a3; 7406 7407 ++vcpu->stat.hypercalls; 7408 7409 if (op_64_bit) { 7410 nr = kvm_rax_read_raw(vcpu); 7411 a0 = kvm_rbx_read_raw(vcpu); 7412 a1 = kvm_rcx_read_raw(vcpu); 7413 a2 = kvm_rdx_read_raw(vcpu); 7414 a3 = kvm_rsi_read_raw(vcpu); 7415 } else { 7416 nr = kvm_eax_read(vcpu); 7417 a0 = kvm_ebx_read(vcpu); 7418 a1 = kvm_ecx_read(vcpu); 7419 a2 = kvm_edx_read(vcpu); 7420 a3 = kvm_esi_read(vcpu); 7421 } 7422 7423 trace_kvm_hypercall(nr, a0, a1, a2, a3); 7424 7425 if (cpl) { 7426 ret = -KVM_EPERM; 7427 goto out; 7428 } 7429 7430 ret = -KVM_ENOSYS; 7431 7432 switch (nr) { 7433 case KVM_HC_VAPIC_POLL_IRQ: 7434 ret = 0; 7435 break; 7436 case KVM_HC_KICK_CPU: 7437 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT)) 7438 break; 7439 7440 kvm_pv_kick_cpu_op(vcpu->kvm, a1); 7441 kvm_sched_yield(vcpu, a1); 7442 ret = 0; 7443 break; 7444 #ifdef CONFIG_X86_64 7445 case KVM_HC_CLOCK_PAIRING: 7446 ret = kvm_pv_clock_pairing(vcpu, a0, a1); 7447 break; 7448 #endif 7449 case KVM_HC_SEND_IPI: 7450 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI)) 7451 break; 7452 7453 ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit); 7454 break; 7455 case KVM_HC_SCHED_YIELD: 7456 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD)) 7457 break; 7458 7459 kvm_sched_yield(vcpu, a0); 7460 ret = 0; 7461 break; 7462 case KVM_HC_MAP_GPA_RANGE: { 7463 u64 gpa = a0, npages = a1, attrs = a2; 7464 7465 ret = -KVM_ENOSYS; 7466 if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE)) 7467 break; 7468 7469 if (!PAGE_ALIGNED(gpa) || !npages || 7470 gpa_to_gfn(gpa) + npages <= gpa_to_gfn(gpa)) { 7471 ret = -KVM_EINVAL; 7472 break; 7473 } 7474 7475 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; 7476 vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE; 7477 /* 7478 * In principle this should have been -KVM_ENOSYS, but userspace (QEMU <=9.2) 7479 * assumed that vcpu->run->hypercall.ret is never changed by KVM and thus that 7480 * it was always zero on KVM_EXIT_HYPERCALL. Since KVM is now overwriting 7481 * vcpu->run->hypercall.ret, ensuring that it is zero to not break QEMU. 7482 */ 7483 vcpu->run->hypercall.ret = 0; 7484 vcpu->run->hypercall.args[0] = gpa; 7485 vcpu->run->hypercall.args[1] = npages; 7486 vcpu->run->hypercall.args[2] = attrs; 7487 vcpu->run->hypercall.flags = 0; 7488 if (op_64_bit) 7489 vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE; 7490 7491 WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ); 7492 vcpu->arch.complete_userspace_io = complete_hypercall; 7493 return 0; 7494 } 7495 default: 7496 ret = -KVM_ENOSYS; 7497 break; 7498 } 7499 7500 out: 7501 vcpu->run->hypercall.ret = ret; 7502 return 1; 7503 } 7504 EXPORT_SYMBOL_FOR_KVM_INTERNAL(____kvm_emulate_hypercall); 7505 7506 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) 7507 { 7508 if (kvm_xen_hypercall_enabled(vcpu->kvm)) 7509 return kvm_xen_hypercall(vcpu); 7510 7511 if (kvm_hv_hypercall_enabled(vcpu)) 7512 return kvm_hv_hypercall(vcpu); 7513 7514 return __kvm_emulate_hypercall(vcpu, kvm_x86_call(get_cpl)(vcpu), 7515 complete_hypercall_exit); 7516 } 7517 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_hypercall); 7518 7519 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt) 7520 { 7521 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7522 char instruction[3]; 7523 unsigned long rip = kvm_rip_read(vcpu); 7524 7525 /* 7526 * If the quirk is disabled, synthesize a #UD and let the guest pick up 7527 * the pieces. 7528 */ 7529 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) { 7530 ctxt->exception.error_code_valid = false; 7531 ctxt->exception.vector = UD_VECTOR; 7532 ctxt->have_exception = true; 7533 return X86EMUL_PROPAGATE_FAULT; 7534 } 7535 7536 kvm_x86_call(patch_hypercall)(vcpu, instruction); 7537 7538 return emulator_write_emulated(ctxt, rip, instruction, 3, 7539 &ctxt->exception); 7540 } 7541 7542 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu) 7543 { 7544 return vcpu->run->request_interrupt_window && 7545 likely(!pic_in_kernel(vcpu->kvm)); 7546 } 7547 7548 /* Called within kvm->srcu read side. */ 7549 static void post_kvm_run_save(struct kvm_vcpu *vcpu) 7550 { 7551 struct kvm_run *kvm_run = vcpu->run; 7552 7553 kvm_run->if_flag = kvm_x86_call(get_if_flag)(vcpu); 7554 kvm_run->cr8 = kvm_get_cr8(vcpu); 7555 kvm_run->apic_base = vcpu->arch.apic_base; 7556 7557 kvm_run->ready_for_interrupt_injection = 7558 pic_in_kernel(vcpu->kvm) || 7559 kvm_vcpu_ready_for_interrupt_injection(vcpu); 7560 7561 if (is_smm(vcpu)) 7562 kvm_run->flags |= KVM_RUN_X86_SMM; 7563 if (is_guest_mode(vcpu)) 7564 kvm_run->flags |= KVM_RUN_X86_GUEST_MODE; 7565 } 7566 7567 int kvm_check_nested_events(struct kvm_vcpu *vcpu) 7568 { 7569 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 7570 kvm_nested_call(triple_fault)(vcpu); 7571 return 1; 7572 } 7573 7574 return kvm_nested_call(check_events)(vcpu); 7575 } 7576 7577 static void kvm_inject_exception(struct kvm_vcpu *vcpu) 7578 { 7579 /* 7580 * Suppress the error code if the vCPU is in Real Mode, as Real Mode 7581 * exceptions don't report error codes. The presence of an error code 7582 * is carried with the exception and only stripped when the exception 7583 * is injected as intercepted #PF VM-Exits for AMD's Paged Real Mode do 7584 * report an error code despite the CPU being in Real Mode. 7585 */ 7586 vcpu->arch.exception.has_error_code &= is_protmode(vcpu); 7587 7588 trace_kvm_inj_exception(vcpu->arch.exception.vector, 7589 vcpu->arch.exception.has_error_code, 7590 vcpu->arch.exception.error_code, 7591 vcpu->arch.exception.injected); 7592 7593 kvm_x86_call(inject_exception)(vcpu); 7594 } 7595 7596 /* 7597 * Check for any event (interrupt or exception) that is ready to be injected, 7598 * and if there is at least one event, inject the event with the highest 7599 * priority. This handles both "pending" events, i.e. events that have never 7600 * been injected into the guest, and "injected" events, i.e. events that were 7601 * injected as part of a previous VM-Enter, but weren't successfully delivered 7602 * and need to be re-injected. 7603 * 7604 * Note, this is not guaranteed to be invoked on a guest instruction boundary, 7605 * i.e. doesn't guarantee that there's an event window in the guest. KVM must 7606 * be able to inject exceptions in the "middle" of an instruction, and so must 7607 * also be able to re-inject NMIs and IRQs in the middle of an instruction. 7608 * I.e. for exceptions and re-injected events, NOT invoking this on instruction 7609 * boundaries is necessary and correct. 7610 * 7611 * For simplicity, KVM uses a single path to inject all events (except events 7612 * that are injected directly from L1 to L2) and doesn't explicitly track 7613 * instruction boundaries for asynchronous events. However, because VM-Exits 7614 * that can occur during instruction execution typically result in KVM skipping 7615 * the instruction or injecting an exception, e.g. instruction and exception 7616 * intercepts, and because pending exceptions have higher priority than pending 7617 * interrupts, KVM still honors instruction boundaries in most scenarios. 7618 * 7619 * But, if a VM-Exit occurs during instruction execution, and KVM does NOT skip 7620 * the instruction or inject an exception, then KVM can incorrecty inject a new 7621 * asynchronous event if the event became pending after the CPU fetched the 7622 * instruction (in the guest). E.g. if a page fault (#PF, #NPF, EPT violation) 7623 * occurs and is resolved by KVM, a coincident NMI, SMI, IRQ, etc... can be 7624 * injected on the restarted instruction instead of being deferred until the 7625 * instruction completes. 7626 * 7627 * In practice, this virtualization hole is unlikely to be observed by the 7628 * guest, and even less likely to cause functional problems. To detect the 7629 * hole, the guest would have to trigger an event on a side effect of an early 7630 * phase of instruction execution, e.g. on the instruction fetch from memory. 7631 * And for it to be a functional problem, the guest would need to depend on the 7632 * ordering between that side effect, the instruction completing, _and_ the 7633 * delivery of the asynchronous event. 7634 */ 7635 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu, 7636 bool *req_immediate_exit) 7637 { 7638 bool can_inject; 7639 int r; 7640 7641 /* 7642 * Process nested events first, as nested VM-Exit supersedes event 7643 * re-injection. If there's an event queued for re-injection, it will 7644 * be saved into the appropriate vmc{b,s}12 fields on nested VM-Exit. 7645 */ 7646 if (is_guest_mode(vcpu)) 7647 r = kvm_check_nested_events(vcpu); 7648 else 7649 r = 0; 7650 7651 /* 7652 * Re-inject exceptions and events *especially* if immediate entry+exit 7653 * to/from L2 is needed, as any event that has already been injected 7654 * into L2 needs to complete its lifecycle before injecting a new event. 7655 * 7656 * Don't re-inject an NMI or interrupt if there is a pending exception. 7657 * This collision arises if an exception occurred while vectoring the 7658 * injected event, KVM intercepted said exception, and KVM ultimately 7659 * determined the fault belongs to the guest and queues the exception 7660 * for injection back into the guest. 7661 * 7662 * "Injected" interrupts can also collide with pending exceptions if 7663 * userspace ignores the "ready for injection" flag and blindly queues 7664 * an interrupt. In that case, prioritizing the exception is correct, 7665 * as the exception "occurred" before the exit to userspace. Trap-like 7666 * exceptions, e.g. most #DBs, have higher priority than interrupts. 7667 * And while fault-like exceptions, e.g. #GP and #PF, are the lowest 7668 * priority, they're only generated (pended) during instruction 7669 * execution, and interrupts are recognized at instruction boundaries. 7670 * Thus a pending fault-like exception means the fault occurred on the 7671 * *previous* instruction and must be serviced prior to recognizing any 7672 * new events in order to fully complete the previous instruction. 7673 */ 7674 if (vcpu->arch.exception.injected) 7675 kvm_inject_exception(vcpu); 7676 else if (kvm_is_exception_pending(vcpu)) 7677 ; /* see above */ 7678 else if (vcpu->arch.nmi_injected) 7679 kvm_x86_call(inject_nmi)(vcpu); 7680 else if (vcpu->arch.interrupt.injected) 7681 kvm_x86_call(inject_irq)(vcpu, true); 7682 7683 /* 7684 * Exceptions that morph to VM-Exits are handled above, and pending 7685 * exceptions on top of injected exceptions that do not VM-Exit should 7686 * either morph to #DF or, sadly, override the injected exception. 7687 */ 7688 WARN_ON_ONCE(vcpu->arch.exception.injected && 7689 vcpu->arch.exception.pending); 7690 7691 /* 7692 * Bail if immediate entry+exit to/from the guest is needed to complete 7693 * nested VM-Enter or event re-injection so that a different pending 7694 * event can be serviced (or if KVM needs to exit to userspace). 7695 * 7696 * Otherwise, continue processing events even if VM-Exit occurred. The 7697 * VM-Exit will have cleared exceptions that were meant for L2, but 7698 * there may now be events that can be injected into L1. 7699 */ 7700 if (r < 0) 7701 goto out; 7702 7703 /* 7704 * A pending exception VM-Exit should either result in nested VM-Exit 7705 * or force an immediate re-entry and exit to/from L2, and exception 7706 * VM-Exits cannot be injected (flag should _never_ be set). 7707 */ 7708 WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected || 7709 vcpu->arch.exception_vmexit.pending); 7710 7711 /* 7712 * New events, other than exceptions, cannot be injected if KVM needs 7713 * to re-inject a previous event. See above comments on re-injecting 7714 * for why pending exceptions get priority. 7715 */ 7716 can_inject = !kvm_event_needs_reinjection(vcpu); 7717 7718 if (vcpu->arch.exception.pending) { 7719 /* 7720 * Fault-class exceptions, except #DBs, set RF=1 in the RFLAGS 7721 * value pushed on the stack. Trap-like exception and all #DBs 7722 * leave RF as-is (KVM follows Intel's behavior in this regard; 7723 * AMD states that code breakpoint #DBs excplitly clear RF=0). 7724 * 7725 * Note, most versions of Intel's SDM and AMD's APM incorrectly 7726 * describe the behavior of General Detect #DBs, which are 7727 * fault-like. They do _not_ set RF, a la code breakpoints. 7728 */ 7729 if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT) 7730 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) | 7731 X86_EFLAGS_RF); 7732 7733 if (vcpu->arch.exception.vector == DB_VECTOR && 7734 vcpu->arch.dr7 & DR7_GD) { 7735 vcpu->arch.dr7 &= ~DR7_GD; 7736 kvm_update_dr7(vcpu); 7737 } 7738 7739 kvm_inject_exception(vcpu); 7740 7741 vcpu->arch.exception.pending = false; 7742 vcpu->arch.exception.injected = true; 7743 7744 can_inject = false; 7745 } 7746 7747 /* Don't inject interrupts if the user asked to avoid doing so */ 7748 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) 7749 return 0; 7750 7751 /* 7752 * Finally, inject interrupt events. If an event cannot be injected 7753 * due to architectural conditions (e.g. IF=0) a window-open exit 7754 * will re-request KVM_REQ_EVENT. Sometimes however an event is pending 7755 * and can architecturally be injected, but we cannot do it right now: 7756 * an interrupt could have arrived just now and we have to inject it 7757 * as a vmexit, or there could already an event in the queue, which is 7758 * indicated by can_inject. In that case we request an immediate exit 7759 * in order to make progress and get back here for another iteration. 7760 * The kvm_x86_ops hooks communicate this by returning -EBUSY. 7761 */ 7762 #ifdef CONFIG_KVM_SMM 7763 if (vcpu->arch.smi_pending) { 7764 r = can_inject ? kvm_x86_call(smi_allowed)(vcpu, true) : 7765 -EBUSY; 7766 if (r < 0) 7767 goto out; 7768 if (r) { 7769 vcpu->arch.smi_pending = false; 7770 ++vcpu->arch.smi_count; 7771 enter_smm(vcpu); 7772 can_inject = false; 7773 } else 7774 kvm_x86_call(enable_smi_window)(vcpu); 7775 } 7776 #endif 7777 7778 if (vcpu->arch.nmi_pending) { 7779 r = can_inject ? kvm_x86_call(nmi_allowed)(vcpu, true) : 7780 -EBUSY; 7781 if (r < 0) 7782 goto out; 7783 if (r) { 7784 --vcpu->arch.nmi_pending; 7785 vcpu->arch.nmi_injected = true; 7786 kvm_x86_call(inject_nmi)(vcpu); 7787 can_inject = false; 7788 WARN_ON(kvm_x86_call(nmi_allowed)(vcpu, true) < 0); 7789 } 7790 if (vcpu->arch.nmi_pending) 7791 kvm_x86_call(enable_nmi_window)(vcpu); 7792 } 7793 7794 if (kvm_cpu_has_injectable_intr(vcpu)) { 7795 r = can_inject ? kvm_x86_call(interrupt_allowed)(vcpu, true) : 7796 -EBUSY; 7797 if (r < 0) 7798 goto out; 7799 if (r) { 7800 int irq = kvm_cpu_get_interrupt(vcpu); 7801 7802 if (likely(irq != -1)) { 7803 kvm_queue_interrupt(vcpu, irq, false); 7804 kvm_x86_call(inject_irq)(vcpu, false); 7805 WARN_ON(kvm_x86_call(interrupt_allowed)(vcpu, true) < 0); 7806 } else { 7807 kvm_warn_on_lost_irq(vcpu); 7808 } 7809 } 7810 if (kvm_cpu_has_injectable_intr(vcpu)) 7811 kvm_x86_call(enable_irq_window)(vcpu); 7812 } 7813 7814 if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, true)) 7815 *req_immediate_exit = true; 7816 7817 /* 7818 * KVM must never queue a new exception while injecting an event; KVM 7819 * is done emulating and should only propagate the to-be-injected event 7820 * to the VMCS/VMCB. Queueing a new exception can put the vCPU into an 7821 * infinite loop as KVM will bail from VM-Enter to inject the pending 7822 * exception and start the cycle all over. 7823 * 7824 * Exempt triple faults as they have special handling and won't put the 7825 * vCPU into an infinite loop. Triple fault can be queued when running 7826 * VMX without unrestricted guest, as that requires KVM to emulate Real 7827 * Mode events (see kvm_inject_realmode_interrupt()). 7828 */ 7829 WARN_ON_ONCE(vcpu->arch.exception.pending || 7830 vcpu->arch.exception_vmexit.pending); 7831 return 0; 7832 7833 out: 7834 if (r == -EBUSY) { 7835 *req_immediate_exit = true; 7836 r = 0; 7837 } 7838 return r; 7839 } 7840 7841 static void process_nmi(struct kvm_vcpu *vcpu) 7842 { 7843 unsigned int limit; 7844 7845 /* 7846 * x86 is limited to one NMI pending, but because KVM can't react to 7847 * incoming NMIs as quickly as bare metal, e.g. if the vCPU is 7848 * scheduled out, KVM needs to play nice with two queued NMIs showing 7849 * up at the same time. To handle this scenario, allow two NMIs to be 7850 * (temporarily) pending so long as NMIs are not blocked and KVM is not 7851 * waiting for a previous NMI injection to complete (which effectively 7852 * blocks NMIs). KVM will immediately inject one of the two NMIs, and 7853 * will request an NMI window to handle the second NMI. 7854 */ 7855 if (kvm_x86_call(get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected) 7856 limit = 1; 7857 else 7858 limit = 2; 7859 7860 /* 7861 * Adjust the limit to account for pending virtual NMIs, which aren't 7862 * tracked in vcpu->arch.nmi_pending. 7863 */ 7864 if (kvm_x86_call(is_vnmi_pending)(vcpu)) 7865 limit--; 7866 7867 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0); 7868 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit); 7869 7870 if (vcpu->arch.nmi_pending && 7871 (kvm_x86_call(set_vnmi_pending)(vcpu))) 7872 vcpu->arch.nmi_pending--; 7873 7874 if (vcpu->arch.nmi_pending) 7875 kvm_make_request(KVM_REQ_EVENT, vcpu); 7876 } 7877 7878 /* Return total number of NMIs pending injection to the VM */ 7879 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu) 7880 { 7881 return vcpu->arch.nmi_pending + 7882 kvm_x86_call(is_vnmi_pending)(vcpu); 7883 } 7884 7885 void kvm_make_scan_ioapic_request_mask(struct kvm *kvm, 7886 unsigned long *vcpu_bitmap) 7887 { 7888 kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, vcpu_bitmap); 7889 } 7890 7891 void kvm_make_scan_ioapic_request(struct kvm *kvm) 7892 { 7893 kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC); 7894 } 7895 7896 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) 7897 { 7898 struct kvm_lapic *apic = vcpu->arch.apic; 7899 bool activate; 7900 7901 if (!lapic_in_kernel(vcpu)) 7902 return; 7903 7904 down_read(&vcpu->kvm->arch.apicv_update_lock); 7905 preempt_disable(); 7906 7907 /* Do not activate APICV when APIC is disabled */ 7908 activate = kvm_vcpu_apicv_activated(vcpu) && 7909 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED); 7910 7911 if (apic->apicv_active == activate) 7912 goto out; 7913 7914 apic->apicv_active = activate; 7915 kvm_apic_update_apicv(vcpu); 7916 kvm_x86_call(refresh_apicv_exec_ctrl)(vcpu); 7917 7918 /* 7919 * When APICv gets disabled, we may still have injected interrupts 7920 * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was 7921 * still active when the interrupt got accepted. Make sure 7922 * kvm_check_and_inject_events() is called to check for that. 7923 */ 7924 if (!apic->apicv_active) 7925 kvm_make_request(KVM_REQ_EVENT, vcpu); 7926 7927 out: 7928 preempt_enable(); 7929 up_read(&vcpu->kvm->arch.apicv_update_lock); 7930 } 7931 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_vcpu_update_apicv); 7932 7933 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) 7934 { 7935 if (!lapic_in_kernel(vcpu)) 7936 return; 7937 7938 /* 7939 * Due to sharing page tables across vCPUs, the xAPIC memslot must be 7940 * deleted if any vCPU has xAPIC virtualization and x2APIC enabled, but 7941 * and hardware doesn't support x2APIC virtualization. E.g. some AMD 7942 * CPUs support AVIC but not x2APIC. KVM still allows enabling AVIC in 7943 * this case so that KVM can use the AVIC doorbell to inject interrupts 7944 * to running vCPUs, but KVM must not create SPTEs for the APIC base as 7945 * the vCPU would incorrectly be able to access the vAPIC page via MMIO 7946 * despite being in x2APIC mode. For simplicity, inhibiting the APIC 7947 * access page is sticky. 7948 */ 7949 if (apic_x2apic_mode(vcpu->arch.apic) && 7950 kvm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization) 7951 kvm_inhibit_apic_access_page(vcpu); 7952 7953 __kvm_vcpu_update_apicv(vcpu); 7954 } 7955 7956 void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, 7957 enum kvm_apicv_inhibit reason, bool set) 7958 { 7959 unsigned long old, new; 7960 7961 lockdep_assert_held_write(&kvm->arch.apicv_update_lock); 7962 7963 if (!(kvm_x86_ops.required_apicv_inhibits & BIT(reason))) 7964 return; 7965 7966 old = new = kvm->arch.apicv_inhibit_reasons; 7967 7968 if (reason != APICV_INHIBIT_REASON_IRQWIN) 7969 set_or_clear_apicv_inhibit(&new, reason, set); 7970 7971 set_or_clear_apicv_inhibit(&new, APICV_INHIBIT_REASON_IRQWIN, 7972 atomic_read(&kvm->arch.apicv_nr_irq_window_req)); 7973 7974 if (!!old != !!new) { 7975 /* 7976 * Kick all vCPUs before setting apicv_inhibit_reasons to avoid 7977 * false positives in the sanity check WARN in vcpu_enter_guest(). 7978 * This task will wait for all vCPUs to ack the kick IRQ before 7979 * updating apicv_inhibit_reasons, and all other vCPUs will 7980 * block on acquiring apicv_update_lock so that vCPUs can't 7981 * redo vcpu_enter_guest() without seeing the new inhibit state. 7982 * 7983 * Note, holding apicv_update_lock and taking it in the read 7984 * side (handling the request) also prevents other vCPUs from 7985 * servicing the request with a stale apicv_inhibit_reasons. 7986 */ 7987 kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE); 7988 kvm->arch.apicv_inhibit_reasons = new; 7989 if (new) { 7990 unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE); 7991 7992 kvm_zap_gfn_range(kvm, gfn, gfn+1); 7993 } 7994 } else { 7995 kvm->arch.apicv_inhibit_reasons = new; 7996 } 7997 } 7998 7999 void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, 8000 enum kvm_apicv_inhibit reason, bool set) 8001 { 8002 if (!enable_apicv) 8003 return; 8004 8005 down_write(&kvm->arch.apicv_update_lock); 8006 __kvm_set_or_clear_apicv_inhibit(kvm, reason, set); 8007 up_write(&kvm->arch.apicv_update_lock); 8008 } 8009 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_or_clear_apicv_inhibit); 8010 8011 void kvm_inc_or_dec_irq_window_inhibit(struct kvm *kvm, bool inc) 8012 { 8013 int add = inc ? 1 : -1; 8014 8015 if (!enable_apicv) 8016 return; 8017 8018 /* 8019 * IRQ windows are requested either because of ExtINT injections, or 8020 * because APICv is already disabled/inhibited for another reason. 8021 * While ExtINT injections are rare and should not happen while the 8022 * vCPU is running its actual workload, it's worth avoiding thrashing 8023 * if the IRQ window is being requested because APICv is already 8024 * inhibited. So, toggle the actual inhibit (which requires taking 8025 * the lock for write) if and only if there's no other inhibit. 8026 * kvm_set_or_clear_apicv_inhibit() always evaluates the IRQ window 8027 * count; thus the IRQ window inhibit call _will_ be lazily updated on 8028 * the next call, if it ever happens. 8029 */ 8030 if (READ_ONCE(kvm->arch.apicv_inhibit_reasons) & ~BIT(APICV_INHIBIT_REASON_IRQWIN)) { 8031 guard(rwsem_read)(&kvm->arch.apicv_update_lock); 8032 if (READ_ONCE(kvm->arch.apicv_inhibit_reasons) & ~BIT(APICV_INHIBIT_REASON_IRQWIN)) { 8033 atomic_add(add, &kvm->arch.apicv_nr_irq_window_req); 8034 return; 8035 } 8036 } 8037 8038 /* 8039 * Strictly speaking, the lock is only needed if going 0->1 or 1->0, 8040 * a la atomic_dec_and_mutex_lock. However, ExtINTs are rare and 8041 * only target a single CPU, so that is the common case; do not 8042 * bother eliding the down_write()/up_write() pair. 8043 */ 8044 guard(rwsem_write)(&kvm->arch.apicv_update_lock); 8045 if (atomic_add_return(add, &kvm->arch.apicv_nr_irq_window_req) == inc) 8046 __kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_IRQWIN, inc); 8047 } 8048 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_inc_or_dec_irq_window_inhibit); 8049 8050 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) 8051 { 8052 if (!kvm_apic_present(vcpu)) 8053 return; 8054 8055 bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256); 8056 vcpu->arch.highest_stale_pending_ioapic_eoi = -1; 8057 8058 kvm_x86_call(sync_pir_to_irr)(vcpu); 8059 8060 if (irqchip_split(vcpu->kvm)) 8061 kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors); 8062 #ifdef CONFIG_KVM_IOAPIC 8063 else if (ioapic_in_kernel(vcpu->kvm)) 8064 kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors); 8065 #endif 8066 8067 if (is_guest_mode(vcpu)) 8068 vcpu->arch.load_eoi_exitmap_pending = true; 8069 else 8070 kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu); 8071 } 8072 8073 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) 8074 { 8075 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 8076 return; 8077 8078 #ifdef CONFIG_KVM_HYPERV 8079 if (to_hv_vcpu(vcpu)) { 8080 u64 eoi_exit_bitmap[4]; 8081 8082 bitmap_or((ulong *)eoi_exit_bitmap, 8083 vcpu->arch.ioapic_handled_vectors, 8084 to_hv_synic(vcpu)->vec_bitmap, 256); 8085 kvm_x86_call(load_eoi_exitmap)(vcpu, eoi_exit_bitmap); 8086 return; 8087 } 8088 #endif 8089 kvm_x86_call(load_eoi_exitmap)( 8090 vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors); 8091 } 8092 8093 void kvm_arch_guest_memory_reclaimed(struct kvm *kvm) 8094 { 8095 kvm_x86_call(guest_memory_reclaimed)(kvm); 8096 } 8097 8098 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) 8099 { 8100 if (!lapic_in_kernel(vcpu)) 8101 return; 8102 8103 kvm_x86_call(set_apic_access_page_addr)(vcpu); 8104 } 8105 8106 static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 8107 { 8108 /* 8109 * Note, nr_memslots_dirty_logging can be changed concurrent with this 8110 * code, but in that case another update request will be made and so 8111 * the guest will never run with a stale PML value. 8112 */ 8113 bool enable = atomic_read(&vcpu->kvm->nr_memslots_dirty_logging); 8114 8115 if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) 8116 return; 8117 8118 kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable); 8119 } 8120 8121 /* 8122 * Called within kvm->srcu read side. 8123 * Returns 1 to let vcpu_run() continue the guest execution loop without 8124 * exiting to the userspace. Otherwise, the value will be returned to the 8125 * userspace. 8126 */ 8127 static int vcpu_enter_guest(struct kvm_vcpu *vcpu) 8128 { 8129 int r; 8130 bool req_int_win = 8131 dm_request_for_irq_injection(vcpu) && 8132 kvm_cpu_accept_dm_intr(vcpu); 8133 fastpath_t exit_fastpath; 8134 u64 run_flags, debug_ctl; 8135 8136 bool req_immediate_exit = false; 8137 8138 if (kvm_request_pending(vcpu)) { 8139 if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) { 8140 r = -EIO; 8141 goto out; 8142 } 8143 8144 if (kvm_dirty_ring_check_request(vcpu)) { 8145 r = 0; 8146 goto out; 8147 } 8148 8149 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { 8150 if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) { 8151 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 8152 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 8153 vcpu->run->internal.ndata = 0; 8154 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 8155 r = 0; 8156 goto out; 8157 } 8158 } 8159 if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)) 8160 kvm_mmu_free_obsolete_roots(vcpu); 8161 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu)) 8162 __kvm_migrate_timers(vcpu); 8163 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) 8164 kvm_update_masterclock(vcpu->kvm); 8165 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu)) 8166 kvm_gen_kvmclock_update(vcpu); 8167 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) { 8168 r = kvm_guest_time_update(vcpu); 8169 if (unlikely(r)) 8170 goto out; 8171 } 8172 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu)) 8173 kvm_mmu_sync_roots(vcpu); 8174 if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu)) 8175 kvm_mmu_load_pgd(vcpu); 8176 8177 /* 8178 * Note, the order matters here, as flushing "all" TLB entries 8179 * also flushes the "current" TLB entries, i.e. servicing the 8180 * flush "all" will clear any request to flush "current". 8181 */ 8182 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 8183 kvm_vcpu_flush_tlb_all(vcpu); 8184 8185 kvm_service_local_tlb_flush_requests(vcpu); 8186 8187 /* 8188 * Fall back to a "full" guest flush if Hyper-V's precise 8189 * flushing fails. Note, Hyper-V's flushing is per-vCPU, but 8190 * the flushes are considered "remote" and not "local" because 8191 * the requests can be initiated from other vCPUs. 8192 */ 8193 #ifdef CONFIG_KVM_HYPERV 8194 if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) && 8195 kvm_hv_vcpu_flush_tlb(vcpu)) 8196 kvm_vcpu_flush_tlb_guest(vcpu); 8197 #endif 8198 8199 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { 8200 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; 8201 r = 0; 8202 goto out; 8203 } 8204 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 8205 if (is_guest_mode(vcpu)) 8206 kvm_nested_call(triple_fault)(vcpu); 8207 8208 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 8209 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 8210 vcpu->mmio_needed = 0; 8211 r = 0; 8212 goto out; 8213 } 8214 } 8215 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) { 8216 /* Page is swapped out. Do synthetic halt */ 8217 vcpu->arch.apf.halted = true; 8218 r = 1; 8219 goto out; 8220 } 8221 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 8222 record_steal_time(vcpu); 8223 if (kvm_check_request(KVM_REQ_PMU, vcpu)) 8224 kvm_pmu_handle_event(vcpu); 8225 if (kvm_check_request(KVM_REQ_PMI, vcpu)) 8226 kvm_pmu_deliver_pmi(vcpu); 8227 #ifdef CONFIG_KVM_SMM 8228 if (kvm_check_request(KVM_REQ_SMI, vcpu)) 8229 process_smi(vcpu); 8230 #endif 8231 if (kvm_check_request(KVM_REQ_NMI, vcpu)) 8232 process_nmi(vcpu); 8233 if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { 8234 if (test_bit(vcpu->arch.pending_ioapic_eoi, 8235 vcpu->arch.ioapic_handled_vectors)) { 8236 vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; 8237 vcpu->run->eoi.vector = 8238 vcpu->arch.pending_ioapic_eoi; 8239 r = 0; 8240 goto out; 8241 } 8242 } 8243 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu)) 8244 vcpu_scan_ioapic(vcpu); 8245 if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu)) 8246 vcpu_load_eoi_exitmap(vcpu); 8247 if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu)) 8248 kvm_vcpu_reload_apic_access_page(vcpu); 8249 #ifdef CONFIG_KVM_HYPERV 8250 if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) { 8251 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 8252 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH; 8253 vcpu->run->system_event.ndata = 0; 8254 r = 0; 8255 goto out; 8256 } 8257 if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) { 8258 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 8259 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET; 8260 vcpu->run->system_event.ndata = 0; 8261 r = 0; 8262 goto out; 8263 } 8264 if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) { 8265 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 8266 8267 vcpu->run->exit_reason = KVM_EXIT_HYPERV; 8268 vcpu->run->hyperv = hv_vcpu->exit; 8269 r = 0; 8270 goto out; 8271 } 8272 8273 /* 8274 * KVM_REQ_HV_STIMER has to be processed after 8275 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers 8276 * depend on the guest clock being up-to-date 8277 */ 8278 if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu)) 8279 kvm_hv_process_stimers(vcpu); 8280 #endif 8281 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) 8282 kvm_vcpu_update_apicv(vcpu); 8283 if (kvm_check_request(KVM_REQ_APF_READY, vcpu)) 8284 kvm_check_async_pf_completion(vcpu); 8285 8286 if (kvm_check_request(KVM_REQ_RECALC_INTERCEPTS, vcpu)) 8287 kvm_x86_call(recalc_intercepts)(vcpu); 8288 8289 if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) 8290 kvm_update_cpu_dirty_logging(vcpu); 8291 8292 if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) { 8293 kvm_vcpu_reset(vcpu, true); 8294 if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE) { 8295 r = 1; 8296 goto out; 8297 } 8298 } 8299 if (kvm_check_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu)) 8300 kvm_x86_call(reload_vmsa)(vcpu); 8301 } 8302 8303 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || 8304 kvm_xen_has_interrupt(vcpu)) { 8305 ++vcpu->stat.req_event; 8306 r = kvm_apic_accept_events(vcpu); 8307 if (r < 0) { 8308 r = 0; 8309 goto out; 8310 } 8311 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 8312 r = 1; 8313 goto out; 8314 } 8315 8316 r = kvm_check_and_inject_events(vcpu, &req_immediate_exit); 8317 if (r < 0) { 8318 r = 0; 8319 goto out; 8320 } 8321 if (req_int_win) 8322 kvm_x86_call(enable_irq_window)(vcpu); 8323 8324 if (kvm_lapic_enabled(vcpu)) { 8325 kvm_lapic_update_cr8_intercept(vcpu); 8326 kvm_lapic_sync_to_vapic(vcpu); 8327 } 8328 } 8329 8330 r = kvm_mmu_reload(vcpu); 8331 if (unlikely(r)) { 8332 goto cancel_injection; 8333 } 8334 8335 preempt_disable(); 8336 8337 kvm_x86_call(prepare_switch_to_guest)(vcpu); 8338 8339 /* 8340 * Disable IRQs before setting IN_GUEST_MODE. Posted interrupt 8341 * IPI are then delayed after guest entry, which ensures that they 8342 * result in virtual interrupt delivery. 8343 */ 8344 local_irq_disable(); 8345 8346 /* Store vcpu->apicv_active before vcpu->mode. */ 8347 smp_store_release(&vcpu->mode, IN_GUEST_MODE); 8348 8349 kvm_vcpu_srcu_read_unlock(vcpu); 8350 8351 /* 8352 * 1) We should set ->mode before checking ->requests. Please see 8353 * the comment in kvm_vcpu_exiting_guest_mode(). 8354 * 8355 * 2) For APICv, we should set ->mode before checking PID.ON. This 8356 * pairs with the memory barrier implicit in pi_test_and_set_on 8357 * (see vmx_deliver_posted_interrupt). 8358 * 8359 * 3) This also orders the write to mode from any reads to the page 8360 * tables done while the VCPU is running. Please see the comment 8361 * in kvm_flush_remote_tlbs. 8362 */ 8363 smp_mb__after_srcu_read_unlock(); 8364 8365 /* 8366 * Process pending posted interrupts to handle the case where the 8367 * notification IRQ arrived in the host, or was never sent (because the 8368 * target vCPU wasn't running). Do this regardless of the vCPU's APICv 8369 * status, KVM doesn't update assigned devices when APICv is inhibited, 8370 * i.e. they can post interrupts even if APICv is temporarily disabled. 8371 */ 8372 if (kvm_lapic_enabled(vcpu)) 8373 kvm_x86_call(sync_pir_to_irr)(vcpu); 8374 8375 if (kvm_vcpu_exit_request(vcpu)) { 8376 vcpu->mode = OUTSIDE_GUEST_MODE; 8377 smp_wmb(); 8378 local_irq_enable(); 8379 preempt_enable(); 8380 kvm_vcpu_srcu_read_lock(vcpu); 8381 r = 1; 8382 goto cancel_injection; 8383 } 8384 8385 run_flags = 0; 8386 if (req_immediate_exit) { 8387 run_flags |= KVM_RUN_FORCE_IMMEDIATE_EXIT; 8388 kvm_make_request(KVM_REQ_EVENT, vcpu); 8389 } 8390 8391 fpregs_assert_state_consistent(); 8392 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 8393 switch_fpu_return(); 8394 8395 if (vcpu->arch.guest_fpu.xfd_err) 8396 wrmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); 8397 8398 kvm_load_xfeatures(vcpu, true); 8399 8400 if (unlikely(vcpu->arch.switch_db_regs && 8401 !(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH))) { 8402 set_debugreg(DR7_FIXED_1, 7); 8403 set_debugreg(vcpu->arch.eff_db[0], 0); 8404 set_debugreg(vcpu->arch.eff_db[1], 1); 8405 set_debugreg(vcpu->arch.eff_db[2], 2); 8406 set_debugreg(vcpu->arch.eff_db[3], 3); 8407 /* When KVM_DEBUGREG_WONT_EXIT, dr6 is accessible in guest. */ 8408 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) 8409 run_flags |= KVM_RUN_LOAD_GUEST_DR6; 8410 } else if (unlikely(hw_breakpoint_active())) { 8411 set_debugreg(DR7_FIXED_1, 7); 8412 } 8413 8414 /* 8415 * Refresh the host DEBUGCTL snapshot after disabling IRQs, as DEBUGCTL 8416 * can be modified in IRQ context, e.g. via SMP function calls. Inform 8417 * vendor code if any host-owned bits were changed, e.g. so that the 8418 * value loaded into hardware while running the guest can be updated. 8419 */ 8420 debug_ctl = get_debugctlmsr(); 8421 if ((debug_ctl ^ vcpu->arch.host_debugctl) & kvm_x86_ops.HOST_OWNED_DEBUGCTL && 8422 !vcpu->arch.guest_state_protected) 8423 run_flags |= KVM_RUN_LOAD_DEBUGCTL; 8424 vcpu->arch.host_debugctl = debug_ctl; 8425 8426 kvm_mediated_pmu_load(vcpu); 8427 8428 guest_timing_enter_irqoff(); 8429 8430 /* 8431 * Swap PKRU with hardware breakpoints disabled to minimize the number 8432 * of flows where non-KVM code can run with guest state loaded. 8433 */ 8434 kvm_load_guest_pkru(vcpu); 8435 8436 for (;;) { 8437 /* 8438 * Assert that vCPU vs. VM APICv state is consistent. An APICv 8439 * update must kick and wait for all vCPUs before toggling the 8440 * per-VM state, and responding vCPUs must wait for the update 8441 * to complete before servicing KVM_REQ_APICV_UPDATE. 8442 */ 8443 WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) && 8444 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED)); 8445 8446 exit_fastpath = kvm_x86_call(vcpu_run)(vcpu, run_flags); 8447 if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST)) 8448 break; 8449 8450 if (kvm_lapic_enabled(vcpu)) 8451 kvm_x86_call(sync_pir_to_irr)(vcpu); 8452 8453 if (unlikely(kvm_vcpu_exit_request(vcpu))) { 8454 exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED; 8455 break; 8456 } 8457 8458 run_flags = 0; 8459 8460 /* Note, VM-Exits that go down the "slow" path are accounted below. */ 8461 ++vcpu->stat.exits; 8462 } 8463 8464 kvm_load_host_pkru(vcpu); 8465 8466 kvm_mediated_pmu_put(vcpu); 8467 8468 /* 8469 * Do this here before restoring debug registers on the host. And 8470 * since we do this before handling the vmexit, a DR access vmexit 8471 * can (a) read the correct value of the debug registers, (b) set 8472 * KVM_DEBUGREG_WONT_EXIT again. 8473 */ 8474 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) { 8475 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP); 8476 WARN_ON(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH); 8477 kvm_x86_call(sync_dirty_debug_regs)(vcpu); 8478 kvm_update_dr0123(vcpu); 8479 vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); 8480 kvm_update_dr7(vcpu); 8481 } 8482 8483 /* 8484 * If the guest has used debug registers, at least dr7 8485 * will be disabled while returning to the host. 8486 * If we don't have active breakpoints in the host, we don't 8487 * care about the messed up debug address registers. But if 8488 * we have some of them active, restore the old state. 8489 */ 8490 if (hw_breakpoint_active()) 8491 hw_breakpoint_restore(); 8492 8493 vcpu->arch.last_vmentry_cpu = vcpu->cpu; 8494 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 8495 8496 vcpu->mode = OUTSIDE_GUEST_MODE; 8497 smp_wmb(); 8498 8499 kvm_load_xfeatures(vcpu, false); 8500 8501 /* 8502 * Sync xfd before calling handle_exit_irqoff() which may 8503 * rely on the fact that guest_fpu::xfd is up-to-date (e.g. 8504 * in #NM irqoff handler). 8505 */ 8506 if (vcpu->arch.xfd_no_write_intercept) 8507 fpu_sync_guest_vmexit_xfd_state(); 8508 8509 kvm_x86_call(handle_exit_irqoff)(vcpu); 8510 8511 if (vcpu->arch.guest_fpu.xfd_err) 8512 wrmsrq(MSR_IA32_XFD_ERR, 0); 8513 8514 /* 8515 * Mark this CPU as needing a branch predictor flush before running 8516 * userspace. Must be done before enabling preemption to ensure it gets 8517 * set for the CPU that actually ran the guest, and not the CPU that it 8518 * may migrate to. 8519 */ 8520 if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER)) 8521 this_cpu_write(x86_ibpb_exit_to_user, true); 8522 8523 /* 8524 * Consume any pending interrupts, including the possible source of 8525 * VM-Exit on SVM and any ticks that occur between VM-Exit and now. 8526 * An instruction is required after local_irq_enable() to fully unblock 8527 * interrupts on processors that implement an interrupt shadow, the 8528 * stat.exits increment will do nicely. 8529 */ 8530 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); 8531 local_irq_enable(); 8532 ++vcpu->stat.exits; 8533 local_irq_disable(); 8534 kvm_after_interrupt(vcpu); 8535 8536 /* 8537 * Wait until after servicing IRQs to account guest time so that any 8538 * ticks that occurred while running the guest are properly accounted 8539 * to the guest. Waiting until IRQs are enabled degrades the accuracy 8540 * of accounting via context tracking, but the loss of accuracy is 8541 * acceptable for all known use cases. 8542 */ 8543 guest_timing_exit_irqoff(); 8544 8545 local_irq_enable(); 8546 preempt_enable(); 8547 8548 kvm_vcpu_srcu_read_lock(vcpu); 8549 8550 /* 8551 * Call this to ensure WC buffers in guest are evicted after each VM 8552 * Exit, so that the evicted WC writes can be snooped across all cpus 8553 */ 8554 smp_mb__after_srcu_read_lock(); 8555 8556 /* 8557 * Profile KVM exit RIPs: 8558 */ 8559 if (unlikely(prof_on == KVM_PROFILING && 8560 !vcpu->arch.guest_state_protected)) { 8561 unsigned long rip = kvm_rip_read(vcpu); 8562 profile_hit(KVM_PROFILING, (void *)rip); 8563 } 8564 8565 if (unlikely(vcpu->arch.tsc_always_catchup)) 8566 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 8567 8568 if (vcpu->arch.apic_attention) 8569 kvm_lapic_sync_from_vapic(vcpu); 8570 8571 r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath); 8572 return r; 8573 8574 cancel_injection: 8575 if (req_immediate_exit) 8576 kvm_make_request(KVM_REQ_EVENT, vcpu); 8577 kvm_x86_call(cancel_injection)(vcpu); 8578 if (unlikely(vcpu->arch.apic_attention)) 8579 kvm_lapic_sync_from_vapic(vcpu); 8580 out: 8581 return r; 8582 } 8583 8584 static bool kvm_vcpu_running(struct kvm_vcpu *vcpu) 8585 { 8586 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && 8587 !vcpu->arch.apf.halted); 8588 } 8589 8590 bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) 8591 { 8592 if (!list_empty_careful(&vcpu->async_pf.done)) 8593 return true; 8594 8595 if (kvm_apic_has_pending_init_or_sipi(vcpu) && 8596 kvm_apic_init_sipi_allowed(vcpu)) 8597 return true; 8598 8599 if (kvm_is_exception_pending(vcpu)) 8600 return true; 8601 8602 if (kvm_test_request(KVM_REQ_NMI, vcpu) || 8603 (vcpu->arch.nmi_pending && 8604 kvm_x86_call(nmi_allowed)(vcpu, false))) 8605 return true; 8606 8607 #ifdef CONFIG_KVM_SMM 8608 if (kvm_test_request(KVM_REQ_SMI, vcpu) || 8609 (vcpu->arch.smi_pending && 8610 kvm_x86_call(smi_allowed)(vcpu, false))) 8611 return true; 8612 #endif 8613 8614 if (kvm_test_request(KVM_REQ_PMI, vcpu)) 8615 return true; 8616 8617 if (kvm_test_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) 8618 return true; 8619 8620 if (kvm_is_interrupt_allowed(vcpu) && kvm_cpu_has_interrupt(vcpu)) 8621 return true; 8622 8623 if (kvm_hv_has_stimer_pending(vcpu)) 8624 return true; 8625 8626 if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, false)) 8627 return true; 8628 8629 if (kvm_xen_has_pending_events(vcpu)) 8630 return true; 8631 8632 return false; 8633 } 8634 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_has_events); 8635 8636 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 8637 { 8638 return kvm_vcpu_running(vcpu) || vcpu->arch.pv.pv_unhalted || 8639 kvm_vcpu_has_events(vcpu); 8640 } 8641 8642 /* Called within kvm->srcu read side. */ 8643 static inline int vcpu_block(struct kvm_vcpu *vcpu) 8644 { 8645 bool hv_timer; 8646 8647 if (!kvm_arch_vcpu_runnable(vcpu)) { 8648 /* 8649 * Switch to the software timer before halt-polling/blocking as 8650 * the guest's timer may be a break event for the vCPU, and the 8651 * hypervisor timer runs only when the CPU is in guest mode. 8652 * Switch before halt-polling so that KVM recognizes an expired 8653 * timer before blocking. 8654 */ 8655 hv_timer = kvm_lapic_hv_timer_in_use(vcpu); 8656 if (hv_timer) 8657 kvm_lapic_switch_to_sw_timer(vcpu); 8658 8659 kvm_vcpu_srcu_read_unlock(vcpu); 8660 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 8661 kvm_vcpu_halt(vcpu); 8662 else 8663 kvm_vcpu_block(vcpu); 8664 kvm_vcpu_srcu_read_lock(vcpu); 8665 8666 if (hv_timer) 8667 kvm_lapic_switch_to_hv_timer(vcpu); 8668 8669 /* 8670 * If the vCPU is not runnable, a signal or another host event 8671 * of some kind is pending; service it without changing the 8672 * vCPU's activity state. 8673 */ 8674 if (!kvm_arch_vcpu_runnable(vcpu)) 8675 return 1; 8676 } 8677 8678 /* 8679 * Evaluate nested events before exiting the halted state. This allows 8680 * the halt state to be recorded properly in the VMCS12's activity 8681 * state field (AMD does not have a similar field and a VM-Exit always 8682 * causes a spurious wakeup from HLT). 8683 */ 8684 if (is_guest_mode(vcpu)) { 8685 int r = kvm_check_nested_events(vcpu); 8686 8687 if (r < 0 && r != -EBUSY) 8688 return 0; 8689 } 8690 8691 if (kvm_apic_accept_events(vcpu) < 0) 8692 return 0; 8693 switch(vcpu->arch.mp_state) { 8694 case KVM_MP_STATE_HALTED: 8695 case KVM_MP_STATE_AP_RESET_HOLD: 8696 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 8697 fallthrough; 8698 case KVM_MP_STATE_RUNNABLE: 8699 vcpu->arch.apf.halted = false; 8700 break; 8701 case KVM_MP_STATE_INIT_RECEIVED: 8702 break; 8703 default: 8704 WARN_ON_ONCE(1); 8705 break; 8706 } 8707 return 1; 8708 } 8709 8710 /* Called within kvm->srcu read side. */ 8711 static int vcpu_run(struct kvm_vcpu *vcpu) 8712 { 8713 int r; 8714 8715 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; 8716 8717 for (;;) { 8718 /* 8719 * If another guest vCPU requests a PV TLB flush in the middle 8720 * of instruction emulation, the rest of the emulation could 8721 * use a stale page translation. Assume that any code after 8722 * this point can start executing an instruction. 8723 */ 8724 vcpu->arch.at_instruction_boundary = false; 8725 if (kvm_vcpu_running(vcpu)) { 8726 r = vcpu_enter_guest(vcpu); 8727 } else { 8728 r = vcpu_block(vcpu); 8729 } 8730 8731 if (r <= 0) 8732 break; 8733 8734 kvm_clear_request(KVM_REQ_UNBLOCK, vcpu); 8735 if (kvm_xen_has_pending_events(vcpu)) 8736 kvm_xen_inject_pending_events(vcpu); 8737 8738 if (kvm_cpu_has_pending_timer(vcpu)) 8739 kvm_inject_pending_timer_irqs(vcpu); 8740 8741 if (dm_request_for_irq_injection(vcpu) && 8742 kvm_vcpu_ready_for_interrupt_injection(vcpu)) { 8743 r = 0; 8744 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; 8745 ++vcpu->stat.request_irq_exits; 8746 break; 8747 } 8748 8749 if (__xfer_to_guest_mode_work_pending()) { 8750 kvm_vcpu_srcu_read_unlock(vcpu); 8751 r = kvm_xfer_to_guest_mode_handle_work(vcpu); 8752 kvm_vcpu_srcu_read_lock(vcpu); 8753 if (r) 8754 return r; 8755 } 8756 } 8757 8758 return r; 8759 } 8760 8761 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason) 8762 { 8763 /* 8764 * The vCPU has halted, e.g. executed HLT. Update the run state if the 8765 * local APIC is in-kernel, the run loop will detect the non-runnable 8766 * state and halt the vCPU. Exit to userspace if the local APIC is 8767 * managed by userspace, in which case userspace is responsible for 8768 * handling wake events. 8769 */ 8770 ++vcpu->stat.halt_exits; 8771 if (lapic_in_kernel(vcpu)) { 8772 if (kvm_vcpu_has_events(vcpu) || vcpu->arch.pv.pv_unhalted) 8773 state = KVM_MP_STATE_RUNNABLE; 8774 kvm_set_mp_state(vcpu, state); 8775 return 1; 8776 } else { 8777 vcpu->run->exit_reason = reason; 8778 return 0; 8779 } 8780 } 8781 8782 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu) 8783 { 8784 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT); 8785 } 8786 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_halt_noskip); 8787 8788 int kvm_emulate_halt(struct kvm_vcpu *vcpu) 8789 { 8790 int ret = kvm_skip_emulated_instruction(vcpu); 8791 /* 8792 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered 8793 * KVM_EXIT_DEBUG here. 8794 */ 8795 return kvm_emulate_halt_noskip(vcpu) && ret; 8796 } 8797 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_halt); 8798 8799 fastpath_t handle_fastpath_hlt(struct kvm_vcpu *vcpu) 8800 { 8801 if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu)) 8802 return EXIT_FASTPATH_NONE; 8803 8804 if (!kvm_emulate_halt(vcpu)) 8805 return EXIT_FASTPATH_EXIT_USERSPACE; 8806 8807 if (kvm_vcpu_running(vcpu)) 8808 return EXIT_FASTPATH_REENTER_GUEST; 8809 8810 return EXIT_FASTPATH_EXIT_HANDLED; 8811 } 8812 EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_hlt); 8813 8814 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu) 8815 { 8816 int ret = kvm_skip_emulated_instruction(vcpu); 8817 8818 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD, 8819 KVM_EXIT_AP_RESET_HOLD) && ret; 8820 } 8821 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_ap_reset_hold); 8822 8823 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu) 8824 { 8825 return kvm_vcpu_apicv_active(vcpu) && 8826 kvm_x86_call(dy_apicv_has_pending_interrupt)(vcpu); 8827 } 8828 8829 bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu) 8830 { 8831 return vcpu->arch.preempted_in_kernel; 8832 } 8833 8834 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) 8835 { 8836 if (READ_ONCE(vcpu->arch.pv.pv_unhalted)) 8837 return true; 8838 8839 if (kvm_test_request(KVM_REQ_NMI, vcpu) || 8840 #ifdef CONFIG_KVM_SMM 8841 kvm_test_request(KVM_REQ_SMI, vcpu) || 8842 #endif 8843 kvm_test_request(KVM_REQ_EVENT, vcpu)) 8844 return true; 8845 8846 return kvm_arch_dy_has_pending_interrupt(vcpu); 8847 } 8848 8849 static inline int complete_emulated_io(struct kvm_vcpu *vcpu) 8850 { 8851 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE); 8852 } 8853 8854 static int complete_emulated_pio(struct kvm_vcpu *vcpu) 8855 { 8856 if (KVM_BUG_ON(!vcpu->arch.pio.count, vcpu->kvm)) 8857 return -EIO; 8858 8859 return complete_emulated_io(vcpu); 8860 } 8861 8862 /* 8863 * Implements the following, as a state machine: 8864 * 8865 * read: 8866 * for each fragment 8867 * for each mmio piece in the fragment 8868 * write gpa, len 8869 * exit 8870 * copy data 8871 * execute insn 8872 * 8873 * write: 8874 * for each fragment 8875 * for each mmio piece in the fragment 8876 * write gpa, len 8877 * copy data 8878 * exit 8879 */ 8880 static int complete_emulated_mmio(struct kvm_vcpu *vcpu) 8881 { 8882 struct kvm_run *run = vcpu->run; 8883 struct kvm_mmio_fragment *frag; 8884 unsigned len; 8885 8886 if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm)) 8887 return -EIO; 8888 8889 /* Complete previous fragment */ 8890 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment]; 8891 len = min(8u, frag->len); 8892 if (!vcpu->mmio_is_write) 8893 memcpy(frag->data, run->mmio.data, len); 8894 8895 if (frag->len <= 8) { 8896 /* Switch to the next fragment. */ 8897 frag++; 8898 vcpu->mmio_cur_fragment++; 8899 } else { 8900 if (WARN_ON_ONCE(frag->data == &frag->val)) 8901 return -EIO; 8902 8903 /* Go forward to the next mmio piece. */ 8904 frag->data += len; 8905 frag->gpa += len; 8906 frag->len -= len; 8907 } 8908 8909 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) { 8910 vcpu->mmio_needed = 0; 8911 8912 /* FIXME: return into emulator if single-stepping. */ 8913 if (vcpu->mmio_is_write) 8914 return 1; 8915 vcpu->mmio_read_completed = 1; 8916 return complete_emulated_io(vcpu); 8917 } 8918 8919 kvm_prepare_emulated_mmio_exit(vcpu, frag); 8920 vcpu->arch.complete_userspace_io = complete_emulated_mmio; 8921 return 0; 8922 } 8923 8924 static int kvm_x86_vcpu_pre_run(struct kvm_vcpu *vcpu) 8925 { 8926 /* 8927 * Userspace may have modified vCPU state, mark nested_run_pending as 8928 * "untrusted" to avoid triggering false-positive WARNs. 8929 */ 8930 if (vcpu->arch.nested_run_pending == KVM_NESTED_RUN_PENDING) 8931 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING_UNTRUSTED; 8932 8933 /* 8934 * SIPI_RECEIVED is obsolete; KVM leaves the vCPU in Wait-For-SIPI and 8935 * tracks the pending SIPI separately. SIPI_RECEIVED is still accepted 8936 * by KVM_SET_VCPU_EVENTS for backwards compatibility, but should be 8937 * converted to INIT_RECEIVED. 8938 */ 8939 if (WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) 8940 return -EINVAL; 8941 8942 /* 8943 * Disallow running the vCPU if userspace forced it into an impossible 8944 * MP_STATE, e.g. if the vCPU is in WFS but SIPI is blocked. 8945 */ 8946 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED && 8947 !kvm_apic_init_sipi_allowed(vcpu)) 8948 return -EINVAL; 8949 8950 if (kvm_x86_call(vcpu_needs_initialization)(vcpu)) 8951 return -EINVAL; 8952 8953 if (kvm_x86_call(unhandleable_emulation_required)(vcpu)) { 8954 kvm_prepare_emulation_failure_exit(vcpu); 8955 return 0; 8956 } 8957 8958 return 1; 8959 } 8960 8961 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 8962 { 8963 struct kvm_queued_exception *ex = &vcpu->arch.exception; 8964 struct kvm_run *kvm_run = vcpu->run; 8965 u64 sync_valid_fields; 8966 int r; 8967 8968 r = kvm_mmu_post_init_vm(vcpu->kvm); 8969 if (r) 8970 return r; 8971 8972 vcpu_load(vcpu); 8973 kvm_sigset_activate(vcpu); 8974 kvm_run->flags = 0; 8975 kvm_load_guest_fpu(vcpu); 8976 8977 kvm_vcpu_srcu_read_lock(vcpu); 8978 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { 8979 if (!vcpu->wants_to_run) { 8980 r = -EINTR; 8981 goto out; 8982 } 8983 8984 /* 8985 * Don't bother switching APIC timer emulation from the 8986 * hypervisor timer to the software timer, the only way for the 8987 * APIC timer to be active is if userspace stuffed vCPU state, 8988 * i.e. put the vCPU into a nonsensical state. Only an INIT 8989 * will transition the vCPU out of UNINITIALIZED (without more 8990 * state stuffing from userspace), which will reset the local 8991 * APIC and thus cancel the timer or drop the IRQ (if the timer 8992 * already expired). 8993 */ 8994 kvm_vcpu_srcu_read_unlock(vcpu); 8995 kvm_vcpu_block(vcpu); 8996 kvm_vcpu_srcu_read_lock(vcpu); 8997 8998 if (kvm_apic_accept_events(vcpu) < 0) { 8999 r = 0; 9000 goto out; 9001 } 9002 r = -EAGAIN; 9003 if (signal_pending(current)) { 9004 r = -EINTR; 9005 kvm_run->exit_reason = KVM_EXIT_INTR; 9006 ++vcpu->stat.signal_exits; 9007 } 9008 goto out; 9009 } 9010 9011 sync_valid_fields = kvm_sync_valid_fields(vcpu->kvm); 9012 if ((kvm_run->kvm_valid_regs & ~sync_valid_fields) || 9013 (kvm_run->kvm_dirty_regs & ~sync_valid_fields)) { 9014 r = -EINVAL; 9015 goto out; 9016 } 9017 9018 if (kvm_run->kvm_dirty_regs) { 9019 r = sync_regs(vcpu); 9020 if (r != 0) 9021 goto out; 9022 } 9023 9024 /* re-sync apic's tpr */ 9025 if (!lapic_in_kernel(vcpu)) { 9026 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) { 9027 r = -EINVAL; 9028 goto out; 9029 } 9030 } 9031 9032 /* 9033 * If userspace set a pending exception and L2 is active, convert it to 9034 * a pending VM-Exit if L1 wants to intercept the exception. 9035 */ 9036 if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) && 9037 kvm_nested_call(is_exception_vmexit)(vcpu, ex->vector, ex->error_code)) { 9038 kvm_queue_exception_vmexit(vcpu, ex->vector, 9039 ex->has_error_code, ex->error_code, 9040 ex->has_payload, ex->payload); 9041 ex->injected = false; 9042 ex->pending = false; 9043 } 9044 vcpu->arch.exception_from_userspace = false; 9045 9046 if (unlikely(vcpu->arch.complete_userspace_io)) { 9047 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io; 9048 vcpu->arch.complete_userspace_io = NULL; 9049 r = cui(vcpu); 9050 if (r <= 0) 9051 goto out; 9052 } else { 9053 WARN_ON_ONCE(vcpu->arch.pio.count); 9054 WARN_ON_ONCE(vcpu->mmio_needed); 9055 } 9056 9057 if (!vcpu->wants_to_run) { 9058 r = -EINTR; 9059 goto out; 9060 } 9061 9062 r = kvm_x86_vcpu_pre_run(vcpu); 9063 if (r <= 0) 9064 goto out; 9065 9066 r = vcpu_run(vcpu); 9067 9068 out: 9069 kvm_put_guest_fpu(vcpu); 9070 if (kvm_run->kvm_valid_regs && likely(!vcpu->arch.guest_state_protected)) 9071 store_regs(vcpu); 9072 post_kvm_run_save(vcpu); 9073 kvm_vcpu_srcu_read_unlock(vcpu); 9074 9075 kvm_sigset_deactivate(vcpu); 9076 vcpu_put(vcpu); 9077 return r; 9078 } 9079 9080 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 9081 struct kvm_mp_state *mp_state) 9082 { 9083 int r; 9084 9085 vcpu_load(vcpu); 9086 kvm_vcpu_srcu_read_lock(vcpu); 9087 9088 r = kvm_apic_accept_events(vcpu); 9089 if (r < 0) 9090 goto out; 9091 r = 0; 9092 9093 if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED || 9094 vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) && 9095 vcpu->arch.pv.pv_unhalted) 9096 mp_state->mp_state = KVM_MP_STATE_RUNNABLE; 9097 else 9098 mp_state->mp_state = vcpu->arch.mp_state; 9099 9100 out: 9101 kvm_vcpu_srcu_read_unlock(vcpu); 9102 vcpu_put(vcpu); 9103 return r; 9104 } 9105 9106 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 9107 struct kvm_mp_state *mp_state) 9108 { 9109 int ret = -EINVAL; 9110 9111 vcpu_load(vcpu); 9112 9113 switch (mp_state->mp_state) { 9114 case KVM_MP_STATE_UNINITIALIZED: 9115 case KVM_MP_STATE_HALTED: 9116 case KVM_MP_STATE_AP_RESET_HOLD: 9117 case KVM_MP_STATE_INIT_RECEIVED: 9118 case KVM_MP_STATE_SIPI_RECEIVED: 9119 if (!lapic_in_kernel(vcpu)) 9120 goto out; 9121 break; 9122 9123 case KVM_MP_STATE_RUNNABLE: 9124 break; 9125 9126 default: 9127 goto out; 9128 } 9129 9130 /* 9131 * SIPI_RECEIVED is obsolete and no longer used internally; KVM instead 9132 * leaves the vCPU in INIT_RECIEVED (Wait-For-SIPI) and pends the SIPI. 9133 * Translate SIPI_RECEIVED as appropriate for backwards compatibility. 9134 */ 9135 if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) { 9136 mp_state->mp_state = KVM_MP_STATE_INIT_RECEIVED; 9137 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events); 9138 } 9139 9140 kvm_set_mp_state(vcpu, mp_state->mp_state); 9141 9142 /* 9143 * Force the vCPU out of any hardware-tracked halted state, e.g. VMX's 9144 * GUEST_ACTIVITY_STATE=HLT, when userspace puts the vCPU into a state 9145 * other than HALTED. The hardware state is sticky across VM-Exit and 9146 * VM-Enter and is not touched by any other ioctl, so a vCPU that halted 9147 * with HLT-exiting disabled stays wedged even after userspace rewrites 9148 * its registers, e.g. when a VMM emulates a machine reset. 9149 */ 9150 if (kvm_hlt_in_guest(vcpu->kvm) && 9151 mp_state->mp_state != KVM_MP_STATE_HALTED) 9152 kvm_x86_call(clear_hlt)(vcpu); 9153 9154 kvm_make_request(KVM_REQ_EVENT, vcpu); 9155 9156 ret = 0; 9157 out: 9158 vcpu_put(vcpu); 9159 return ret; 9160 } 9161 9162 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, 9163 int reason, bool has_error_code, u32 error_code) 9164 { 9165 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 9166 int ret; 9167 9168 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_CET)) { 9169 u64 u_cet, s_cet; 9170 9171 /* 9172 * Check both User and Supervisor on task switches as inter- 9173 * privilege level task switches are impacted by CET at both 9174 * the current privilege level and the new privilege level, and 9175 * that information is not known at this time. The expectation 9176 * is that the guest won't require emulation of task switches 9177 * while using IBT or Shadow Stacks. 9178 */ 9179 if (__kvm_emulate_msr_read(vcpu, MSR_IA32_U_CET, &u_cet) || 9180 __kvm_emulate_msr_read(vcpu, MSR_IA32_S_CET, &s_cet)) 9181 goto unhandled_task_switch; 9182 9183 if ((u_cet | s_cet) & (CET_ENDBR_EN | CET_SHSTK_EN)) 9184 goto unhandled_task_switch; 9185 } 9186 9187 init_emulate_ctxt(vcpu); 9188 9189 ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason, 9190 has_error_code, error_code); 9191 9192 /* 9193 * Report an error userspace if MMIO is needed, as KVM doesn't support 9194 * MMIO during a task switch (or any other complex operation). 9195 */ 9196 if (ret || vcpu->mmio_needed) 9197 goto unhandled_task_switch; 9198 9199 kvm_rip_write(vcpu, ctxt->eip); 9200 kvm_set_rflags(vcpu, ctxt->eflags); 9201 return 1; 9202 9203 unhandled_task_switch: 9204 vcpu->mmio_needed = false; 9205 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 9206 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 9207 vcpu->run->internal.ndata = 0; 9208 return 0; 9209 } 9210 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_task_switch); 9211 9212 static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm) 9213 { 9214 bool set = false; 9215 struct kvm_vcpu *vcpu; 9216 unsigned long i; 9217 9218 if (!enable_apicv) 9219 return; 9220 9221 down_write(&kvm->arch.apicv_update_lock); 9222 9223 kvm_for_each_vcpu(i, vcpu, kvm) { 9224 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) { 9225 set = true; 9226 break; 9227 } 9228 } 9229 __kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_BLOCKIRQ, set); 9230 up_write(&kvm->arch.apicv_update_lock); 9231 } 9232 9233 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 9234 struct kvm_guest_debug *dbg) 9235 { 9236 unsigned long rflags; 9237 int i, r; 9238 9239 if (vcpu->arch.guest_state_protected) 9240 return -EINVAL; 9241 9242 vcpu_load(vcpu); 9243 9244 if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) { 9245 r = -EBUSY; 9246 if (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected) 9247 goto out; 9248 if (dbg->control & KVM_GUESTDBG_INJECT_DB) 9249 kvm_queue_exception(vcpu, DB_VECTOR); 9250 else 9251 kvm_queue_exception(vcpu, BP_VECTOR); 9252 } 9253 9254 /* 9255 * Read rflags as long as potentially injected trace flags are still 9256 * filtered out. 9257 */ 9258 rflags = kvm_get_rflags(vcpu); 9259 9260 vcpu->guest_debug = dbg->control; 9261 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE)) 9262 vcpu->guest_debug = 0; 9263 9264 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 9265 for (i = 0; i < KVM_NR_DB_REGS; ++i) 9266 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; 9267 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7]; 9268 } else { 9269 for (i = 0; i < KVM_NR_DB_REGS; i++) 9270 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 9271 } 9272 kvm_update_dr7(vcpu); 9273 9274 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 9275 vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu); 9276 9277 /* 9278 * Trigger an rflags update that will inject or remove the trace 9279 * flags. 9280 */ 9281 kvm_set_rflags(vcpu, rflags); 9282 9283 kvm_x86_call(update_exception_bitmap)(vcpu); 9284 9285 kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm); 9286 9287 r = 0; 9288 9289 out: 9290 vcpu_put(vcpu); 9291 return r; 9292 } 9293 9294 /* 9295 * Translate a guest virtual address to a guest physical address. 9296 */ 9297 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 9298 struct kvm_translation *tr) 9299 { 9300 unsigned long vaddr = tr->linear_address; 9301 gpa_t gpa; 9302 int idx; 9303 9304 vcpu_load(vcpu); 9305 9306 idx = srcu_read_lock(&vcpu->kvm->srcu); 9307 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL); 9308 srcu_read_unlock(&vcpu->kvm->srcu, idx); 9309 tr->physical_address = gpa; 9310 tr->valid = gpa != INVALID_GPA; 9311 tr->writeable = 1; 9312 tr->usermode = 0; 9313 9314 vcpu_put(vcpu); 9315 return 0; 9316 } 9317 9318 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 9319 { 9320 struct fxregs_state *fxsave; 9321 9322 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 9323 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 9324 9325 vcpu_load(vcpu); 9326 9327 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; 9328 memcpy(fpu->fpr, fxsave->st_space, 128); 9329 fpu->fcw = fxsave->cwd; 9330 fpu->fsw = fxsave->swd; 9331 fpu->ftwx = fxsave->twd; 9332 fpu->last_opcode = fxsave->fop; 9333 fpu->last_ip = fxsave->rip; 9334 fpu->last_dp = fxsave->rdp; 9335 memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space)); 9336 9337 vcpu_put(vcpu); 9338 return 0; 9339 } 9340 9341 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 9342 { 9343 struct fxregs_state *fxsave; 9344 9345 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 9346 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 9347 9348 vcpu_load(vcpu); 9349 9350 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; 9351 9352 memcpy(fxsave->st_space, fpu->fpr, 128); 9353 fxsave->cwd = fpu->fcw; 9354 fxsave->swd = fpu->fsw; 9355 fxsave->twd = fpu->ftwx; 9356 fxsave->fop = fpu->last_opcode; 9357 fxsave->rip = fpu->last_ip; 9358 fxsave->rdp = fpu->last_dp; 9359 memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space)); 9360 9361 vcpu_put(vcpu); 9362 return 0; 9363 } 9364 9365 static void store_regs(struct kvm_vcpu *vcpu) 9366 { 9367 kvm_run_sync_regs_to_user(vcpu); 9368 9369 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS) 9370 kvm_vcpu_ioctl_x86_get_vcpu_events( 9371 vcpu, &vcpu->run->s.regs.events); 9372 } 9373 9374 static int sync_regs(struct kvm_vcpu *vcpu) 9375 { 9376 if (kvm_run_sync_regs_from_user(vcpu)) 9377 return -EINVAL; 9378 9379 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) { 9380 struct kvm_vcpu_events events = vcpu->run->s.regs.events; 9381 9382 if (kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events)) 9383 return -EINVAL; 9384 9385 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS; 9386 } 9387 9388 return 0; 9389 } 9390 9391 #define PERF_MEDIATED_PMU_MSG \ 9392 "Failed to enable mediated vPMU, try disabling system wide perf events and nmi_watchdog.\n" 9393 9394 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 9395 { 9396 int r; 9397 9398 if (kvm_check_tsc_unstable() && kvm->created_vcpus) 9399 pr_warn_once("SMP vm created on host with unstable TSC; " 9400 "guest TSC will not be reliable\n"); 9401 9402 if (!kvm->arch.max_vcpu_ids) 9403 kvm->arch.max_vcpu_ids = KVM_MAX_VCPU_IDS; 9404 9405 if (id >= kvm->arch.max_vcpu_ids) 9406 return -EINVAL; 9407 9408 /* 9409 * Note, any actions done by .vcpu_create() must be idempotent with 9410 * respect to creating multiple vCPUs, and therefore are not undone if 9411 * creating a vCPU fails (including failure during pre-create). 9412 */ 9413 r = kvm_x86_call(vcpu_precreate)(kvm); 9414 if (r) 9415 return r; 9416 9417 if (enable_mediated_pmu && kvm->arch.enable_pmu && 9418 !kvm->arch.created_mediated_pmu) { 9419 if (irqchip_in_kernel(kvm)) { 9420 r = perf_create_mediated_pmu(); 9421 if (r) { 9422 pr_warn_ratelimited(PERF_MEDIATED_PMU_MSG); 9423 return r; 9424 } 9425 kvm->arch.created_mediated_pmu = true; 9426 } else { 9427 kvm->arch.enable_pmu = false; 9428 } 9429 } 9430 return 0; 9431 } 9432 9433 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 9434 { 9435 struct page *page; 9436 int r; 9437 9438 vcpu->arch.last_vmentry_cpu = -1; 9439 bitmap_fill(vcpu->arch.regs_avail, NR_VCPU_TOTAL_REGS); 9440 bitmap_fill(vcpu->arch.regs_dirty, NR_VCPU_TOTAL_REGS); 9441 9442 kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm); 9443 9444 if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) 9445 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 9446 else 9447 kvm_set_mp_state(vcpu, KVM_MP_STATE_UNINITIALIZED); 9448 9449 r = kvm_mmu_create(vcpu); 9450 if (r < 0) 9451 return r; 9452 9453 r = kvm_create_lapic(vcpu); 9454 if (r < 0) 9455 goto fail_mmu_destroy; 9456 9457 r = -ENOMEM; 9458 9459 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 9460 if (!page) 9461 goto fail_free_lapic; 9462 vcpu->arch.pio_data = page_address(page); 9463 9464 vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64), 9465 GFP_KERNEL_ACCOUNT); 9466 vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64), 9467 GFP_KERNEL_ACCOUNT); 9468 if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks) 9469 goto fail_free_mce_banks; 9470 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS; 9471 9472 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, 9473 GFP_KERNEL_ACCOUNT)) 9474 goto fail_free_mce_banks; 9475 9476 if (!alloc_emulate_ctxt(vcpu)) 9477 goto free_wbinvd_dirty_mask; 9478 9479 if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) { 9480 pr_err("failed to allocate vcpu's fpu\n"); 9481 goto free_emulate_ctxt; 9482 } 9483 9484 kvm_async_pf_hash_reset(vcpu); 9485 9486 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) { 9487 vcpu->arch.arch_capabilities = kvm_get_arch_capabilities(); 9488 vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT; 9489 vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap; 9490 } 9491 kvm_pmu_init(vcpu); 9492 9493 vcpu->arch.pending_external_vector = -1; 9494 vcpu->arch.preempted_in_kernel = false; 9495 9496 #if IS_ENABLED(CONFIG_HYPERV) 9497 vcpu->arch.hv_root_tdp = INVALID_PAGE; 9498 #endif 9499 9500 r = kvm_x86_call(vcpu_create)(vcpu); 9501 if (r) 9502 goto free_guest_fpu; 9503 9504 kvm_xen_init_vcpu(vcpu); 9505 vcpu_load(vcpu); 9506 kvm_vcpu_after_set_cpuid(vcpu); 9507 kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz); 9508 kvm_vcpu_reset(vcpu, false); 9509 kvm_init_mmu(vcpu); 9510 vcpu_put(vcpu); 9511 return 0; 9512 9513 free_guest_fpu: 9514 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); 9515 free_emulate_ctxt: 9516 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); 9517 free_wbinvd_dirty_mask: 9518 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); 9519 fail_free_mce_banks: 9520 kfree(vcpu->arch.mce_banks); 9521 kfree(vcpu->arch.mci_ctl2_banks); 9522 free_page((unsigned long)vcpu->arch.pio_data); 9523 fail_free_lapic: 9524 kvm_free_lapic(vcpu); 9525 fail_mmu_destroy: 9526 kvm_mmu_destroy(vcpu); 9527 return r; 9528 } 9529 9530 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 9531 { 9532 if (mutex_lock_killable(&vcpu->mutex)) 9533 return; 9534 vcpu_load(vcpu); 9535 kvm_synchronize_tsc(vcpu, NULL); 9536 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) 9537 kvm_update_masterclock(vcpu->kvm); 9538 vcpu_put(vcpu); 9539 9540 /* poll control enabled by default */ 9541 vcpu->arch.msr_kvm_poll_control = 1; 9542 9543 mutex_unlock(&vcpu->mutex); 9544 } 9545 9546 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 9547 { 9548 int idx, cpu; 9549 9550 kvm_clear_async_pf_completion_queue(vcpu); 9551 kvm_mmu_unload(vcpu); 9552 9553 kvmclock_reset(vcpu); 9554 9555 for_each_possible_cpu(cpu) 9556 cmpxchg(per_cpu_ptr(&last_vcpu, cpu), vcpu, NULL); 9557 9558 kvm_x86_call(vcpu_free)(vcpu); 9559 9560 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); 9561 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); 9562 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); 9563 9564 kvm_xen_destroy_vcpu(vcpu); 9565 kvm_hv_vcpu_uninit(vcpu); 9566 kvm_pmu_destroy(vcpu); 9567 kfree(vcpu->arch.mce_banks); 9568 kfree(vcpu->arch.mci_ctl2_banks); 9569 kvm_free_lapic(vcpu); 9570 idx = srcu_read_lock(&vcpu->kvm->srcu); 9571 kvm_mmu_destroy(vcpu); 9572 srcu_read_unlock(&vcpu->kvm->srcu, idx); 9573 free_page((unsigned long)vcpu->arch.pio_data); 9574 kvfree(vcpu->arch.cpuid_entries); 9575 } 9576 9577 static void kvm_xstate_reset(struct kvm_vcpu *vcpu, bool init_event) 9578 { 9579 struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate; 9580 u64 xfeatures_mask; 9581 bool fpu_in_use; 9582 int i; 9583 9584 /* 9585 * Guest FPU state is zero allocated and so doesn't need to be manually 9586 * cleared on RESET, i.e. during vCPU creation. 9587 */ 9588 if (!init_event || !fpstate) 9589 return; 9590 9591 /* 9592 * On INIT, only select XSTATE components are zeroed, most components 9593 * are unchanged. Currently, the only components that are zeroed and 9594 * supported by KVM are MPX and CET related. 9595 */ 9596 xfeatures_mask = (kvm_caps.supported_xcr0 | kvm_caps.supported_xss) & 9597 (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR | 9598 XFEATURE_MASK_CET_ALL); 9599 if (!xfeatures_mask) 9600 return; 9601 9602 BUILD_BUG_ON(sizeof(xfeatures_mask) * BITS_PER_BYTE <= XFEATURE_MAX); 9603 9604 /* 9605 * Unload guest FPU state (if necessary) before zeroing XSTATE fields 9606 * as the kernel can only modify the state when its resident in memory, 9607 * i.e. when it's not loaded into hardware. 9608 * 9609 * WARN if the vCPU's desire to run, i.e. whether or not its in KVM_RUN, 9610 * doesn't match the loaded/in-use state of the FPU, as KVM_RUN is the 9611 * only path that can trigger INIT emulation _and_ loads FPU state, and 9612 * KVM_RUN should _always_ load FPU state. 9613 */ 9614 WARN_ON_ONCE(vcpu->wants_to_run != fpstate->in_use); 9615 fpu_in_use = fpstate->in_use; 9616 if (fpu_in_use) 9617 kvm_put_guest_fpu(vcpu); 9618 for_each_set_bit(i, (unsigned long *)&xfeatures_mask, XFEATURE_MAX) 9619 fpstate_clear_xstate_component(fpstate, i); 9620 if (fpu_in_use) 9621 kvm_load_guest_fpu(vcpu); 9622 } 9623 9624 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 9625 { 9626 struct kvm_cpuid_entry2 *cpuid_0x1; 9627 unsigned long old_cr0 = kvm_read_cr0(vcpu); 9628 unsigned long new_cr0; 9629 9630 /* 9631 * Several of the "set" flows, e.g. ->set_cr0(), read other registers 9632 * to handle side effects. RESET emulation hits those flows and relies 9633 * on emulated/virtualized registers, including those that are loaded 9634 * into hardware, to be zeroed at vCPU creation. Use CRs as a sentinel 9635 * to detect improper or missing initialization. 9636 */ 9637 WARN_ON_ONCE(!init_event && 9638 (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu))); 9639 9640 /* 9641 * SVM doesn't unconditionally VM-Exit on INIT and SHUTDOWN, thus it's 9642 * possible to INIT the vCPU while L2 is active. Force the vCPU back 9643 * into L1 as EFER.SVME is cleared on INIT (along with all other EFER 9644 * bits), i.e. virtualization is disabled. 9645 */ 9646 if (is_guest_mode(vcpu)) 9647 kvm_leave_nested(vcpu); 9648 9649 kvm_lapic_reset(vcpu, init_event); 9650 9651 WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu)); 9652 vcpu->arch.hflags = 0; 9653 9654 vcpu->arch.smi_pending = 0; 9655 vcpu->arch.smi_count = 0; 9656 atomic_set(&vcpu->arch.nmi_queued, 0); 9657 vcpu->arch.nmi_pending = 0; 9658 vcpu->arch.nmi_injected = false; 9659 kvm_clear_interrupt_queue(vcpu); 9660 kvm_clear_exception_queue(vcpu); 9661 9662 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); 9663 kvm_update_dr0123(vcpu); 9664 vcpu->arch.dr6 = DR6_ACTIVE_LOW; 9665 vcpu->arch.dr7 = DR7_FIXED_1; 9666 kvm_update_dr7(vcpu); 9667 9668 vcpu->arch.cr2 = 0; 9669 9670 kvm_make_request(KVM_REQ_EVENT, vcpu); 9671 vcpu->arch.apf.msr_en_val = 0; 9672 vcpu->arch.apf.msr_int_val = 0; 9673 vcpu->arch.st.msr_val = 0; 9674 9675 kvmclock_reset(vcpu); 9676 9677 kvm_clear_async_pf_completion_queue(vcpu); 9678 kvm_async_pf_hash_reset(vcpu); 9679 vcpu->arch.apf.halted = false; 9680 9681 kvm_xstate_reset(vcpu, init_event); 9682 9683 if (!init_event) { 9684 vcpu->arch.smbase = 0x30000; 9685 9686 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT; 9687 9688 vcpu->arch.msr_misc_features_enables = 0; 9689 vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL | 9690 MSR_IA32_MISC_ENABLE_BTS_UNAVAIL; 9691 9692 __kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP); 9693 kvm_msr_write(vcpu, MSR_IA32_XSS, 0); 9694 } 9695 9696 /* All GPRs except RDX (handled below) are zeroed on RESET/INIT. */ 9697 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs)); 9698 kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP); 9699 9700 /* 9701 * Fall back to KVM's default Family/Model/Stepping of 0x600 (P6/Athlon) 9702 * if no CPUID match is found. Note, it's impossible to get a match at 9703 * RESET since KVM emulates RESET before exposing the vCPU to userspace, 9704 * i.e. it's impossible for kvm_find_cpuid_entry() to find a valid entry 9705 * on RESET. But, go through the motions in case that's ever remedied. 9706 */ 9707 cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1); 9708 kvm_edx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600); 9709 9710 kvm_x86_call(vcpu_reset)(vcpu, init_event); 9711 9712 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED); 9713 kvm_rip_write(vcpu, 0xfff0); 9714 9715 vcpu->arch.cr3 = 0; 9716 kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); 9717 9718 /* 9719 * CR0.CD/NW are set on RESET, preserved on INIT. Note, some versions 9720 * of Intel's SDM list CD/NW as being set on INIT, but they contradict 9721 * (or qualify) that with a footnote stating that CD/NW are preserved. 9722 */ 9723 new_cr0 = X86_CR0_ET; 9724 if (init_event) 9725 new_cr0 |= (old_cr0 & (X86_CR0_NW | X86_CR0_CD)); 9726 else 9727 new_cr0 |= X86_CR0_NW | X86_CR0_CD; 9728 9729 kvm_x86_call(set_cr0)(vcpu, new_cr0); 9730 kvm_x86_call(set_cr4)(vcpu, 0); 9731 kvm_x86_call(set_efer)(vcpu, 0); 9732 kvm_x86_call(update_exception_bitmap)(vcpu); 9733 9734 /* 9735 * Unconditionally reset the MMU, even though super strictly speaking, 9736 * a MMU reset is only necessary if and only one of the page role bits 9737 * is changing. All of the CR0/CR4/EFER bits effectively get covered 9738 * by checking CR0.PG=1, as CR0.WP and the paging-related bits in CR4 9739 * and EFER are irrelevant if CR0.PG was '0'. Subtly, if the vCPU was 9740 * in guest mode (L2) or SMM (although emulating INIT in SMM _should_ 9741 * be impossible), an MMU reset is also necessary as "guest mode" and 9742 * "smm" are both encoded in the shadow page role. As a result, the 9743 * only scenarios where an MMU reset is *not* needed is if the vCPU is 9744 * in Real Mode or Unpaged Protected Mode, and micro-optimizing those 9745 * paths are not worth the risk of running with a stale MMU context. 9746 */ 9747 kvm_mmu_reset_context(vcpu); 9748 9749 /* 9750 * Intel's SDM states that all TLB entries are flushed on INIT "as with 9751 * a hardware reset", and AMD's APM states the TLBs are flushed on 9752 * "External initialization of the processor". Flush the guest TLB for 9753 * both RESET and INIT, regardless of vendor, even though both the SDM 9754 * and APM contradict the above statements in other tables. There is 9755 * no meaningful benefit in relying on the guest to flush the TLB, and 9756 * a spurious TLB flush is benign and likely negligible from a 9757 * performance perspective (especially for RESET). 9758 */ 9759 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 9760 } 9761 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_reset); 9762 9763 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 9764 { 9765 struct kvm_segment cs; 9766 9767 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS); 9768 cs.selector = vector << 8; 9769 cs.base = vector << 12; 9770 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS); 9771 kvm_rip_write(vcpu, 0); 9772 } 9773 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_deliver_sipi_vector); 9774 9775 void kvm_arch_enable_virtualization(void) 9776 { 9777 x86_virt_register_emergency_callback(kvm_x86_ops.emergency_disable_virtualization_cpu); 9778 } 9779 9780 void kvm_arch_disable_virtualization(void) 9781 { 9782 x86_virt_unregister_emergency_callback(kvm_x86_ops.emergency_disable_virtualization_cpu); 9783 } 9784 9785 int kvm_arch_enable_virtualization_cpu(void) 9786 { 9787 struct kvm *kvm; 9788 struct kvm_vcpu *vcpu; 9789 unsigned long i; 9790 int ret; 9791 u64 local_tsc; 9792 u64 max_tsc = 0; 9793 bool stable, backwards_tsc = false; 9794 9795 kvm_user_return_msr_cpu_online(); 9796 9797 ret = kvm_x86_check_processor_compatibility(); 9798 if (ret) 9799 return ret; 9800 9801 ret = kvm_x86_call(enable_virtualization_cpu)(); 9802 if (ret != 0) 9803 return ret; 9804 9805 local_tsc = rdtsc(); 9806 stable = !kvm_check_tsc_unstable(); 9807 list_for_each_entry(kvm, &vm_list, vm_list) { 9808 kvm_for_each_vcpu(i, vcpu, kvm) { 9809 if (!stable && vcpu->cpu == smp_processor_id()) 9810 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 9811 if (stable && vcpu->arch.last_host_tsc > local_tsc) { 9812 backwards_tsc = true; 9813 if (vcpu->arch.last_host_tsc > max_tsc) 9814 max_tsc = vcpu->arch.last_host_tsc; 9815 } 9816 } 9817 } 9818 9819 /* 9820 * Sometimes, even reliable TSCs go backwards. This happens on 9821 * platforms that reset TSC during suspend or hibernate actions, but 9822 * maintain synchronization. We must compensate. Fortunately, we can 9823 * detect that condition here, which happens early in CPU bringup, 9824 * before any KVM threads can be running. Unfortunately, we can't 9825 * bring the TSCs fully up to date with real time, as we aren't yet far 9826 * enough into CPU bringup that we know how much real time has actually 9827 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot 9828 * variables that haven't been updated yet. 9829 * 9830 * So we simply find the maximum observed TSC above, then record the 9831 * adjustment to TSC in each VCPU. When the VCPU later gets loaded, 9832 * the adjustment will be applied. Note that we accumulate 9833 * adjustments, in case multiple suspend cycles happen before some VCPU 9834 * gets a chance to run again. In the event that no KVM threads get a 9835 * chance to run, we will miss the entire elapsed period, as we'll have 9836 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may 9837 * loose cycle time. This isn't too big a deal, since the loss will be 9838 * uniform across all VCPUs (not to mention the scenario is extremely 9839 * unlikely). It is possible that a second hibernate recovery happens 9840 * much faster than a first, causing the observed TSC here to be 9841 * smaller; this would require additional padding adjustment, which is 9842 * why we set last_host_tsc to the local tsc observed here. 9843 * 9844 * N.B. - this code below runs only on platforms with reliable TSC, 9845 * as that is the only way backwards_tsc is set above. Also note 9846 * that this runs for ALL vcpus, which is not a bug; all VCPUs should 9847 * have the same delta_cyc adjustment applied if backwards_tsc 9848 * is detected. Note further, this adjustment is only done once, 9849 * as we reset last_host_tsc on all VCPUs to stop this from being 9850 * called multiple times (one for each physical CPU bringup). 9851 * 9852 * Platforms with unreliable TSCs don't have to deal with this, they 9853 * will be compensated by the logic in vcpu_load, which sets the TSC to 9854 * catchup mode. This will catchup all VCPUs to real time, but cannot 9855 * guarantee that they stay in perfect synchronization. 9856 */ 9857 if (backwards_tsc) { 9858 u64 delta_cyc = max_tsc - local_tsc; 9859 list_for_each_entry(kvm, &vm_list, vm_list) { 9860 kvm->arch.backwards_tsc_observed = true; 9861 kvm_for_each_vcpu(i, vcpu, kvm) { 9862 vcpu->arch.tsc_offset_adjustment += delta_cyc; 9863 vcpu->arch.last_host_tsc = local_tsc; 9864 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 9865 } 9866 9867 /* 9868 * We have to disable TSC offset matching.. if you were 9869 * booting a VM while issuing an S4 host suspend.... 9870 * you may have some problem. Solving this issue is 9871 * left as an exercise to the reader. 9872 */ 9873 kvm->arch.last_tsc_nsec = 0; 9874 kvm->arch.last_tsc_write = 0; 9875 } 9876 9877 } 9878 return 0; 9879 } 9880 9881 void kvm_arch_shutdown(void) 9882 { 9883 /* 9884 * Set virt_rebooting to indicate that KVM has asynchronously disabled 9885 * hardware virtualization, i.e. that errors and/or exceptions on SVM 9886 * and VMX instructions are expected and should be ignored. 9887 */ 9888 virt_rebooting = true; 9889 9890 /* 9891 * Ensure virt_rebooting is visible before IPIs are sent to other CPUs 9892 * to disable virtualization. Effectively pairs with the reception of 9893 * the IPI (virt_rebooting is read in task/exception context, but only 9894 * _needs_ to be read as %true after the IPI function callback disables 9895 * virtualization). 9896 */ 9897 smp_wmb(); 9898 } 9899 9900 void kvm_arch_disable_virtualization_cpu(void) 9901 { 9902 kvm_x86_call(disable_virtualization_cpu)(); 9903 9904 /* 9905 * Leave the user-return notifiers as-is when disabling virtualization 9906 * for reboot, i.e. when disabling via IPI function call, and instead 9907 * pin kvm.ko (if it's a module) to defend against use-after-free (in 9908 * the *very* unlikely scenario module unload is racing with reboot). 9909 * On a forced reboot, tasks aren't frozen before shutdown, and so KVM 9910 * could be actively modifying user-return MSR state when the IPI to 9911 * disable virtualization arrives. Handle the extreme edge case here 9912 * instead of trying to account for it in the normal flows. 9913 */ 9914 if (in_task() || WARN_ON_ONCE(!virt_rebooting)) 9915 drop_user_return_notifiers(); 9916 else 9917 __module_get(THIS_MODULE); 9918 } 9919 9920 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu) 9921 { 9922 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id; 9923 } 9924 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_is_reset_bsp); 9925 9926 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) 9927 { 9928 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0; 9929 } 9930 9931 void kvm_arch_free_vm(struct kvm *kvm) 9932 { 9933 #if IS_ENABLED(CONFIG_HYPERV) 9934 kfree(kvm->arch.hv_pa_pg); 9935 #endif 9936 __kvm_arch_free_vm(kvm); 9937 } 9938 9939 9940 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 9941 { 9942 int ret; 9943 unsigned long flags; 9944 9945 if (!kvm_is_vm_type_supported(type)) 9946 return -EINVAL; 9947 9948 kvm->arch.vm_type = type; 9949 kvm->arch.has_private_mem = 9950 (type == KVM_X86_SW_PROTECTED_VM); 9951 /* Decided by the vendor code for other VM types. */ 9952 kvm->arch.pre_fault_allowed = 9953 type == KVM_X86_DEFAULT_VM || type == KVM_X86_SW_PROTECTED_VM; 9954 kvm->arch.disabled_quirks = kvm_caps.inapplicable_quirks & kvm_caps.supported_quirks; 9955 9956 ret = kvm_page_track_init(kvm); 9957 if (ret) 9958 goto out; 9959 9960 ret = kvm_mmu_init_vm(kvm); 9961 if (ret) 9962 goto out_cleanup_page_track; 9963 9964 ret = kvm_x86_call(vm_init)(kvm); 9965 if (ret) 9966 goto out_uninit_mmu; 9967 9968 atomic_set(&kvm->arch.noncoherent_dma_count, 0); 9969 9970 raw_spin_lock_init(&kvm->arch.tsc_write_lock); 9971 mutex_init(&kvm->arch.apic_map_lock); 9972 seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock); 9973 ratelimit_state_init(&kvm->arch.kvmclock_update_rs, HZ, 10); 9974 ratelimit_set_flags(&kvm->arch.kvmclock_update_rs, RATELIMIT_MSG_ON_RELEASE); 9975 kvm->arch.kvmclock_offset = -get_kvmclock_base_ns(); 9976 9977 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 9978 pvclock_update_vm_gtod_copy(kvm); 9979 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 9980 9981 kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz; 9982 kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT; 9983 kvm->arch.guest_can_read_msr_platform_info = true; 9984 kvm->arch.enable_pmu = enable_pmu && !kvm->arch.has_protected_pmu; 9985 9986 #if IS_ENABLED(CONFIG_HYPERV) 9987 spin_lock_init(&kvm->arch.hv_root_tdp_lock); 9988 kvm->arch.hv_root_tdp = INVALID_PAGE; 9989 #endif 9990 9991 kvm_apicv_init(kvm); 9992 kvm_hv_init_vm(kvm); 9993 kvm_xen_init_vm(kvm); 9994 9995 if (ignore_msrs && !report_ignored_msrs) { 9996 pr_warn_once("Running KVM with ignore_msrs=1 and report_ignored_msrs=0 is not a\n" 9997 "a supported configuration. Lying to the guest about the existence of MSRs\n" 9998 "may cause the guest operating system to hang or produce errors. If a guest\n" 9999 "does not run without ignore_msrs=1, please report it to kvm@vger.kernel.org.\n"); 10000 } 10001 10002 once_init(&kvm->arch.nx_once); 10003 return 0; 10004 10005 out_uninit_mmu: 10006 kvm_mmu_uninit_vm(kvm); 10007 out_cleanup_page_track: 10008 kvm_page_track_cleanup(kvm); 10009 out: 10010 return ret; 10011 } 10012 10013 /** 10014 * __x86_set_memory_region: Setup KVM internal memory slot 10015 * 10016 * @kvm: the kvm pointer to the VM. 10017 * @id: the slot ID to setup. 10018 * @gpa: the GPA to install the slot (unused when @size == 0). 10019 * @size: the size of the slot. Set to zero to uninstall a slot. 10020 * 10021 * This function helps to setup a KVM internal memory slot. Specify 10022 * @size > 0 to install a new slot, while @size == 0 to uninstall a 10023 * slot. The return code can be one of the following: 10024 * 10025 * HVA: on success (uninstall will return a bogus HVA) 10026 * -errno: on error 10027 * 10028 * The caller should always use IS_ERR() to check the return value 10029 * before use. Note, the KVM internal memory slots are guaranteed to 10030 * remain valid and unchanged until the VM is destroyed, i.e., the 10031 * GPA->HVA translation will not change. However, the HVA is a user 10032 * address, i.e. its accessibility is not guaranteed, and must be 10033 * accessed via __copy_{to,from}_user(). 10034 */ 10035 void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, 10036 u32 size) 10037 { 10038 int i, r; 10039 unsigned long hva, old_npages; 10040 struct kvm_memslots *slots = kvm_memslots(kvm); 10041 struct kvm_memory_slot *slot; 10042 10043 lockdep_assert_held(&kvm->slots_lock); 10044 10045 if (WARN_ON(id >= KVM_MEM_SLOTS_NUM)) 10046 return ERR_PTR_USR(-EINVAL); 10047 10048 slot = id_to_memslot(slots, id); 10049 if (size) { 10050 if (slot && slot->npages) 10051 return ERR_PTR_USR(-EEXIST); 10052 10053 /* 10054 * MAP_SHARED to prevent internal slot pages from being moved 10055 * by fork()/COW. 10056 */ 10057 hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE, 10058 MAP_SHARED | MAP_ANONYMOUS, 0); 10059 if (IS_ERR_VALUE(hva)) 10060 return (void __user *)hva; 10061 } else { 10062 if (!slot || !slot->npages) 10063 return NULL; 10064 10065 old_npages = slot->npages; 10066 hva = slot->userspace_addr; 10067 } 10068 10069 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { 10070 struct kvm_userspace_memory_region2 m; 10071 10072 m.slot = id | (i << 16); 10073 m.flags = 0; 10074 m.guest_phys_addr = gpa; 10075 m.userspace_addr = hva; 10076 m.memory_size = size; 10077 r = kvm_set_internal_memslot(kvm, &m); 10078 if (r < 0) 10079 return ERR_PTR_USR(r); 10080 } 10081 10082 if (!size) 10083 vm_munmap(hva, old_npages * PAGE_SIZE); 10084 10085 return (void __user *)hva; 10086 } 10087 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__x86_set_memory_region); 10088 10089 void kvm_arch_pre_destroy_vm(struct kvm *kvm) 10090 { 10091 /* 10092 * Stop all background workers and kthreads before destroying vCPUs, as 10093 * iterating over vCPUs in a different task while vCPUs are being freed 10094 * is unsafe, i.e. will lead to use-after-free. The PIT also needs to 10095 * be stopped before IRQ routing is freed. 10096 * 10097 * Do NOT free the in-kernel PIC or I/O APIC here (but as above, make 10098 * sure to flush any background work), as KVM expects interrupt routing 10099 * structures to be valid until vCPUs are destroyed. 10100 */ 10101 #ifdef CONFIG_KVM_IOAPIC 10102 kvm_free_pit(kvm); 10103 if (kvm->arch.vioapic) 10104 cancel_delayed_work_sync(&kvm->arch.vioapic->eoi_inject); 10105 #endif 10106 10107 kvm_mmu_pre_destroy_vm(kvm); 10108 kvm_x86_call(vm_pre_destroy)(kvm); 10109 } 10110 10111 void kvm_arch_destroy_vm(struct kvm *kvm) 10112 { 10113 if (current->mm == kvm->mm) { 10114 /* 10115 * Free memory regions allocated on behalf of userspace, 10116 * unless the memory map has changed due to process exit 10117 * or fd copying. 10118 */ 10119 mutex_lock(&kvm->slots_lock); 10120 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 10121 0, 0); 10122 __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 10123 0, 0); 10124 __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0); 10125 mutex_unlock(&kvm->slots_lock); 10126 } 10127 if (kvm->arch.created_mediated_pmu) 10128 perf_release_mediated_pmu(); 10129 kvm_destroy_vcpus(kvm); 10130 kvm_free_msr_filter((void * __force)kvm->arch.msr_filter); 10131 #ifdef CONFIG_KVM_IOAPIC 10132 kvm_pic_destroy(kvm); 10133 kvm_ioapic_destroy(kvm); 10134 #endif 10135 kvfree(rcu_dereference_check(kvm->arch.apic_map, 1)); 10136 kfree((void * __force)kvm->arch.pmu_event_filter); 10137 kvm_mmu_uninit_vm(kvm); 10138 kvm_page_track_cleanup(kvm); 10139 kvm_xen_destroy_vm(kvm); 10140 kvm_hv_destroy_vm(kvm); 10141 kvm_x86_call(vm_destroy)(kvm); 10142 } 10143 10144 static void memslot_rmap_free(struct kvm_memory_slot *slot) 10145 { 10146 int i; 10147 10148 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 10149 vfree(slot->arch.rmap[i]); 10150 slot->arch.rmap[i] = NULL; 10151 } 10152 } 10153 10154 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) 10155 { 10156 int i; 10157 10158 memslot_rmap_free(slot); 10159 10160 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { 10161 vfree(slot->arch.lpage_info[i - 1]); 10162 slot->arch.lpage_info[i - 1] = NULL; 10163 } 10164 10165 kvm_page_track_free_memslot(slot); 10166 } 10167 10168 int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages) 10169 { 10170 const int sz = sizeof(*slot->arch.rmap[0]); 10171 int i; 10172 10173 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 10174 int level = i + 1; 10175 int lpages = __kvm_mmu_slot_lpages(slot, npages, level); 10176 10177 if (slot->arch.rmap[i]) 10178 continue; 10179 10180 slot->arch.rmap[i] = __vcalloc(lpages, sz, GFP_KERNEL_ACCOUNT); 10181 if (!slot->arch.rmap[i]) { 10182 memslot_rmap_free(slot); 10183 return -ENOMEM; 10184 } 10185 } 10186 10187 return 0; 10188 } 10189 10190 static int kvm_alloc_memslot_metadata(struct kvm *kvm, 10191 struct kvm_memory_slot *slot) 10192 { 10193 unsigned long npages = slot->npages; 10194 int i, r; 10195 10196 /* 10197 * Clear out the previous array pointers for the KVM_MR_MOVE case. The 10198 * old arrays will be freed by kvm_set_memory_region() if installing 10199 * the new memslot is successful. 10200 */ 10201 memset(&slot->arch, 0, sizeof(slot->arch)); 10202 10203 if (kvm_memslots_have_rmaps(kvm)) { 10204 r = memslot_rmap_alloc(slot, npages); 10205 if (r) 10206 return r; 10207 } 10208 10209 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { 10210 struct kvm_lpage_info *linfo; 10211 unsigned long ugfn; 10212 int lpages; 10213 int level = i + 1; 10214 10215 lpages = __kvm_mmu_slot_lpages(slot, npages, level); 10216 10217 linfo = __vcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT); 10218 if (!linfo) 10219 goto out_free; 10220 10221 slot->arch.lpage_info[i - 1] = linfo; 10222 10223 if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1)) 10224 linfo[0].disallow_lpage = 1; 10225 if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1)) 10226 linfo[lpages - 1].disallow_lpage = 1; 10227 ugfn = slot->userspace_addr >> PAGE_SHIFT; 10228 /* 10229 * If the gfn and userspace address are not aligned wrt each 10230 * other, disable large page support for this slot. 10231 */ 10232 if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) { 10233 unsigned long j; 10234 10235 for (j = 0; j < lpages; ++j) 10236 linfo[j].disallow_lpage = 1; 10237 } 10238 } 10239 10240 #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES 10241 kvm_mmu_init_memslot_memory_attributes(kvm, slot); 10242 #endif 10243 10244 if (kvm_page_track_create_memslot(kvm, slot, npages)) 10245 goto out_free; 10246 10247 return 0; 10248 10249 out_free: 10250 memslot_rmap_free(slot); 10251 10252 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { 10253 vfree(slot->arch.lpage_info[i - 1]); 10254 slot->arch.lpage_info[i - 1] = NULL; 10255 } 10256 return -ENOMEM; 10257 } 10258 10259 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) 10260 { 10261 struct kvm_vcpu *vcpu; 10262 unsigned long i; 10263 10264 /* 10265 * memslots->generation has been incremented. 10266 * mmio generation may have reached its maximum value. 10267 */ 10268 kvm_mmu_invalidate_mmio_sptes(kvm, gen); 10269 10270 /* Force re-initialization of steal_time cache */ 10271 kvm_for_each_vcpu(i, vcpu, kvm) 10272 kvm_vcpu_kick(vcpu); 10273 } 10274 10275 int kvm_arch_prepare_memory_region(struct kvm *kvm, 10276 const struct kvm_memory_slot *old, 10277 struct kvm_memory_slot *new, 10278 enum kvm_mr_change change) 10279 { 10280 /* 10281 * KVM doesn't support moving memslots when there are external page 10282 * trackers attached to the VM, i.e. if KVMGT is in use. 10283 */ 10284 if (change == KVM_MR_MOVE && kvm_page_track_has_external_user(kvm)) 10285 return -EINVAL; 10286 10287 if (change == KVM_MR_CREATE || change == KVM_MR_MOVE) { 10288 if ((new->base_gfn + new->npages - 1) > kvm_mmu_max_gfn()) 10289 return -EINVAL; 10290 10291 if (kvm_is_gfn_alias(kvm, new->base_gfn + new->npages - 1)) 10292 return -EINVAL; 10293 10294 return kvm_alloc_memslot_metadata(kvm, new); 10295 } 10296 10297 if (change == KVM_MR_FLAGS_ONLY) 10298 memcpy(&new->arch, &old->arch, sizeof(old->arch)); 10299 else if (WARN_ON_ONCE(change != KVM_MR_DELETE)) 10300 return -EIO; 10301 10302 return 0; 10303 } 10304 10305 10306 static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable) 10307 { 10308 int nr_slots; 10309 10310 if (!kvm->arch.cpu_dirty_log_size) 10311 return; 10312 10313 nr_slots = atomic_read(&kvm->nr_memslots_dirty_logging); 10314 if ((enable && nr_slots == 1) || !nr_slots) 10315 kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING); 10316 } 10317 10318 static void kvm_mmu_slot_apply_flags(struct kvm *kvm, 10319 struct kvm_memory_slot *old, 10320 const struct kvm_memory_slot *new, 10321 enum kvm_mr_change change) 10322 { 10323 u32 old_flags = old ? old->flags : 0; 10324 u32 new_flags = new ? new->flags : 0; 10325 bool log_dirty_pages = new_flags & KVM_MEM_LOG_DIRTY_PAGES; 10326 10327 /* 10328 * Update CPU dirty logging if dirty logging is being toggled. This 10329 * applies to all operations. 10330 */ 10331 if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES) 10332 kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages); 10333 10334 /* 10335 * Nothing more to do for RO slots (which can't be dirtied and can't be 10336 * made writable) or CREATE/MOVE/DELETE of a slot. 10337 * 10338 * For a memslot with dirty logging disabled: 10339 * CREATE: No dirty mappings will already exist. 10340 * MOVE/DELETE: The old mappings will already have been cleaned up by 10341 * kvm_arch_flush_shadow_memslot() 10342 * 10343 * For a memslot with dirty logging enabled: 10344 * CREATE: No shadow pages exist, thus nothing to write-protect 10345 * and no dirty bits to clear. 10346 * MOVE/DELETE: The old mappings will already have been cleaned up by 10347 * kvm_arch_flush_shadow_memslot(). 10348 */ 10349 if ((change != KVM_MR_FLAGS_ONLY) || (new_flags & KVM_MEM_READONLY)) 10350 return; 10351 10352 /* 10353 * READONLY and non-flags changes were filtered out above, and the only 10354 * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty 10355 * logging isn't being toggled on or off. 10356 */ 10357 if (WARN_ON_ONCE(!((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES))) 10358 return; 10359 10360 if (!log_dirty_pages) { 10361 /* 10362 * Recover huge page mappings in the slot now that dirty logging 10363 * is disabled, i.e. now that KVM does not have to track guest 10364 * writes at 4KiB granularity. 10365 * 10366 * Dirty logging might be disabled by userspace if an ongoing VM 10367 * live migration is cancelled and the VM must continue running 10368 * on the source. 10369 */ 10370 kvm_mmu_recover_huge_pages(kvm, new); 10371 } else { 10372 /* 10373 * Initially-all-set does not require write protecting any page, 10374 * because they're all assumed to be dirty. 10375 */ 10376 if (kvm_dirty_log_manual_protect_and_init_set(kvm)) 10377 return; 10378 10379 if (READ_ONCE(eager_page_split)) 10380 kvm_mmu_slot_try_split_huge_pages(kvm, new, PG_LEVEL_4K); 10381 10382 if (kvm->arch.cpu_dirty_log_size) { 10383 kvm_mmu_slot_leaf_clear_dirty(kvm, new); 10384 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M); 10385 } else { 10386 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K); 10387 } 10388 10389 /* 10390 * Unconditionally flush the TLBs after enabling dirty logging. 10391 * A flush is almost always going to be necessary (see below), 10392 * and unconditionally flushing allows the helpers to omit 10393 * the subtly complex checks when removing write access. 10394 * 10395 * Do the flush outside of mmu_lock to reduce the amount of 10396 * time mmu_lock is held. Flushing after dropping mmu_lock is 10397 * safe as KVM only needs to guarantee the slot is fully 10398 * write-protected before returning to userspace, i.e. before 10399 * userspace can consume the dirty status. 10400 * 10401 * Flushing outside of mmu_lock requires KVM to be careful when 10402 * making decisions based on writable status of an SPTE, e.g. a 10403 * !writable SPTE doesn't guarantee a CPU can't perform writes. 10404 * 10405 * Specifically, KVM also write-protects guest page tables to 10406 * monitor changes when using shadow paging, and must guarantee 10407 * no CPUs can write to those page before mmu_lock is dropped. 10408 * Because CPUs may have stale TLB entries at this point, a 10409 * !writable SPTE doesn't guarantee CPUs can't perform writes. 10410 * 10411 * KVM also allows making SPTES writable outside of mmu_lock, 10412 * e.g. to allow dirty logging without taking mmu_lock. 10413 * 10414 * To handle these scenarios, KVM uses a separate software-only 10415 * bit (MMU-writable) to track if a SPTE is !writable due to 10416 * a guest page table being write-protected (KVM clears the 10417 * MMU-writable flag when write-protecting for shadow paging). 10418 * 10419 * The use of MMU-writable is also the primary motivation for 10420 * the unconditional flush. Because KVM must guarantee that a 10421 * CPU doesn't contain stale, writable TLB entries for a 10422 * !MMU-writable SPTE, KVM must flush if it encounters any 10423 * MMU-writable SPTE regardless of whether the actual hardware 10424 * writable bit was set. I.e. KVM is almost guaranteed to need 10425 * to flush, while unconditionally flushing allows the "remove 10426 * write access" helpers to ignore MMU-writable entirely. 10427 * 10428 * See is_writable_pte() for more details (the case involving 10429 * access-tracked SPTEs is particularly relevant). 10430 */ 10431 kvm_flush_remote_tlbs_memslot(kvm, new); 10432 } 10433 } 10434 10435 void kvm_arch_commit_memory_region(struct kvm *kvm, 10436 struct kvm_memory_slot *old, 10437 const struct kvm_memory_slot *new, 10438 enum kvm_mr_change change) 10439 { 10440 if (change == KVM_MR_DELETE) 10441 kvm_page_track_delete_slot(kvm, old); 10442 10443 if (!kvm->arch.n_requested_mmu_pages && 10444 (change == KVM_MR_CREATE || change == KVM_MR_DELETE)) { 10445 unsigned long nr_mmu_pages; 10446 10447 nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO; 10448 nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES); 10449 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); 10450 } 10451 10452 kvm_mmu_slot_apply_flags(kvm, old, new, change); 10453 10454 /* Free the arrays associated with the old memslot. */ 10455 if (change == KVM_MR_MOVE) 10456 kvm_arch_free_memslot(kvm, old); 10457 } 10458 10459 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 10460 { 10461 WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu)); 10462 10463 if (vcpu->arch.guest_state_protected) 10464 return true; 10465 10466 return kvm_x86_call(get_cpl)(vcpu) == 0; 10467 } 10468 10469 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) 10470 { 10471 WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu)); 10472 10473 if (vcpu->arch.guest_state_protected) 10474 return 0; 10475 10476 return kvm_rip_read(vcpu); 10477 } 10478 10479 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 10480 { 10481 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 10482 } 10483 10484 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) 10485 { 10486 BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU)); 10487 10488 return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU)); 10489 } 10490 10491 static inline u32 kvm_async_pf_next_probe(u32 key) 10492 { 10493 return (key + 1) & (ASYNC_PF_PER_VCPU - 1); 10494 } 10495 10496 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 10497 { 10498 u32 key = kvm_async_pf_hash_fn(gfn); 10499 10500 while (vcpu->arch.apf.gfns[key] != ~0) 10501 key = kvm_async_pf_next_probe(key); 10502 10503 vcpu->arch.apf.gfns[key] = gfn; 10504 } 10505 10506 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn) 10507 { 10508 int i; 10509 u32 key = kvm_async_pf_hash_fn(gfn); 10510 10511 for (i = 0; i < ASYNC_PF_PER_VCPU && 10512 (vcpu->arch.apf.gfns[key] != gfn && 10513 vcpu->arch.apf.gfns[key] != ~0); i++) 10514 key = kvm_async_pf_next_probe(key); 10515 10516 return key; 10517 } 10518 10519 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 10520 { 10521 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn; 10522 } 10523 10524 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 10525 { 10526 u32 i, j, k; 10527 10528 i = j = kvm_async_pf_gfn_slot(vcpu, gfn); 10529 10530 if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn)) 10531 return; 10532 10533 while (true) { 10534 vcpu->arch.apf.gfns[i] = ~0; 10535 do { 10536 j = kvm_async_pf_next_probe(j); 10537 if (vcpu->arch.apf.gfns[j] == ~0) 10538 return; 10539 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]); 10540 /* 10541 * k lies cyclically in ]i,j] 10542 * | i.k.j | 10543 * |....j i.k.| or |.k..j i...| 10544 */ 10545 } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j)); 10546 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j]; 10547 i = j; 10548 } 10549 } 10550 10551 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu) 10552 { 10553 u32 reason = KVM_PV_REASON_PAGE_NOT_PRESENT; 10554 10555 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason, 10556 sizeof(reason)); 10557 } 10558 10559 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token) 10560 { 10561 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token); 10562 10563 return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, 10564 &token, offset, sizeof(token)); 10565 } 10566 10567 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu) 10568 { 10569 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token); 10570 u32 val; 10571 10572 if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, 10573 &val, offset, sizeof(val))) 10574 return false; 10575 10576 return !val; 10577 } 10578 10579 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) 10580 { 10581 10582 if (!kvm_pv_async_pf_enabled(vcpu)) 10583 return false; 10584 10585 if (!(vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_SEND_ALWAYS) && 10586 (vcpu->arch.guest_state_protected || !kvm_x86_call(get_cpl)(vcpu))) 10587 return false; 10588 10589 if (is_guest_mode(vcpu)) { 10590 /* 10591 * L1 needs to opt into the special #PF vmexits that are 10592 * used to deliver async page faults. 10593 */ 10594 return vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; 10595 } else { 10596 /* 10597 * Play it safe in case the guest temporarily disables paging. 10598 * The real mode IDT in particular is unlikely to have a #PF 10599 * exception setup. 10600 */ 10601 return is_paging(vcpu); 10602 } 10603 } 10604 10605 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu) 10606 { 10607 if (unlikely(!lapic_in_kernel(vcpu) || 10608 kvm_event_needs_reinjection(vcpu) || 10609 kvm_is_exception_pending(vcpu))) 10610 return false; 10611 10612 if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu)) 10613 return false; 10614 10615 /* 10616 * If interrupts are off we cannot even use an artificial 10617 * halt state. 10618 */ 10619 return kvm_is_interrupt_allowed(vcpu); 10620 } 10621 10622 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 10623 struct kvm_async_pf *work) 10624 { 10625 struct x86_exception fault; 10626 10627 trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa); 10628 kvm_add_async_pf_gfn(vcpu, work->arch.gfn); 10629 10630 if (kvm_can_deliver_async_pf(vcpu) && 10631 !apf_put_user_notpresent(vcpu)) { 10632 fault.vector = PF_VECTOR; 10633 fault.error_code_valid = true; 10634 fault.error_code = 0; 10635 fault.nested_page_fault = false; 10636 fault.address = work->arch.token; 10637 fault.async_page_fault = true; 10638 kvm_inject_page_fault(vcpu, &fault, false); 10639 return true; 10640 } else { 10641 /* 10642 * It is not possible to deliver a paravirtualized asynchronous 10643 * page fault, but putting the guest in an artificial halt state 10644 * can be beneficial nevertheless: if an interrupt arrives, we 10645 * can deliver it timely and perhaps the guest will schedule 10646 * another process. When the instruction that triggered a page 10647 * fault is retried, hopefully the page will be ready in the host. 10648 */ 10649 kvm_make_request(KVM_REQ_APF_HALT, vcpu); 10650 return false; 10651 } 10652 } 10653 10654 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 10655 struct kvm_async_pf *work) 10656 { 10657 struct kvm_lapic_irq irq = { 10658 .delivery_mode = APIC_DM_FIXED, 10659 .vector = vcpu->arch.apf.vec 10660 }; 10661 10662 if (work->wakeup_all) 10663 work->arch.token = ~0; /* broadcast wakeup */ 10664 else 10665 kvm_del_async_pf_gfn(vcpu, work->arch.gfn); 10666 trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa); 10667 10668 if ((work->wakeup_all || work->notpresent_injected) && 10669 kvm_pv_async_pf_enabled(vcpu) && 10670 !apf_put_user_ready(vcpu, work->arch.token)) { 10671 WRITE_ONCE(vcpu->arch.apf.pageready_pending, true); 10672 kvm_apic_set_irq(vcpu, &irq, NULL); 10673 } 10674 10675 vcpu->arch.apf.halted = false; 10676 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 10677 } 10678 10679 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) 10680 { 10681 kvm_make_request(KVM_REQ_APF_READY, vcpu); 10682 10683 /* Pairs with smp_store_mb() in kvm_set_msr_common(). */ 10684 smp_mb__after_atomic(); 10685 10686 if (!READ_ONCE(vcpu->arch.apf.pageready_pending)) 10687 kvm_vcpu_kick(vcpu); 10688 } 10689 10690 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) 10691 { 10692 if (!kvm_pv_async_pf_enabled(vcpu)) 10693 return true; 10694 else 10695 return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu); 10696 } 10697 10698 static void kvm_noncoherent_dma_assignment_start_or_stop(struct kvm *kvm) 10699 { 10700 /* 10701 * Non-coherent DMA assignment and de-assignment may affect whether or 10702 * not KVM honors guest PAT, and thus may cause changes in EPT SPTEs 10703 * due to toggling the "ignore PAT" bit. Zap all SPTEs when the first 10704 * (or last) non-coherent device is (un)registered to so that new SPTEs 10705 * with the correct "ignore guest PAT" setting are created. 10706 * 10707 * If KVM always honors guest PAT, however, there is nothing to do. 10708 */ 10709 if (kvm_check_has_quirk(kvm, KVM_X86_QUIRK_IGNORE_GUEST_PAT)) 10710 kvm_zap_gfn_range(kvm, gpa_to_gfn(0), gpa_to_gfn(~0ULL)); 10711 } 10712 10713 void kvm_arch_register_noncoherent_dma(struct kvm *kvm) 10714 { 10715 if (atomic_inc_return(&kvm->arch.noncoherent_dma_count) == 1) 10716 kvm_noncoherent_dma_assignment_start_or_stop(kvm); 10717 } 10718 10719 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm) 10720 { 10721 if (!atomic_dec_return(&kvm->arch.noncoherent_dma_count)) 10722 kvm_noncoherent_dma_assignment_start_or_stop(kvm); 10723 } 10724 10725 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm) 10726 { 10727 return atomic_read(&kvm->arch.noncoherent_dma_count); 10728 } 10729 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_arch_has_noncoherent_dma); 10730 10731 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) 10732 { 10733 return (vcpu->arch.msr_kvm_poll_control & 1) == 0; 10734 } 10735 10736 #ifdef CONFIG_KVM_GUEST_MEMFD 10737 bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) 10738 { 10739 /* 10740 * INIT_SHARED is supported if in-place conversion is enabled, or if 10741 * the VM doesn't support private memory. If the VM has private memory 10742 * and in-place conversion is disabled, then guest_memfd can _only_ be 10743 * used for private memory. 10744 */ 10745 return gmem_in_place_conversion || !kvm_arch_has_private_mem(kvm); 10746 } 10747 10748 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT 10749 int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, 10750 kvm_pfn_t nr_pages) 10751 { 10752 return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages); 10753 } 10754 10755 void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages) 10756 { 10757 kvm_x86_call(gmem_make_shared)(pfn, nr_pages); 10758 } 10759 #endif 10760 10761 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM 10762 void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages) 10763 { 10764 kvm_x86_call(gmem_make_shared)(pfn, nr_pages); 10765 } 10766 #endif 10767 10768 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE 10769 void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) 10770 { 10771 kvm_x86_call(gmem_invalidate_range)(kvm, range); 10772 } 10773 #endif 10774 #endif 10775 10776 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code) 10777 { 10778 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 10779 struct x86_exception fault; 10780 u64 access = error_code & 10781 (PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK); 10782 10783 if (!(error_code & PFERR_PRESENT_MASK) || 10784 gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, &fault) != INVALID_GPA) { 10785 /* 10786 * If gva_walk->gva_to_gpa succeeded, the page 10787 * tables probably do not match the TLB. Just proceed 10788 * with the error code that the processor gave. 10789 */ 10790 fault.vector = PF_VECTOR; 10791 fault.error_code_valid = true; 10792 fault.error_code = error_code; 10793 fault.nested_page_fault = false; 10794 fault.address = gva; 10795 fault.async_page_fault = false; 10796 } 10797 gva_walk->inject_page_fault(vcpu, &fault, true); 10798 } 10799 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fixup_and_inject_pf_error); 10800 10801 /* 10802 * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns 10803 * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value 10804 * indicates whether exit to userspace is needed. 10805 */ 10806 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r, 10807 struct x86_exception *e) 10808 { 10809 if (r == X86EMUL_PROPAGATE_FAULT) { 10810 if (KVM_BUG_ON(!e, vcpu->kvm)) 10811 return -EIO; 10812 10813 kvm_inject_emulated_page_fault(vcpu, e); 10814 return 1; 10815 } 10816 10817 /* 10818 * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED 10819 * while handling a VMX instruction KVM could've handled the request 10820 * correctly by exiting to userspace and performing I/O but there 10821 * doesn't seem to be a real use-case behind such requests, just return 10822 * KVM_EXIT_INTERNAL_ERROR for now. 10823 */ 10824 kvm_prepare_emulation_failure_exit(vcpu); 10825 10826 return 0; 10827 } 10828 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_memory_failure); 10829 10830 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) 10831 { 10832 bool pcid_enabled; 10833 struct x86_exception e; 10834 struct { 10835 u64 pcid; 10836 u64 gla; 10837 } operand; 10838 int r; 10839 10840 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 10841 if (r != X86EMUL_CONTINUE) 10842 return kvm_handle_memory_failure(vcpu, r, &e); 10843 10844 if (operand.pcid >> 12 != 0) { 10845 kvm_inject_gp(vcpu, 0); 10846 return 1; 10847 } 10848 10849 if (WARN_ON_ONCE(tdp_enabled)) 10850 return 0; 10851 10852 pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE); 10853 10854 switch (type) { 10855 case INVPCID_TYPE_INDIV_ADDR: 10856 /* 10857 * LAM doesn't apply to addresses that are inputs to TLB 10858 * invalidation. 10859 */ 10860 if ((!pcid_enabled && (operand.pcid != 0)) || 10861 is_noncanonical_invlpg_address(operand.gla, vcpu)) { 10862 kvm_inject_gp(vcpu, 0); 10863 return 1; 10864 } 10865 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid); 10866 return kvm_skip_emulated_instruction(vcpu); 10867 10868 case INVPCID_TYPE_SINGLE_CTXT: 10869 if (!pcid_enabled && (operand.pcid != 0)) { 10870 kvm_inject_gp(vcpu, 0); 10871 return 1; 10872 } 10873 10874 /* 10875 * When ERAPS is supported, invalidating a specific PCID clears 10876 * the RAP (Return Address Predicator). 10877 */ 10878 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS)) 10879 kvm_register_mark_dirty(vcpu, VCPU_REG_ERAPS); 10880 10881 kvm_invalidate_pcid(vcpu, operand.pcid); 10882 return kvm_skip_emulated_instruction(vcpu); 10883 10884 case INVPCID_TYPE_ALL_NON_GLOBAL: 10885 /* 10886 * Currently, KVM doesn't mark global entries in the shadow 10887 * page tables, so a non-global flush just degenerates to a 10888 * global flush. If needed, we could optimize this later by 10889 * keeping track of global entries in shadow page tables. 10890 */ 10891 10892 fallthrough; 10893 case INVPCID_TYPE_ALL_INCL_GLOBAL: 10894 /* 10895 * Don't bother marking VCPU_REG_ERAPS dirty, SVM will take 10896 * care of doing so when emulating the full guest TLB flush 10897 * (the RAP is cleared on all implicit TLB flushes). 10898 */ 10899 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 10900 return kvm_skip_emulated_instruction(vcpu); 10901 10902 default: 10903 kvm_inject_gp(vcpu, 0); 10904 return 1; 10905 } 10906 } 10907 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_invpcid); 10908 10909 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu) 10910 { 10911 struct kvm_run *run = vcpu->run; 10912 struct kvm_mmio_fragment *frag; 10913 unsigned int len; 10914 10915 if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm)) 10916 return -EIO; 10917 10918 /* Complete previous fragment */ 10919 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment]; 10920 len = min(8u, frag->len); 10921 if (!vcpu->mmio_is_write) 10922 memcpy(frag->data, run->mmio.data, len); 10923 10924 if (frag->len <= 8) { 10925 /* Switch to the next fragment. */ 10926 frag++; 10927 vcpu->mmio_cur_fragment++; 10928 } else { 10929 /* Go forward to the next mmio piece. */ 10930 frag->data += len; 10931 frag->gpa += len; 10932 frag->len -= len; 10933 } 10934 10935 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) { 10936 vcpu->mmio_needed = 0; 10937 10938 /* 10939 * All done, as frag->data always points at the GHCB scratch 10940 * area and VMGEXIT is trap-like (RIP is advanced by hardware). 10941 */ 10942 return 1; 10943 } 10944 10945 // More MMIO is needed 10946 kvm_prepare_emulated_mmio_exit(vcpu, frag); 10947 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; 10948 return 0; 10949 } 10950 10951 int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa, 10952 unsigned int bytes, void *data) 10953 { 10954 struct kvm_mmio_fragment *frag; 10955 int handled; 10956 10957 if (!data || WARN_ON_ONCE(object_is_on_stack(data))) 10958 return -EINVAL; 10959 10960 if (is_write) 10961 handled = vcpu_mmio_write(vcpu, gpa, bytes, data); 10962 else 10963 handled = vcpu_mmio_read(vcpu, gpa, bytes, data); 10964 if (handled == bytes) 10965 return 1; 10966 10967 bytes -= handled; 10968 gpa += handled; 10969 data += handled; 10970 10971 /* 10972 * TODO: Determine whether or not userspace plays nice with MMIO 10973 * requests that split a page boundary. 10974 */ 10975 frag = vcpu->mmio_fragments; 10976 frag->len = bytes; 10977 frag->gpa = gpa; 10978 frag->data = data; 10979 10980 vcpu->mmio_needed = 1; 10981 vcpu->mmio_cur_fragment = 0; 10982 vcpu->mmio_nr_fragments = 1; 10983 vcpu->mmio_is_write = is_write; 10984 10985 kvm_prepare_emulated_mmio_exit(vcpu, frag); 10986 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; 10987 return 0; 10988 } 10989 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio); 10990 10991 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size) 10992 { 10993 vcpu->arch.sev_pio_count -= count; 10994 vcpu->arch.sev_pio_data += count * size; 10995 } 10996 10997 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size, 10998 unsigned int port); 10999 11000 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu) 11001 { 11002 int size = vcpu->arch.pio.size; 11003 int port = vcpu->arch.pio.port; 11004 11005 vcpu->arch.pio.count = 0; 11006 if (vcpu->arch.sev_pio_count) 11007 return kvm_sev_es_outs(vcpu, size, port); 11008 return 1; 11009 } 11010 11011 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size, 11012 unsigned int port) 11013 { 11014 for (;;) { 11015 unsigned int count = 11016 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count); 11017 int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count); 11018 11019 /* memcpy done already by emulator_pio_out. */ 11020 advance_sev_es_emulated_pio(vcpu, count, size); 11021 if (!ret) 11022 break; 11023 11024 /* Emulation done by the kernel. */ 11025 if (!vcpu->arch.sev_pio_count) 11026 return 1; 11027 } 11028 11029 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs; 11030 return 0; 11031 } 11032 11033 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size, 11034 unsigned int port); 11035 11036 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu) 11037 { 11038 unsigned count = vcpu->arch.pio.count; 11039 int size = vcpu->arch.pio.size; 11040 int port = vcpu->arch.pio.port; 11041 11042 complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data); 11043 advance_sev_es_emulated_pio(vcpu, count, size); 11044 if (vcpu->arch.sev_pio_count) 11045 return kvm_sev_es_ins(vcpu, size, port); 11046 return 1; 11047 } 11048 11049 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size, 11050 unsigned int port) 11051 { 11052 for (;;) { 11053 unsigned int count = 11054 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count); 11055 if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count)) 11056 break; 11057 11058 /* Emulation done by the kernel. */ 11059 advance_sev_es_emulated_pio(vcpu, count, size); 11060 if (!vcpu->arch.sev_pio_count) 11061 return 1; 11062 } 11063 11064 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins; 11065 return 0; 11066 } 11067 11068 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size, 11069 unsigned int port, void *data, unsigned int count, 11070 int in) 11071 { 11072 vcpu->arch.sev_pio_data = data; 11073 vcpu->arch.sev_pio_count = count; 11074 return in ? kvm_sev_es_ins(vcpu, size, port) 11075 : kvm_sev_es_outs(vcpu, size, port); 11076 } 11077 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_string_io); 11078 11079 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_entry); 11080 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit); 11081 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_mmio); 11082 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio); 11083 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); 11084 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); 11085 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr); 11086 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr); 11087 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter); 11088 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit); 11089 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject); 11090 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit); 11091 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed); 11092 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga); 11093 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit); 11094 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts); 11095 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset); 11096 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update); 11097 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full); 11098 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access); 11099 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi); 11100 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log); 11101 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_kick_vcpu_slowpath); 11102 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_doorbell); 11103 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq); 11104 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter); 11105 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit); 11106 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_enter); 11107 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_exit); 11108 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_rmp_fault); 11109 11110 static int __init kvm_x86_init(void) 11111 { 11112 kvm_init_xstate_sizes(); 11113 11114 kvm_mmu_x86_module_init(); 11115 mitigate_smt_rsb &= boot_cpu_has_bug(X86_BUG_SMT_RSB) && cpu_smt_possible(); 11116 return 0; 11117 } 11118 module_init(kvm_x86_init); 11119 11120 static void __exit kvm_x86_exit(void) 11121 { 11122 WARN_ON_ONCE(static_branch_unlikely(&kvm_has_noapic_vcpu)); 11123 } 11124 module_exit(kvm_x86_exit); 11125