1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * derived from drivers/kvm/kvm_main.c 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright (C) 2008 Qumranet, Inc. 9 * Copyright IBM Corporation, 2008 10 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 11 * 12 * Authors: 13 * Avi Kivity <avi@qumranet.com> 14 * Yaniv Kamay <yaniv@qumranet.com> 15 * Amit Shah <amit.shah@qumranet.com> 16 * Ben-Ami Yassour <benami@il.ibm.com> 17 */ 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/kvm_host.h> 21 #include "irq.h" 22 #include "ioapic.h" 23 #include "mmu.h" 24 #include "i8254.h" 25 #include "tss.h" 26 #include "regs.h" 27 #include "kvm_emulate.h" 28 #include "mmu/page_track.h" 29 #include "x86.h" 30 #include "cpuid.h" 31 #include "pmu.h" 32 #include "hyperv.h" 33 #include "lapic.h" 34 #include "xen.h" 35 #include "smm.h" 36 37 #include <linux/clocksource.h> 38 #include <linux/timekeeping.h> 39 #include <linux/interrupt.h> 40 #include <linux/kvm.h> 41 #include <linux/fs.h> 42 #include <linux/vmalloc.h> 43 #include <linux/export.h> 44 #include <linux/moduleparam.h> 45 #include <linux/mman.h> 46 #include <linux/highmem.h> 47 #include <linux/iommu.h> 48 #include <linux/cpufreq.h> 49 #include <linux/user-return-notifier.h> 50 #include <linux/srcu.h> 51 #include <linux/slab.h> 52 #include <linux/perf_event.h> 53 #include <linux/uaccess.h> 54 #include <linux/hash.h> 55 #include <linux/pci.h> 56 #include <linux/timekeeper_internal.h> 57 #include <linux/pvclock_gtod.h> 58 #include <linux/kvm_irqfd.h> 59 #include <linux/irqbypass.h> 60 #include <linux/sched/stat.h> 61 #include <linux/sched/isolation.h> 62 #include <linux/mem_encrypt.h> 63 #include <linux/suspend.h> 64 #include <linux/smp.h> 65 #include <linux/units.h> 66 67 #include <trace/events/ipi.h> 68 #include <trace/events/kvm.h> 69 70 #include <asm/debugreg.h> 71 #include <asm/msr.h> 72 #include <asm/desc.h> 73 #include <asm/mce.h> 74 #include <asm/pkru.h> 75 #include <linux/kernel_stat.h> 76 #include <asm/fpu/api.h> 77 #include <asm/fpu/xcr.h> 78 #include <asm/fpu/xstate.h> 79 #include <asm/pvclock.h> 80 #include <asm/div64.h> 81 #include <asm/irq_remapping.h> 82 #include <asm/mshyperv.h> 83 #include <asm/hypervisor.h> 84 #include <asm/tlbflush.h> 85 #include <asm/emulate_prefix.h> 86 #include <asm/sgx.h> 87 #include <asm/virt.h> 88 89 #include <clocksource/hyperv_timer.h> 90 91 #define CREATE_TRACE_POINTS 92 #include "trace.h" 93 94 /* 95 * Note, kvm_caps fields should *never* have default values, all fields must be 96 * recomputed from scratch during vendor module load, e.g. to account for a 97 * vendor module being reloaded with different module parameters. 98 */ 99 struct kvm_caps kvm_caps __read_mostly; 100 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_caps); 101 102 struct kvm_host_values kvm_host __read_mostly; 103 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_host); 104 105 #define ERR_PTR_USR(e) ((void __user *)ERR_PTR(e)) 106 107 #define emul_to_vcpu(ctxt) \ 108 ((struct kvm_vcpu *)(ctxt)->vcpu) 109 110 /* 111 * KVM previously used a u32 field in kvm_run to indicate the hypercall was 112 * initiated from long mode. KVM now sets bit 0 to indicate long mode, but the 113 * remaining 31 lower bits must be 0 to preserve ABI. 114 */ 115 #define KVM_EXIT_HYPERCALL_MBZ GENMASK_ULL(31, 1) 116 #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) 117 118 #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE 119 120 #define KVM_GUESTDBG_VALID_MASK \ 121 (KVM_GUESTDBG_ENABLE | \ 122 KVM_GUESTDBG_SINGLESTEP | \ 123 KVM_GUESTDBG_USE_HW_BP | \ 124 KVM_GUESTDBG_USE_SW_BP | \ 125 KVM_GUESTDBG_INJECT_BP | \ 126 KVM_GUESTDBG_INJECT_DB | \ 127 KVM_GUESTDBG_BLOCKIRQ) 128 129 #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ 130 KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK | \ 131 KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \ 132 KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) 133 134 #define KVM_CLOCK_VALID_FLAGS \ 135 (KVM_CLOCK_TSC_STABLE | KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC) 136 137 #define KVM_X86_VALID_QUIRKS \ 138 (KVM_X86_QUIRK_LINT0_REENABLED | \ 139 KVM_X86_QUIRK_CD_NW_CLEARED | \ 140 KVM_X86_QUIRK_LAPIC_MMIO_HOLE | \ 141 KVM_X86_QUIRK_OUT_7E_INC_RIP | \ 142 KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT | \ 143 KVM_X86_QUIRK_FIX_HYPERCALL_INSN | \ 144 KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS | \ 145 KVM_X86_QUIRK_SLOT_ZAP_ALL | \ 146 KVM_X86_QUIRK_STUFF_FEATURE_MSRS | \ 147 KVM_X86_QUIRK_IGNORE_GUEST_PAT | \ 148 KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM | \ 149 KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT) 150 151 #define KVM_X86_CONDITIONAL_QUIRKS \ 152 (KVM_X86_QUIRK_CD_NW_CLEARED | \ 153 KVM_X86_QUIRK_IGNORE_GUEST_PAT) 154 155 #define KVM_BUS_LOCK_DETECTION_VALID_MODE (KVM_BUS_LOCK_DETECTION_OFF | \ 156 KVM_BUS_LOCK_DETECTION_EXIT) 157 158 #define KVM_X86_NOTIFY_VMEXIT_VALID_BITS (KVM_X86_NOTIFY_VMEXIT_ENABLED | \ 159 KVM_X86_NOTIFY_VMEXIT_USER) 160 161 static void process_nmi(struct kvm_vcpu *vcpu); 162 static void store_regs(struct kvm_vcpu *vcpu); 163 static int sync_regs(struct kvm_vcpu *vcpu); 164 165 static DEFINE_MUTEX(vendor_module_lock); 166 167 struct kvm_x86_ops kvm_x86_ops __read_mostly; 168 struct kvm_x86_nested_ops kvm_nested_ops __read_mostly; 169 170 #define KVM_X86_OP(func) \ 171 DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ 172 *(((struct kvm_x86_ops *)0)->func)); 173 #define KVM_X86_OP_OPTIONAL KVM_X86_OP 174 #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP 175 #include <asm/kvm-x86-ops.h> 176 EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits); 177 EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg); 178 EXPORT_STATIC_CALL_GPL(kvm_x86_get_cpl); 179 180 #define KVM_X86_NESTED_OP(func) \ 181 DEFINE_STATIC_CALL_NULL(kvm_x86_nested_##func, \ 182 *(((struct kvm_x86_nested_ops *)0)->func)); 183 #define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP 184 #define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP 185 #include <asm/kvm-x86-nested-ops.h> 186 187 unsigned int min_timer_period_us = 200; 188 module_param(min_timer_period_us, uint, 0644); 189 190 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */ 191 static u32 __read_mostly tsc_tolerance_ppm = 250; 192 module_param(tsc_tolerance_ppm, uint, 0644); 193 194 bool __read_mostly enable_vmware_backdoor = false; 195 module_param(enable_vmware_backdoor, bool, 0444); 196 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_vmware_backdoor); 197 198 /* 199 * Flags to manipulate forced emulation behavior (any non-zero value will 200 * enable forced emulation). 201 */ 202 #define KVM_FEP_CLEAR_RFLAGS_RF BIT(1) 203 static int __read_mostly force_emulation_prefix; 204 module_param(force_emulation_prefix, int, 0644); 205 206 int __read_mostly pi_inject_timer = -1; 207 module_param(pi_inject_timer, bint, 0644); 208 209 /* Enable/disable SMT_RSB bug mitigation */ 210 static bool __read_mostly mitigate_smt_rsb; 211 module_param(mitigate_smt_rsb, bool, 0444); 212 213 #define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ 214 | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ 215 | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ 216 | XFEATURE_MASK_PKRU | XFEATURE_MASK_XTILE) 217 218 #define XFEATURE_MASK_CET_ALL (XFEATURE_MASK_CET_USER | XFEATURE_MASK_CET_KERNEL) 219 /* 220 * Note, KVM supports exposing PT to the guest, but does not support context 221 * switching PT via XSTATE (KVM's PT virtualization relies on perf; swapping 222 * PT via guest XSTATE would clobber perf state), i.e. KVM doesn't support 223 * IA32_XSS[bit 8] (guests can/must use RDMSR/WRMSR to save/restore PT MSRs). 224 */ 225 #define KVM_SUPPORTED_XSS (XFEATURE_MASK_CET_ALL) 226 227 bool __read_mostly allow_smaller_maxphyaddr = 0; 228 EXPORT_SYMBOL_FOR_KVM_INTERNAL(allow_smaller_maxphyaddr); 229 230 bool __read_mostly enable_apicv = true; 231 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_apicv); 232 233 bool __read_mostly enable_ipiv = true; 234 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_ipiv); 235 236 bool __read_mostly enable_device_posted_irqs = true; 237 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_device_posted_irqs); 238 239 const struct kvm_stats_desc kvm_vm_stats_desc[] = { 240 KVM_GENERIC_VM_STATS(), 241 STATS_DESC_COUNTER(VM, mmu_shadow_zapped), 242 STATS_DESC_COUNTER(VM, mmu_pte_write), 243 STATS_DESC_COUNTER(VM, mmu_pde_zapped), 244 STATS_DESC_COUNTER(VM, mmu_flooded), 245 STATS_DESC_COUNTER(VM, mmu_recycled), 246 STATS_DESC_COUNTER(VM, mmu_cache_miss), 247 STATS_DESC_ICOUNTER(VM, mmu_unsync), 248 STATS_DESC_ICOUNTER(VM, mmu_shadow_pages), 249 STATS_DESC_ICOUNTER(VM, pages_4k), 250 STATS_DESC_ICOUNTER(VM, pages_2m), 251 STATS_DESC_ICOUNTER(VM, pages_1g), 252 STATS_DESC_ICOUNTER(VM, nx_lpage_splits), 253 STATS_DESC_PCOUNTER(VM, max_mmu_rmap_size), 254 STATS_DESC_PCOUNTER(VM, max_mmu_page_hash_collisions) 255 }; 256 257 const struct kvm_stats_header kvm_vm_stats_header = { 258 .name_size = KVM_STATS_NAME_SIZE, 259 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), 260 .id_offset = sizeof(struct kvm_stats_header), 261 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 262 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 263 sizeof(kvm_vm_stats_desc), 264 }; 265 266 const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { 267 KVM_GENERIC_VCPU_STATS(), 268 STATS_DESC_COUNTER(VCPU, pf_taken), 269 STATS_DESC_COUNTER(VCPU, pf_fixed), 270 STATS_DESC_COUNTER(VCPU, pf_emulate), 271 STATS_DESC_COUNTER(VCPU, pf_spurious), 272 STATS_DESC_COUNTER(VCPU, pf_fast), 273 STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created), 274 STATS_DESC_COUNTER(VCPU, pf_guest), 275 STATS_DESC_COUNTER(VCPU, tlb_flush), 276 STATS_DESC_COUNTER(VCPU, invlpg), 277 STATS_DESC_COUNTER(VCPU, exits), 278 STATS_DESC_COUNTER(VCPU, io_exits), 279 STATS_DESC_COUNTER(VCPU, mmio_exits), 280 STATS_DESC_COUNTER(VCPU, signal_exits), 281 STATS_DESC_COUNTER(VCPU, irq_window_exits), 282 STATS_DESC_COUNTER(VCPU, nmi_window_exits), 283 STATS_DESC_COUNTER(VCPU, l1d_flush), 284 STATS_DESC_COUNTER(VCPU, halt_exits), 285 STATS_DESC_COUNTER(VCPU, request_irq_exits), 286 STATS_DESC_COUNTER(VCPU, irq_exits), 287 STATS_DESC_COUNTER(VCPU, host_state_reload), 288 STATS_DESC_COUNTER(VCPU, fpu_reload), 289 STATS_DESC_COUNTER(VCPU, insn_emulation), 290 STATS_DESC_COUNTER(VCPU, insn_emulation_fail), 291 STATS_DESC_COUNTER(VCPU, hypercalls), 292 STATS_DESC_COUNTER(VCPU, irq_injections), 293 STATS_DESC_COUNTER(VCPU, nmi_injections), 294 STATS_DESC_COUNTER(VCPU, req_event), 295 STATS_DESC_COUNTER(VCPU, nested_run), 296 STATS_DESC_COUNTER(VCPU, directed_yield_attempted), 297 STATS_DESC_COUNTER(VCPU, directed_yield_successful), 298 STATS_DESC_COUNTER(VCPU, preemption_reported), 299 STATS_DESC_COUNTER(VCPU, preemption_other), 300 STATS_DESC_IBOOLEAN(VCPU, guest_mode), 301 STATS_DESC_COUNTER(VCPU, notify_window_exits), 302 }; 303 304 const struct kvm_stats_header kvm_vcpu_stats_header = { 305 .name_size = KVM_STATS_NAME_SIZE, 306 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 307 .id_offset = sizeof(struct kvm_stats_header), 308 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 309 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 310 sizeof(kvm_vcpu_stats_desc), 311 }; 312 313 static struct kmem_cache *x86_emulator_cache; 314 315 static struct kmem_cache *kvm_alloc_emulator_cache(void) 316 { 317 unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src); 318 unsigned int size = sizeof(struct x86_emulate_ctxt); 319 320 return kmem_cache_create_usercopy("x86_emulator", size, 321 __alignof__(struct x86_emulate_ctxt), 322 SLAB_ACCOUNT, useroffset, 323 size - useroffset, NULL); 324 } 325 326 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt); 327 328 /* 329 * Handle a fault on a hardware virtualization (VMX or SVM) instruction. 330 * 331 * Hardware virtualization extension instructions may fault if a reboot turns 332 * off virtualization while processes are running. Usually after catching the 333 * fault we just panic; during reboot instead the instruction is ignored. 334 */ 335 noinstr void kvm_spurious_fault(void) 336 { 337 /* Fault while not rebooting. We want the trace. */ 338 BUG_ON(!virt_rebooting); 339 } 340 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_spurious_fault); 341 342 #define EXCPT_BENIGN 0 343 #define EXCPT_CONTRIBUTORY 1 344 #define EXCPT_PF 2 345 346 static int exception_class(int vector) 347 { 348 switch (vector) { 349 case PF_VECTOR: 350 return EXCPT_PF; 351 case DE_VECTOR: 352 case TS_VECTOR: 353 case NP_VECTOR: 354 case SS_VECTOR: 355 case GP_VECTOR: 356 return EXCPT_CONTRIBUTORY; 357 default: 358 break; 359 } 360 return EXCPT_BENIGN; 361 } 362 363 #define EXCPT_FAULT 0 364 #define EXCPT_TRAP 1 365 #define EXCPT_ABORT 2 366 #define EXCPT_INTERRUPT 3 367 #define EXCPT_DB 4 368 369 static int exception_type(int vector) 370 { 371 unsigned int mask; 372 373 if (WARN_ON(vector > 31 || vector == NMI_VECTOR)) 374 return EXCPT_INTERRUPT; 375 376 mask = 1 << vector; 377 378 /* 379 * #DBs can be trap-like or fault-like, the caller must check other CPU 380 * state, e.g. DR6, to determine whether a #DB is a trap or fault. 381 */ 382 if (mask & (1 << DB_VECTOR)) 383 return EXCPT_DB; 384 385 if (mask & ((1 << BP_VECTOR) | (1 << OF_VECTOR))) 386 return EXCPT_TRAP; 387 388 if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR))) 389 return EXCPT_ABORT; 390 391 /* Reserved exceptions will result in fault */ 392 return EXCPT_FAULT; 393 } 394 395 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, 396 struct kvm_queued_exception *ex) 397 { 398 if (!ex->has_payload) 399 return; 400 401 switch (ex->vector) { 402 case DB_VECTOR: 403 /* 404 * DR6 is a mess. Reserved/unused bits are fixed-to-1, and so 405 * to maintain backwards compatibility with existing software, 406 * features that use previously-reserved bits have active-low 407 * semantics, i.e. clear the bit when the feature is present in 408 * the payload. 409 * 410 * Further complicating matters, some DR6 bits are preserved by 411 * hardware, while others are explicitly modified on every #DB. 412 * The trap bits are always set based on the payload, as is the 413 * RTM flag (but it's active low). All other bits are modified 414 * if and only if a relevant debug exception occurs, e.g. BD, 415 * BS, and BT are never cleared by hardware, and BLD is never 416 * set by hardware (when supported, excepting RESET). 417 * 418 * Lastly, the payload does NOT have active-low semantics, e.g. 419 * so that it's compatible VMX's pending debug exceptions and 420 * qualification fields, and to avoid bleeding the DR6 madness 421 * into other KVM code. 422 * 423 * To compute DR6: 424 * 425 * 1. "Reset" the bits that are modified on all #DBs 426 * 2. Clear active-low bits that are present in the payload. 427 * 3. Set active-high bits that are present in the payload. 428 * 4. Clear fixed-0 bits. 429 * 5. Set fixed-1 bits. 430 */ 431 vcpu->arch.dr6 &= ~DR_TRAP_BITS; 432 vcpu->arch.dr6 |= DR6_RTM; 433 vcpu->arch.dr6 &= ~(ex->payload & DR6_ACTIVE_LOW); 434 vcpu->arch.dr6 |= (ex->payload & ~DR6_ACTIVE_LOW); 435 436 /* 437 * The #DB payload is defined as compatible with the 'pending 438 * debug exceptions' field under VMX, not DR6. While bit 12 is 439 * defined in the 'pending debug exceptions' field (enabled 440 * breakpoint), it is reserved and must be zero in DR6. 441 */ 442 vcpu->arch.dr6 &= ~BIT(12); 443 vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); 444 break; 445 case PF_VECTOR: 446 vcpu->arch.cr2 = ex->payload; 447 break; 448 } 449 450 ex->has_payload = false; 451 ex->payload = 0; 452 } 453 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_deliver_exception_payload); 454 455 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector, 456 bool has_error_code, u32 error_code, 457 bool has_payload, unsigned long payload) 458 { 459 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 460 461 kvm_make_request(KVM_REQ_EVENT, vcpu); 462 463 ex->vector = vector; 464 ex->injected = false; 465 ex->pending = true; 466 ex->has_error_code = has_error_code; 467 ex->error_code = error_code; 468 ex->has_payload = has_payload; 469 ex->payload = payload; 470 } 471 472 static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr, 473 bool has_error, u32 error_code, 474 bool has_payload, unsigned long payload) 475 { 476 u32 prev_nr; 477 int class1, class2; 478 479 kvm_make_request(KVM_REQ_EVENT, vcpu); 480 481 /* 482 * If the exception is destined for L2, morph it to a VM-Exit if L1 483 * wants to intercept the exception. 484 */ 485 if (is_guest_mode(vcpu) && 486 kvm_nested_call(is_exception_vmexit)(vcpu, nr, error_code)) { 487 kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code, 488 has_payload, payload); 489 return; 490 } 491 492 if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) { 493 queue: 494 vcpu->arch.exception.pending = true; 495 vcpu->arch.exception.injected = false; 496 497 vcpu->arch.exception.has_error_code = has_error; 498 vcpu->arch.exception.vector = nr; 499 vcpu->arch.exception.error_code = error_code; 500 vcpu->arch.exception.has_payload = has_payload; 501 vcpu->arch.exception.payload = payload; 502 return; 503 } 504 505 /* to check exception */ 506 prev_nr = vcpu->arch.exception.vector; 507 if (prev_nr == DF_VECTOR) { 508 /* triple fault -> shutdown */ 509 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 510 return; 511 } 512 class1 = exception_class(prev_nr); 513 class2 = exception_class(nr); 514 if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) || 515 (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) { 516 /* 517 * Synthesize #DF. Clear the previously injected or pending 518 * exception so as not to incorrectly trigger shutdown. 519 */ 520 vcpu->arch.exception.injected = false; 521 vcpu->arch.exception.pending = false; 522 523 kvm_queue_exception_e(vcpu, DF_VECTOR, 0); 524 } else { 525 /* replace previous exception with a new one in a hope 526 that instruction re-execution will regenerate lost 527 exception */ 528 goto queue; 529 } 530 } 531 532 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) 533 { 534 kvm_multiple_exception(vcpu, nr, false, 0, false, 0); 535 } 536 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_queue_exception); 537 538 539 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, 540 unsigned long payload) 541 { 542 kvm_multiple_exception(vcpu, nr, false, 0, true, payload); 543 } 544 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_queue_exception_p); 545 546 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr, 547 u32 error_code, unsigned long payload) 548 { 549 kvm_multiple_exception(vcpu, nr, true, error_code, true, payload); 550 } 551 552 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr, 553 bool has_error_code, u32 error_code) 554 { 555 556 /* 557 * On VM-Entry, an exception can be pending if and only if event 558 * injection was blocked by nested_run_pending. In that case, however, 559 * vcpu_enter_guest() requests an immediate exit, and the guest 560 * shouldn't proceed far enough to need reinjection. 561 */ 562 WARN_ON_ONCE(kvm_is_exception_pending(vcpu)); 563 564 /* 565 * Do not check for interception when injecting an event for L2, as the 566 * exception was checked for intercept when it was original queued, and 567 * re-checking is incorrect if _L1_ injected the exception, in which 568 * case it's exempt from interception. 569 */ 570 kvm_make_request(KVM_REQ_EVENT, vcpu); 571 572 vcpu->arch.exception.injected = true; 573 vcpu->arch.exception.has_error_code = has_error_code; 574 vcpu->arch.exception.vector = nr; 575 vcpu->arch.exception.error_code = error_code; 576 vcpu->arch.exception.has_payload = false; 577 vcpu->arch.exception.payload = 0; 578 } 579 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_requeue_exception); 580 581 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) 582 { 583 if (err) 584 kvm_inject_gp(vcpu, 0); 585 else 586 return kvm_skip_emulated_instruction(vcpu); 587 588 return 1; 589 } 590 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_complete_insn_gp); 591 592 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault, 593 bool from_hardware) 594 { 595 ++vcpu->stat.pf_guest; 596 597 /* 598 * Async #PF in L2 is always forwarded to L1 as a VM-Exit regardless of 599 * whether or not L1 wants to intercept "regular" #PF. 600 */ 601 if (is_guest_mode(vcpu) && fault->async_page_fault) 602 kvm_queue_exception_vmexit(vcpu, PF_VECTOR, 603 true, fault->error_code, 604 true, fault->address); 605 else 606 kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code, 607 fault->address); 608 } 609 610 void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, 611 struct x86_exception *fault, 612 bool from_hardware) 613 { 614 struct kvm_pagewalk *fault_walk; 615 616 WARN_ON_ONCE(fault->vector != PF_VECTOR); 617 618 fault_walk = fault->nested_page_fault ? &vcpu->arch.ngpa_walk : 619 &vcpu->arch.gva_walk; 620 621 /* 622 * Invalidate the TLB entry for the faulting address, if it exists, 623 * else the access will fault indefinitely (and to emulate hardware). 624 */ 625 if ((fault->error_code & PFERR_PRESENT_MASK) && 626 !(fault->error_code & PFERR_RSVD_MASK)) 627 kvm_mmu_invalidate_addr(vcpu, fault_walk, fault->address, 628 KVM_MMU_ROOT_CURRENT); 629 630 fault_walk->inject_page_fault(vcpu, fault, from_hardware); 631 } 632 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_inject_emulated_page_fault); 633 634 void kvm_inject_nmi(struct kvm_vcpu *vcpu) 635 { 636 atomic_inc(&vcpu->arch.nmi_queued); 637 kvm_make_request(KVM_REQ_NMI, vcpu); 638 } 639 640 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) 641 { 642 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0); 643 } 644 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_queue_exception_e); 645 646 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) 647 { 648 if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE)) 649 return true; 650 651 kvm_queue_exception(vcpu, UD_VECTOR); 652 return false; 653 } 654 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr); 655 656 static void kvm_load_xfeatures(struct kvm_vcpu *vcpu, bool load_guest) 657 { 658 if (vcpu->arch.guest_state_protected) 659 return; 660 661 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) 662 return; 663 664 if (vcpu->arch.xcr0 != kvm_host.xcr0) 665 xsetbv(XCR_XFEATURE_ENABLED_MASK, 666 load_guest ? vcpu->arch.xcr0 : kvm_host.xcr0); 667 668 if (guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) && 669 vcpu->arch.ia32_xss != kvm_host.xss) 670 wrmsrq(MSR_IA32_XSS, load_guest ? vcpu->arch.ia32_xss : kvm_host.xss); 671 } 672 673 static void kvm_load_guest_pkru(struct kvm_vcpu *vcpu) 674 { 675 if (vcpu->arch.guest_state_protected) 676 return; 677 678 if (cpu_feature_enabled(X86_FEATURE_PKU) && 679 vcpu->arch.pkru != vcpu->arch.host_pkru && 680 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || 681 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) 682 wrpkru(vcpu->arch.pkru); 683 } 684 685 static void kvm_load_host_pkru(struct kvm_vcpu *vcpu) 686 { 687 if (vcpu->arch.guest_state_protected) 688 return; 689 690 if (cpu_feature_enabled(X86_FEATURE_PKU) && 691 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || 692 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) { 693 vcpu->arch.pkru = rdpkru(); 694 if (vcpu->arch.pkru != vcpu->arch.host_pkru) 695 wrpkru(vcpu->arch.host_pkru); 696 } 697 } 698 699 int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) 700 { 701 u64 xcr0 = xcr; 702 u64 old_xcr0 = vcpu->arch.xcr0; 703 u64 valid_bits; 704 705 /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now */ 706 if (index != XCR_XFEATURE_ENABLED_MASK) 707 return 1; 708 if (!(xcr0 & XFEATURE_MASK_FP)) 709 return 1; 710 if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE)) 711 return 1; 712 713 /* 714 * Do not allow the guest to set bits that we do not support 715 * saving. However, xcr0 bit 0 is always set, even if the 716 * emulated CPU does not support XSAVE (see kvm_vcpu_reset()). 717 */ 718 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP; 719 if (xcr0 & ~valid_bits) 720 return 1; 721 722 if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) != 723 (!(xcr0 & XFEATURE_MASK_BNDCSR))) 724 return 1; 725 726 if (xcr0 & XFEATURE_MASK_AVX512) { 727 if (!(xcr0 & XFEATURE_MASK_YMM)) 728 return 1; 729 if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512) 730 return 1; 731 } 732 733 if ((xcr0 & XFEATURE_MASK_XTILE) && 734 ((xcr0 & XFEATURE_MASK_XTILE) != XFEATURE_MASK_XTILE)) 735 return 1; 736 737 vcpu->arch.xcr0 = xcr0; 738 739 if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND) 740 vcpu->arch.cpuid_dynamic_bits_dirty = true; 741 return 0; 742 } 743 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_set_xcr); 744 745 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu) 746 { 747 /* Note, #UD due to CR4.OSXSAVE=0 has priority over the intercept. */ 748 if (kvm_x86_call(get_cpl)(vcpu) != 0 || 749 __kvm_set_xcr(vcpu, kvm_ecx_read(vcpu), kvm_read_edx_eax(vcpu))) { 750 kvm_inject_gp(vcpu, 0); 751 return 1; 752 } 753 754 return kvm_skip_emulated_instruction(vcpu); 755 } 756 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_xsetbv); 757 758 void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) 759 { 760 struct kvm_mmu *mmu = vcpu->arch.mmu; 761 unsigned long roots_to_free = 0; 762 int i; 763 764 /* 765 * MOV CR3 and INVPCID are usually not intercepted when using TDP, but 766 * this is reachable when running EPT=1 and unrestricted_guest=0, and 767 * also via the emulator. KVM's TDP page tables are not in the scope of 768 * the invalidation, but the guest's TLB entries need to be flushed as 769 * the CPU may have cached entries in its TLB for the target PCID. 770 */ 771 if (unlikely(tdp_enabled)) { 772 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 773 return; 774 } 775 776 /* 777 * If neither the current CR3 nor any of the prev_roots use the given 778 * PCID, then nothing needs to be done here because a resync will 779 * happen anyway before switching to any other CR3. 780 */ 781 if (kvm_get_active_pcid(vcpu) == pcid) { 782 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 783 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 784 } 785 786 /* 787 * If PCID is disabled, there is no need to free prev_roots even if the 788 * PCIDs for them are also 0, because MOV to CR3 always flushes the TLB 789 * with PCIDE=0. 790 */ 791 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) 792 return; 793 794 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 795 if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid) 796 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 797 798 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); 799 } 800 801 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) 802 { 803 u32 pmc = kvm_ecx_read(vcpu); 804 u64 data; 805 806 if (kvm_pmu_rdpmc(vcpu, pmc, &data)) { 807 kvm_inject_gp(vcpu, 0); 808 return 1; 809 } 810 811 kvm_eax_write(vcpu, data); 812 kvm_edx_write(vcpu, data >> 32); 813 return kvm_skip_emulated_instruction(vcpu); 814 } 815 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdpmc); 816 817 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu) 818 { 819 return kvm_skip_emulated_instruction(vcpu); 820 } 821 822 int kvm_emulate_invd(struct kvm_vcpu *vcpu) 823 { 824 /* Treat an INVD instruction as a NOP and just skip it. */ 825 return kvm_emulate_as_nop(vcpu); 826 } 827 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_invd); 828 829 fastpath_t handle_fastpath_invd(struct kvm_vcpu *vcpu) 830 { 831 if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu)) 832 return EXIT_FASTPATH_NONE; 833 834 if (!kvm_emulate_invd(vcpu)) 835 return EXIT_FASTPATH_EXIT_USERSPACE; 836 837 return EXIT_FASTPATH_REENTER_GUEST; 838 } 839 EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_invd); 840 841 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu) 842 { 843 kvm_queue_exception(vcpu, UD_VECTOR); 844 return 1; 845 } 846 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_invalid_op); 847 848 849 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn) 850 { 851 bool enabled; 852 853 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS)) 854 goto emulate_as_nop; 855 856 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT)) 857 enabled = guest_cpu_cap_has(vcpu, X86_FEATURE_MWAIT); 858 else 859 enabled = vcpu->arch.ia32_misc_enable_msr & MSR_IA32_MISC_ENABLE_MWAIT; 860 861 if (!enabled) 862 return kvm_handle_invalid_op(vcpu); 863 864 emulate_as_nop: 865 pr_warn_once("%s instruction emulated as NOP!\n", insn); 866 return kvm_emulate_as_nop(vcpu); 867 } 868 int kvm_emulate_mwait(struct kvm_vcpu *vcpu) 869 { 870 return kvm_emulate_monitor_mwait(vcpu, "MWAIT"); 871 } 872 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_mwait); 873 874 int kvm_emulate_monitor(struct kvm_vcpu *vcpu) 875 { 876 return kvm_emulate_monitor_mwait(vcpu, "MONITOR"); 877 } 878 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_monitor); 879 880 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu) 881 { 882 xfer_to_guest_mode_prepare(); 883 884 return READ_ONCE(vcpu->mode) == EXITING_GUEST_MODE || 885 kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending(); 886 } 887 888 #ifdef CONFIG_X86_64 889 struct pvclock_clock { 890 int vclock_mode; 891 u64 cycle_last; 892 u64 mask; 893 u32 mult; 894 u32 shift; 895 u64 base_cycles; 896 u64 offset; 897 }; 898 899 struct pvclock_gtod_data { 900 seqcount_t seq; 901 902 struct pvclock_clock clock; /* extract of a clocksource struct */ 903 struct pvclock_clock raw_clock; /* extract of a clocksource struct */ 904 905 ktime_t offs_boot; 906 u64 wall_time_sec; 907 }; 908 909 static struct pvclock_gtod_data pvclock_gtod_data; 910 911 static void update_pvclock_gtod(struct timekeeper *tk) 912 { 913 struct pvclock_gtod_data *vdata = &pvclock_gtod_data; 914 915 write_seqcount_begin(&vdata->seq); 916 917 /* copy pvclock gtod data */ 918 vdata->clock.vclock_mode = tk->tkr_mono.clock->vdso_clock_mode; 919 vdata->clock.cycle_last = tk->tkr_mono.cycle_last; 920 vdata->clock.mask = tk->tkr_mono.mask; 921 vdata->clock.mult = tk->tkr_mono.mult; 922 vdata->clock.shift = tk->tkr_mono.shift; 923 vdata->clock.base_cycles = tk->tkr_mono.xtime_nsec; 924 vdata->clock.offset = tk->tkr_mono.base; 925 926 vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->vdso_clock_mode; 927 vdata->raw_clock.cycle_last = tk->tkr_raw.cycle_last; 928 vdata->raw_clock.mask = tk->tkr_raw.mask; 929 vdata->raw_clock.mult = tk->tkr_raw.mult; 930 vdata->raw_clock.shift = tk->tkr_raw.shift; 931 vdata->raw_clock.base_cycles = tk->tkr_raw.xtime_nsec; 932 vdata->raw_clock.offset = tk->tkr_raw.base; 933 934 vdata->wall_time_sec = tk->xtime_sec; 935 936 vdata->offs_boot = tk->offs_boot; 937 938 write_seqcount_end(&vdata->seq); 939 } 940 #endif 941 942 static s64 get_kvmclock_base_ns(void) 943 { 944 /* Count up from boot time, but with the frequency of the raw clock. */ 945 return ktime_to_ns(ktime_mono_to_any(ktime_get_raw(), TK_OFFS_BOOT)); 946 } 947 948 static uint32_t div_frac(uint32_t dividend, uint32_t divisor) 949 { 950 do_shl32_div32(dividend, divisor); 951 return dividend; 952 } 953 954 static void kvm_get_time_scale(u64 scaled_hz, u64 base_hz, 955 s8 *pshift, u32 *pmultiplier) 956 { 957 u64 scaled_hz_u64 = scaled_hz; 958 s32 shift = 0; 959 u64 base_hz_u64; 960 u32 base32; 961 962 /* 963 * This function calculates a fixed-point multiplier and shift such 964 * that: 965 * time_ns = (tsc_cycles << shift) * multiplier >> 32 966 * 967 * Where tsc_cycles tick at base_hz, and time_ns should count at 968 * scaled_hz (typically NSEC_PER_SEC for a TSC→nanoseconds conversion). 969 * 970 * The multiplier is: (scaled_hz << 32) / base_hz, adjusted by shift 971 * to keep everything in range. 972 */ 973 974 base_hz_u64 = base_hz; 975 976 /* 977 * Start by shifting base_hz right until it fits in 32 bits, and 978 * is lower than double the target rate. This introduces a negative 979 * shift value which would result in pvclock_scale_delta() shifting 980 * the actual tick count right before performing the multiplication. 981 */ 982 while (base_hz_u64 > scaled_hz_u64 * 2 || base_hz_u64 >> 32) { 983 base_hz_u64 >>= 1; 984 shift--; 985 } 986 987 /* Now the shifted base_hz fits in 32 bits. */ 988 base32 = (u32)base_hz_u64; 989 990 /* 991 * Next, shift scaled_hz right until it fits in 32 bits, and ensure 992 * that the shifted base_hz is strictly larger (so that the result of the 993 * final division also fits in 32 bits). 994 */ 995 while (base32 <= scaled_hz_u64 || scaled_hz_u64 >> 32) { 996 if (scaled_hz_u64 >> 32 || base32 & BIT(31)) 997 scaled_hz_u64 >>= 1; 998 else 999 base32 <<= 1; 1000 shift++; 1001 } 1002 1003 *pshift = shift; 1004 *pmultiplier = div_frac(scaled_hz_u64, base32); 1005 } 1006 1007 #ifdef CONFIG_X86_64 1008 static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0); 1009 #endif 1010 1011 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz); 1012 static unsigned long max_tsc_khz; 1013 1014 static u32 adjust_tsc_khz(u32 khz, s32 ppm) 1015 { 1016 u64 v = (u64)khz * (1000000 + ppm); 1017 do_div(v, 1000000); 1018 return v; 1019 } 1020 1021 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier); 1022 1023 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale) 1024 { 1025 u64 ratio; 1026 1027 /* Guest TSC same frequency as host TSC? */ 1028 if (!scale) { 1029 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio); 1030 return 0; 1031 } 1032 1033 /* TSC scaling supported? */ 1034 if (!kvm_caps.has_tsc_control) { 1035 if (user_tsc_khz > tsc_khz) { 1036 vcpu->arch.tsc_catchup = 1; 1037 vcpu->arch.tsc_always_catchup = 1; 1038 return 0; 1039 } else { 1040 pr_warn_ratelimited("user requested TSC rate below hardware speed\n"); 1041 return -1; 1042 } 1043 } 1044 1045 /* TSC scaling required - calculate ratio */ 1046 ratio = mul_u64_u32_div(1ULL << kvm_caps.tsc_scaling_ratio_frac_bits, 1047 user_tsc_khz, tsc_khz); 1048 1049 if (ratio == 0 || ratio >= kvm_caps.max_tsc_scaling_ratio) { 1050 pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n", 1051 user_tsc_khz); 1052 return -1; 1053 } 1054 1055 kvm_vcpu_write_tsc_multiplier(vcpu, ratio); 1056 return 0; 1057 } 1058 1059 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz) 1060 { 1061 u32 thresh_lo, thresh_hi; 1062 int use_scaling = 0; 1063 1064 /* tsc_khz can be zero if TSC calibration fails */ 1065 if (user_tsc_khz == 0) { 1066 /* set tsc_scaling_ratio to a safe value */ 1067 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio); 1068 return -1; 1069 } 1070 1071 /* Compute a scale to convert nanoseconds in TSC cycles */ 1072 kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC, 1073 &vcpu->arch.virtual_tsc_shift, 1074 &vcpu->arch.virtual_tsc_mult); 1075 vcpu->arch.virtual_tsc_khz = user_tsc_khz; 1076 1077 /* 1078 * Compute the variation in TSC rate which is acceptable 1079 * within the range of tolerance and decide if the 1080 * rate being applied is within that bounds of the hardware 1081 * rate. If so, no scaling or compensation need be done. 1082 */ 1083 thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm); 1084 thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm); 1085 if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) { 1086 pr_debug("requested TSC rate %u falls outside tolerance [%u,%u]\n", 1087 user_tsc_khz, thresh_lo, thresh_hi); 1088 use_scaling = 1; 1089 } 1090 return set_tsc_khz(vcpu, user_tsc_khz, use_scaling); 1091 } 1092 1093 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) 1094 { 1095 s64 delta_ns = kernel_ns - vcpu->arch.this_tsc_nsec; 1096 u64 tsc; 1097 1098 /* Handle negative deltas gracefully (master clock ref may be earlier) */ 1099 tsc = pvclock_scale_delta(abs(delta_ns), 1100 vcpu->arch.virtual_tsc_mult, 1101 vcpu->arch.virtual_tsc_shift); 1102 1103 return vcpu->arch.this_tsc_write + (delta_ns >= 0 ? tsc : -tsc); 1104 } 1105 1106 #ifdef CONFIG_X86_64 1107 static inline bool gtod_is_based_on_tsc(int mode) 1108 { 1109 return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK; 1110 } 1111 #endif 1112 1113 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation) 1114 { 1115 #ifdef CONFIG_X86_64 1116 struct kvm_arch *ka = &vcpu->kvm->arch; 1117 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 1118 1119 /* 1120 * To use the masterclock, the host clocksource must be based on TSC 1121 * and all vCPUs must have matching TSCs. Note, the count for matching 1122 * vCPUs doesn't include the reference vCPU, hence "+1". 1123 */ 1124 bool use_master_clock = (ka->nr_vcpus_matched_tsc + 1 == 1125 atomic_read(&vcpu->kvm->online_vcpus)) && 1126 gtod_is_based_on_tsc(gtod->clock.vclock_mode); 1127 1128 /* 1129 * Request a masterclock update if the masterclock needs to be toggled 1130 * on/off, or when starting a new generation and the masterclock is 1131 * enabled (compute_guest_tsc() requires the masterclock snapshot to be 1132 * taken _after_ the new generation is created). 1133 */ 1134 if ((ka->use_master_clock && new_generation) || 1135 (ka->use_master_clock != use_master_clock)) 1136 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 1137 1138 trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc, 1139 atomic_read(&vcpu->kvm->online_vcpus), 1140 ka->use_master_clock, gtod->clock.vclock_mode); 1141 #endif 1142 } 1143 1144 /* 1145 * Multiply tsc by a fixed point number represented by ratio. 1146 * 1147 * The most significant 64-N bits (mult) of ratio represent the 1148 * integral part of the fixed point number; the remaining N bits 1149 * (frac) represent the fractional part, ie. ratio represents a fixed 1150 * point number (mult + frac * 2^(-N)). 1151 * 1152 * N equals to kvm_caps.tsc_scaling_ratio_frac_bits. 1153 */ 1154 static inline u64 __scale_tsc(u64 ratio, u64 tsc) 1155 { 1156 return mul_u64_u64_shr(tsc, ratio, kvm_caps.tsc_scaling_ratio_frac_bits); 1157 } 1158 1159 u64 kvm_scale_tsc(u64 tsc, u64 ratio) 1160 { 1161 u64 _tsc = tsc; 1162 1163 if (ratio != kvm_caps.default_tsc_scaling_ratio) 1164 _tsc = __scale_tsc(ratio, tsc); 1165 1166 return _tsc; 1167 } 1168 1169 u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, 1170 u64 target_tsc) 1171 { 1172 u64 tsc; 1173 1174 tsc = kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); 1175 1176 return target_tsc - tsc; 1177 } 1178 1179 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) 1180 { 1181 return vcpu->arch.l1_tsc_offset + 1182 kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); 1183 } 1184 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_l1_tsc); 1185 1186 u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier) 1187 { 1188 u64 nested_offset; 1189 1190 if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio) 1191 nested_offset = l1_offset; 1192 else 1193 nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier, 1194 kvm_caps.tsc_scaling_ratio_frac_bits); 1195 1196 nested_offset += l2_offset; 1197 return nested_offset; 1198 } 1199 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_offset); 1200 1201 u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) 1202 { 1203 u8 frac_bits = kvm_caps.tsc_scaling_ratio_frac_bits; 1204 u64 nested_multiplier; 1205 1206 if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio) 1207 return l1_multiplier; 1208 1209 /* 1210 * The shift is fixed on both AMD and Intel, and operates on a 64-bit 1211 * value. I.e. a shift greater than 63 is completely nonsensical. 1212 */ 1213 if (WARN_ON_ONCE(frac_bits > 63)) 1214 return l1_multiplier; 1215 1216 /* 1217 * If the resulting multiplier can't be programmed into hardware, run 1218 * L2 at the minimum/maximum frequency supported by hardware, i.e. 1219 * saturate L2's frequency on both sides. Because L2's frequency needs 1220 * to be distilled down to a single multiplier to get from: 1221 * 1222 * L2 = (((L0 * L1_mult) >> frac) * L2_mult) >> frac) 1223 * 1224 * to: 1225 * 1226 * L2 = (L0 * mult) >> frac 1227 * 1228 * very small/large L1 and L2 multipliers can underflow/overflow the 1229 * minimum/maximum multiplier supported by hardware when combined into 1230 * a single value. 1231 * 1232 * Manually check for the case where the result would overflow a u64, 1233 * i.e. if the multiplier would be silently truncated before the "too 1234 * large" check. Avoid doing the multiply twice in the common case 1235 * where the compiler natively supports 128-bit values. 1236 */ 1237 #ifdef CONFIG_ARCH_SUPPORTS_INT128 1238 unsigned __int128 m = (unsigned __int128)l1_multiplier * l2_multiplier; 1239 1240 if (m >> (64 + frac_bits)) 1241 return kvm_caps.max_tsc_scaling_ratio; 1242 1243 nested_multiplier = m >> frac_bits; 1244 #else 1245 if (mul_u64_u64_shr(l1_multiplier, l2_multiplier, 64 + frac_bits)) 1246 return kvm_caps.max_tsc_scaling_ratio; 1247 1248 nested_multiplier = mul_u64_u64_shr(l1_multiplier, l2_multiplier, frac_bits); 1249 #endif 1250 if (nested_multiplier > kvm_caps.max_tsc_scaling_ratio) 1251 return kvm_caps.max_tsc_scaling_ratio; 1252 1253 /* The minimum multiplier is '1' on both AMD and Intel. */ 1254 return nested_multiplier ?: 1; 1255 } 1256 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_multiplier); 1257 1258 void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset) 1259 { 1260 if (vcpu->arch.guest_tsc_protected) 1261 return; 1262 1263 trace_kvm_write_tsc_offset(vcpu->vcpu_id, 1264 vcpu->arch.l1_tsc_offset, 1265 l1_offset); 1266 1267 vcpu->arch.l1_tsc_offset = l1_offset; 1268 1269 /* 1270 * If we are here because L1 chose not to trap WRMSR to TSC then 1271 * according to the spec this should set L1's TSC (as opposed to 1272 * setting L1's offset for L2). 1273 */ 1274 if (is_guest_mode(vcpu)) 1275 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 1276 l1_offset, 1277 kvm_x86_call(get_l2_tsc_offset)(vcpu), 1278 kvm_x86_call(get_l2_tsc_multiplier)(vcpu)); 1279 else 1280 vcpu->arch.tsc_offset = l1_offset; 1281 1282 kvm_x86_call(write_tsc_offset)(vcpu); 1283 } 1284 1285 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier) 1286 { 1287 vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier; 1288 1289 /* Userspace is changing the multiplier while L2 is active */ 1290 if (is_guest_mode(vcpu)) 1291 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( 1292 l1_multiplier, 1293 kvm_x86_call(get_l2_tsc_multiplier)(vcpu)); 1294 else 1295 vcpu->arch.tsc_scaling_ratio = l1_multiplier; 1296 1297 if (kvm_caps.has_tsc_control) 1298 kvm_x86_call(write_tsc_multiplier)(vcpu); 1299 } 1300 1301 static inline bool kvm_check_tsc_unstable(void) 1302 { 1303 #ifdef CONFIG_X86_64 1304 /* 1305 * TSC is marked unstable when we're running on Hyper-V, 1306 * 'TSC page' clocksource is good. 1307 */ 1308 if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK) 1309 return false; 1310 #endif 1311 return check_tsc_unstable(); 1312 } 1313 1314 /* 1315 * Infers attempts to synchronize the guest's tsc from host writes. Sets the 1316 * offset for the vcpu and tracks the TSC matching generation that the vcpu 1317 * participates in. 1318 */ 1319 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, 1320 u64 ns, bool matched, bool user_set_tsc) 1321 { 1322 struct kvm *kvm = vcpu->kvm; 1323 1324 lockdep_assert_held(&kvm->arch.tsc_write_lock); 1325 1326 if (vcpu->arch.guest_tsc_protected) 1327 return; 1328 1329 if (user_set_tsc) 1330 vcpu->kvm->arch.user_set_tsc = true; 1331 1332 /* 1333 * We also track th most recent recorded KHZ, write and time to 1334 * allow the matching interval to be extended at each write. 1335 */ 1336 kvm->arch.last_tsc_nsec = ns; 1337 kvm->arch.last_tsc_write = tsc; 1338 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; 1339 kvm->arch.last_tsc_offset = offset; 1340 kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 1341 1342 vcpu->arch.last_guest_tsc = tsc; 1343 1344 kvm_vcpu_write_tsc_offset(vcpu, offset); 1345 1346 if (!matched) { 1347 /* 1348 * We split periods of matched TSC writes into generations. 1349 * For each generation, we track the original measured 1350 * nanosecond time, offset, and write, so if TSCs are in 1351 * sync, we can match exact offset, and if not, we can match 1352 * exact software computation in compute_guest_tsc() 1353 * 1354 * These values are tracked in kvm->arch.cur_xxx variables. 1355 */ 1356 kvm->arch.cur_tsc_generation++; 1357 kvm->arch.cur_tsc_nsec = ns; 1358 kvm->arch.cur_tsc_write = tsc; 1359 kvm->arch.cur_tsc_offset = offset; 1360 kvm->arch.nr_vcpus_matched_tsc = 0; 1361 } else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) { 1362 kvm->arch.nr_vcpus_matched_tsc++; 1363 } 1364 1365 /* Keep track of which generation this VCPU has synchronized to */ 1366 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation; 1367 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec; 1368 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write; 1369 1370 kvm_track_tsc_matching(vcpu, !matched); 1371 } 1372 1373 void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) 1374 { 1375 u64 data = user_value ? *user_value : 0; 1376 struct kvm *kvm = vcpu->kvm; 1377 u64 offset, ns, elapsed; 1378 unsigned long flags; 1379 bool matched = false; 1380 bool synchronizing = false; 1381 1382 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 1383 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); 1384 ns = get_kvmclock_base_ns(); 1385 elapsed = ns - kvm->arch.last_tsc_nsec; 1386 1387 if (vcpu->arch.virtual_tsc_khz) { 1388 if (data == 0) { 1389 /* 1390 * Force synchronization when creating a vCPU, or when 1391 * userspace explicitly writes a zero value. 1392 */ 1393 synchronizing = true; 1394 } else if (kvm->arch.user_set_tsc) { 1395 u64 tsc_exp = kvm->arch.last_tsc_write + 1396 nsec_to_cycles(vcpu, elapsed); 1397 u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL; 1398 /* 1399 * Here lies UAPI baggage: when a user-initiated TSC write has 1400 * a small delta (1 second) of virtual cycle time against the 1401 * previously set vCPU, we assume that they were intended to be 1402 * in sync and the delta was only due to the racy nature of the 1403 * legacy API. 1404 * 1405 * This trick falls down when restoring a guest which genuinely 1406 * has been running for less time than the 1 second of imprecision 1407 * which we allow for in the legacy API. In this case, the first 1408 * value written by userspace (on any vCPU) should not be subject 1409 * to this 'correction' to make it sync up with values that only 1410 * come from the kernel's default vCPU creation. Make the 1-second 1411 * slop hack only trigger if the user_set_tsc flag is already set. 1412 */ 1413 synchronizing = data < tsc_exp + tsc_hz && 1414 data + tsc_hz > tsc_exp; 1415 } 1416 } 1417 1418 1419 /* 1420 * For a reliable TSC, we can match TSC offsets, and for an unstable 1421 * TSC, we add elapsed time in this computation. We could let the 1422 * compensation code attempt to catch up if we fall behind, but 1423 * it's better to try to match offsets from the beginning. 1424 */ 1425 if (synchronizing && 1426 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) { 1427 if (!kvm_check_tsc_unstable()) { 1428 offset = kvm->arch.cur_tsc_offset; 1429 } else { 1430 u64 delta = nsec_to_cycles(vcpu, elapsed); 1431 data += delta; 1432 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); 1433 } 1434 matched = true; 1435 } 1436 1437 __kvm_synchronize_tsc(vcpu, offset, data, ns, matched, !!user_value); 1438 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 1439 } 1440 1441 #ifdef CONFIG_X86_64 1442 1443 static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap, 1444 u64 *tsc_timestamp) 1445 { 1446 /* 1447 * ktime_get_snapshot_id() cannot fail for standard clock IDs 1448 * (only for invalid/aux clocks or during suspend, with a WARN). 1449 */ 1450 if (!snap->valid) 1451 return false; 1452 1453 if (snap->cs_id == CSID_X86_TSC) { 1454 *tsc_timestamp = snap->cycles; 1455 return true; 1456 } 1457 1458 if (snap->hw_csid == CSID_X86_TSC && snap->hw_cycles) { 1459 *tsc_timestamp = snap->hw_cycles; 1460 return true; 1461 } 1462 1463 return false; 1464 } 1465 1466 /* 1467 * Calculates the kvmclock_base_ns (CLOCK_MONOTONIC_RAW + boot time) and 1468 * reports the TSC value from which it do so. Returns true if host is 1469 * using TSC based clocksource. 1470 */ 1471 static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) 1472 { 1473 struct system_time_snapshot snap = {}; 1474 1475 ktime_get_snapshot_id(CLOCK_MONOTONIC_RAW, &snap); 1476 if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) 1477 return false; 1478 1479 *kernel_ns = ktime_to_ns(ktime_mono_to_any(snap.systime, TK_OFFS_BOOT)); 1480 return true; 1481 } 1482 1483 /* 1484 * Calculates CLOCK_MONOTONIC and reports the TSC value from which it did 1485 * so. Returns true if host is using TSC based clocksource. 1486 */ 1487 bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) 1488 { 1489 struct system_time_snapshot snap = {}; 1490 1491 ktime_get_snapshot_id(CLOCK_MONOTONIC, &snap); 1492 if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) 1493 return false; 1494 1495 *kernel_ns = ktime_to_ns(snap.systime); 1496 return true; 1497 } 1498 1499 /* 1500 * Calculates CLOCK_REALTIME and reports the TSC value from which it did 1501 * so. Returns true if host is using TSC based clocksource. 1502 * 1503 * DO NOT USE this for anything related to migration. You want CLOCK_TAI 1504 * for that. 1505 */ 1506 static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, 1507 u64 *tsc_timestamp) 1508 { 1509 struct system_time_snapshot snap = {}; 1510 1511 ktime_get_snapshot_id(CLOCK_REALTIME, &snap); 1512 if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) 1513 return false; 1514 1515 *ts = ktime_to_timespec64(snap.systime); 1516 return true; 1517 } 1518 #endif 1519 1520 /* 1521 * 1522 * Assuming a stable TSC across physical CPUS, and a stable TSC 1523 * across virtual CPUs, the following condition is possible. 1524 * Each numbered line represents an event visible to both 1525 * CPUs at the next numbered event. 1526 * 1527 * "timespecX" represents host monotonic time. "tscX" represents 1528 * RDTSC value. 1529 * 1530 * VCPU0 on CPU0 | VCPU1 on CPU1 1531 * 1532 * 1. read timespec0,tsc0 1533 * 2. | timespec1 = timespec0 + N 1534 * | tsc1 = tsc0 + M 1535 * 3. transition to guest | transition to guest 1536 * 4. ret0 = timespec0 + (rdtsc - tsc0) | 1537 * 5. | ret1 = timespec1 + (rdtsc - tsc1) 1538 * | ret1 = timespec0 + N + (rdtsc - (tsc0 + M)) 1539 * 1540 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity: 1541 * 1542 * - ret0 < ret1 1543 * - timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M)) 1544 * ... 1545 * - 0 < N - M => M < N 1546 * 1547 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not 1548 * always the case (the difference between two distinct xtime instances 1549 * might be smaller then the difference between corresponding TSC reads, 1550 * when updating guest vcpus pvclock areas). 1551 * 1552 * To avoid that problem, do not allow visibility of distinct 1553 * system_timestamp/tsc_timestamp values simultaneously: use a master 1554 * copy of host monotonic time values. Update that master copy 1555 * in lockstep. 1556 * 1557 * Rely on synchronization of host TSCs and guest TSCs for monotonicity. 1558 * 1559 */ 1560 1561 static unsigned long get_cpu_tsc_khz(void); 1562 1563 static void pvclock_update_vm_gtod_copy(struct kvm *kvm) 1564 { 1565 #ifdef CONFIG_X86_64 1566 struct kvm_arch *ka = &kvm->arch; 1567 int vclock_mode; 1568 bool host_tsc_clocksource, vcpus_matched; 1569 1570 lockdep_assert_held(&kvm->arch.tsc_write_lock); 1571 vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == 1572 atomic_read(&kvm->online_vcpus)); 1573 1574 /* 1575 * If the host uses TSC clock, then passthrough TSC as stable 1576 * to the guest. 1577 */ 1578 host_tsc_clocksource = kvm_get_time_and_clockread( 1579 &ka->master_kernel_ns, 1580 &ka->master_cycle_now); 1581 1582 ka->use_master_clock = host_tsc_clocksource && vcpus_matched 1583 && !ka->backwards_tsc_observed 1584 && !ka->boot_vcpu_runs_old_kvmclock; 1585 1586 if (ka->use_master_clock) { 1587 u64 tsc_hz; 1588 1589 atomic_set(&kvm_guest_has_master_clock, 1); 1590 1591 /* 1592 * Copy the scaling ratio and precompute the mul/shift for 1593 * converting guest TSC to nanoseconds. These are used by 1594 * get_kvmclock() to compute kvmclock from the host TSC 1595 * without needing a vCPU reference. 1596 */ 1597 ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio; 1598 tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; 1599 if (tsc_hz && kvm_caps.has_tsc_control) 1600 tsc_hz = kvm_scale_tsc(tsc_hz, 1601 ka->master_tsc_scaling_ratio); 1602 if (tsc_hz) 1603 kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, 1604 &ka->master_tsc_shift, 1605 &ka->master_tsc_mul); 1606 else 1607 ka->use_master_clock = false; 1608 } 1609 1610 vclock_mode = pvclock_gtod_data.clock.vclock_mode; 1611 trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode, 1612 vcpus_matched); 1613 #endif 1614 } 1615 1616 static void kvm_make_mclock_inprogress_request(struct kvm *kvm) 1617 { 1618 kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS); 1619 } 1620 1621 static void __kvm_start_pvclock_update(struct kvm *kvm) 1622 { 1623 raw_spin_lock_irq(&kvm->arch.tsc_write_lock); 1624 write_seqcount_begin(&kvm->arch.pvclock_sc); 1625 } 1626 1627 static void kvm_start_pvclock_update(struct kvm *kvm) 1628 { 1629 kvm_make_mclock_inprogress_request(kvm); 1630 1631 /* no guest entries from this point */ 1632 __kvm_start_pvclock_update(kvm); 1633 } 1634 1635 static void kvm_end_pvclock_update(struct kvm *kvm) 1636 { 1637 struct kvm_arch *ka = &kvm->arch; 1638 struct kvm_vcpu *vcpu; 1639 unsigned long i; 1640 1641 write_seqcount_end(&ka->pvclock_sc); 1642 raw_spin_unlock_irq(&ka->tsc_write_lock); 1643 kvm_for_each_vcpu(i, vcpu, kvm) 1644 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 1645 1646 /* guest entries allowed */ 1647 kvm_for_each_vcpu(i, vcpu, kvm) 1648 kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu); 1649 } 1650 1651 static void kvm_update_masterclock(struct kvm *kvm) 1652 { 1653 kvm_hv_request_tsc_page_update(kvm); 1654 kvm_start_pvclock_update(kvm); 1655 pvclock_update_vm_gtod_copy(kvm); 1656 kvm_end_pvclock_update(kvm); 1657 } 1658 1659 /* 1660 * Use the kernel's tsc_khz directly if the TSC is constant, otherwise use KVM's 1661 * per-CPU value (which may be zero if a CPU is going offline). Note, tsc_khz 1662 * can change during boot even if the TSC is constant, as it's possible for KVM 1663 * to be loaded before TSC calibration completes. Ideally, KVM would get a 1664 * notification when calibration completes, but practically speaking calibration 1665 * will complete before userspace is alive enough to create VMs. 1666 */ 1667 static unsigned long get_cpu_tsc_khz(void) 1668 { 1669 if (cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC)) 1670 return tsc_khz; 1671 else 1672 return __this_cpu_read(cpu_tsc_khz); 1673 } 1674 1675 /* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */ 1676 static bool __get_kvmclock_master_clock(struct kvm *kvm, 1677 struct kvm_clock_data *data) 1678 { 1679 #ifdef CONFIG_X86_64 1680 struct kvm_arch *ka = &kvm->arch; 1681 struct pvclock_vcpu_time_info hv_clock; 1682 struct timespec64 ts; 1683 1684 if (!ka->use_master_clock) 1685 return false; 1686 1687 if (!kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) 1688 return false; 1689 1690 data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; 1691 data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC | 1692 KVM_CLOCK_TSC_STABLE; 1693 1694 hv_clock.tsc_timestamp = ka->master_cycle_now; 1695 hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; 1696 1697 /* 1698 * Use the precomputed guest-TSC-based mul/shift so that the kvmclock 1699 * value matches what the guest computes from its own TSC. 1700 */ 1701 hv_clock.tsc_shift = ka->master_tsc_shift; 1702 hv_clock.tsc_to_system_mul = ka->master_tsc_mul; 1703 1704 if (kvm_caps.has_tsc_control) { 1705 u64 tsc_delta = data->host_tsc - ka->master_cycle_now; 1706 1707 tsc_delta = kvm_scale_tsc(tsc_delta, ka->master_tsc_scaling_ratio); 1708 data->clock = hv_clock.system_time + 1709 pvclock_scale_delta(tsc_delta, 1710 hv_clock.tsc_to_system_mul, 1711 hv_clock.tsc_shift); 1712 } else { 1713 data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); 1714 } 1715 return true; 1716 #else 1717 return false; 1718 #endif 1719 } 1720 1721 static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) 1722 { 1723 struct kvm_arch *ka = &kvm->arch; 1724 unsigned seq; 1725 1726 do { 1727 data->flags = 0; 1728 1729 seq = read_seqcount_begin(&ka->pvclock_sc); 1730 if (!__get_kvmclock_master_clock(kvm, data)) 1731 data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; 1732 } while (read_seqcount_retry(&ka->pvclock_sc, seq)); 1733 } 1734 1735 u64 get_kvmclock_ns(struct kvm *kvm) 1736 { 1737 struct kvm_clock_data data; 1738 1739 get_kvmclock(kvm, &data); 1740 return data.clock; 1741 } 1742 1743 static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock, 1744 struct kvm_vcpu *vcpu, 1745 struct gfn_to_pfn_cache *gpc, 1746 unsigned int offset) 1747 { 1748 struct pvclock_vcpu_time_info *guest_hv_clock; 1749 struct pvclock_vcpu_time_info hv_clock; 1750 unsigned long flags; 1751 1752 memcpy(&hv_clock, ref_hv_clock, sizeof(hv_clock)); 1753 1754 read_lock_irqsave(&gpc->lock, flags); 1755 while (!kvm_gpc_check(gpc, offset + sizeof(*guest_hv_clock))) { 1756 read_unlock_irqrestore(&gpc->lock, flags); 1757 1758 if (kvm_gpc_refresh(gpc, offset + sizeof(*guest_hv_clock))) 1759 return; 1760 1761 read_lock_irqsave(&gpc->lock, flags); 1762 } 1763 1764 guest_hv_clock = (void *)(gpc->khva + offset); 1765 1766 /* 1767 * This VCPU is paused, but it's legal for a guest to read another 1768 * VCPU's kvmclock, so we really have to follow the specification where 1769 * it says that version is odd if data is being modified, and even after 1770 * it is consistent. 1771 */ 1772 1773 guest_hv_clock->version = hv_clock.version = (guest_hv_clock->version + 1) | 1; 1774 smp_wmb(); 1775 1776 /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */ 1777 hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED); 1778 1779 memcpy(guest_hv_clock, &hv_clock, sizeof(*guest_hv_clock)); 1780 1781 smp_wmb(); 1782 1783 guest_hv_clock->version = ++hv_clock.version; 1784 1785 kvm_gpc_mark_dirty_in_slot(gpc); 1786 read_unlock_irqrestore(&gpc->lock, flags); 1787 1788 trace_kvm_pvclock_update(vcpu->vcpu_id, &hv_clock); 1789 } 1790 1791 int kvm_guest_time_update(struct kvm_vcpu *v) 1792 { 1793 u64 tgt_tsc_hz, tsc_timestamp, host_tsc, master_tsc, master_ns; 1794 struct kvm_arch *ka __maybe_unused = &v->kvm->arch; 1795 struct pvclock_vcpu_time_info hv_clock = {}; 1796 struct kvm_vcpu_arch *vcpu = &v->arch; 1797 s64 kernel_ns; 1798 1799 /* 1800 * If the host uses TSC clock, then passthrough TSC as stable 1801 * to the guest. 1802 */ 1803 #ifdef CONFIG_X86_64 1804 bool use_master_clock; 1805 unsigned int seq; 1806 1807 do { 1808 seq = read_seqcount_begin(&ka->pvclock_sc); 1809 use_master_clock = ka->use_master_clock; 1810 if (!use_master_clock) 1811 continue; 1812 1813 if (!kvm_get_time_and_clockread(&kernel_ns, &host_tsc)) { 1814 use_master_clock = false; 1815 continue; 1816 } 1817 1818 master_tsc = ka->master_cycle_now; 1819 master_ns = ka->master_kernel_ns; 1820 } while (read_seqcount_retry(&ka->pvclock_sc, seq)); 1821 #else 1822 const bool use_master_clock = false; 1823 #endif 1824 /* 1825 * Ensure reading the TSC+frequency pair is done on the same CPU. When 1826 * NOT using the master clock, the TSC frequency may vary between CPUs. 1827 */ 1828 preempt_disable(); 1829 tgt_tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; 1830 if (unlikely(tgt_tsc_hz == 0)) { 1831 preempt_enable(); 1832 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); 1833 return 1; 1834 } 1835 if (!use_master_clock) { 1836 host_tsc = rdtsc(); 1837 kernel_ns = get_kvmclock_base_ns(); 1838 } 1839 1840 tsc_timestamp = kvm_read_l1_tsc(v, host_tsc); 1841 1842 /* 1843 * We may have to catch up the TSC to match elapsed wall clock 1844 * time for two reasons, even if kvmclock is used. 1845 * 1) CPU could have been running below the maximum TSC rate 1846 * 2) Broken TSC compensation resets the base at each VCPU 1847 * entry to avoid unknown leaps of TSC even when running 1848 * again on the same CPU. This may cause apparent elapsed 1849 * time to disappear, and the guest to stand still or run 1850 * very slowly. 1851 */ 1852 if (vcpu->tsc_catchup) { 1853 u64 tsc = compute_guest_tsc(v, kernel_ns); 1854 if (tsc > tsc_timestamp) { 1855 adjust_tsc_offset_guest(v, tsc - tsc_timestamp); 1856 tsc_timestamp = tsc; 1857 } 1858 } 1859 1860 /* 1861 * Refresh L1's last "observed" TSC to match the PV clock's timestamp, 1862 * e.g. so that the guest can't see a TSC that's behind the reference. 1863 */ 1864 vcpu->last_guest_tsc = tsc_timestamp; 1865 1866 preempt_enable(); 1867 1868 /* With all the info we got, fill in the values */ 1869 1870 if (kvm_caps.has_tsc_control) { 1871 tgt_tsc_hz = kvm_scale_tsc(tgt_tsc_hz, 1872 v->arch.l1_tsc_scaling_ratio); 1873 tgt_tsc_hz = tgt_tsc_hz ? : 1; 1874 } 1875 1876 if (unlikely(vcpu->hw_tsc_hz != tgt_tsc_hz)) { 1877 kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_hz, 1878 &vcpu->pvclock_tsc_shift, 1879 &vcpu->pvclock_tsc_mul); 1880 vcpu->hw_tsc_hz = tgt_tsc_hz; 1881 } 1882 1883 hv_clock.tsc_shift = vcpu->pvclock_tsc_shift; 1884 hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul; 1885 /* 1886 * If the master clock is NOT in use, the reference time placed in the 1887 * hv_clock is "now". If master clock is in use, the reference time is 1888 * the master clock's snapshot from some time in the past, not "now". 1889 */ 1890 if (use_master_clock) { 1891 hv_clock.tsc_timestamp = kvm_read_l1_tsc(v, master_tsc); 1892 hv_clock.system_time = master_ns + v->kvm->arch.kvmclock_offset; 1893 } else { 1894 hv_clock.tsc_timestamp = tsc_timestamp; 1895 hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; 1896 } 1897 1898 /* If the host uses TSC clocksource, then it is stable */ 1899 hv_clock.flags = 0; 1900 if (use_master_clock) 1901 hv_clock.flags |= PVCLOCK_TSC_STABLE_BIT; 1902 1903 if (vcpu->pv_time.active) { 1904 /* 1905 * GUEST_STOPPED is only supported by kvmclock, and KVM's 1906 * historic behavior is to only process the request if kvmclock 1907 * is active/enabled. 1908 */ 1909 if (vcpu->pvclock_set_guest_stopped_request) { 1910 hv_clock.flags |= PVCLOCK_GUEST_STOPPED; 1911 vcpu->pvclock_set_guest_stopped_request = false; 1912 } 1913 kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->pv_time, 0); 1914 1915 hv_clock.flags &= ~PVCLOCK_GUEST_STOPPED; 1916 } 1917 1918 kvm_hv_setup_tsc_page(v->kvm, &hv_clock); 1919 1920 #ifdef CONFIG_KVM_XEN 1921 /* 1922 * For Xen guests we may need to override PVCLOCK_TSC_STABLE_BIT as unless 1923 * explicitly told to use TSC as its clocksource Xen will not set this bit. 1924 * This default behaviour led to bugs in some guest kernels which cause 1925 * problems if they observe PVCLOCK_TSC_STABLE_BIT in the pvclock flags. 1926 * 1927 * Note! Clear TSC_STABLE only for Xen clocks, i.e. the order matters! 1928 */ 1929 if (ka->xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE) 1930 hv_clock.flags &= ~PVCLOCK_TSC_STABLE_BIT; 1931 1932 if (vcpu->xen.vcpu_info_cache.active) 1933 kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->xen.vcpu_info_cache, 1934 offsetof(struct compat_vcpu_info, time)); 1935 if (vcpu->xen.vcpu_time_info_cache.active) 1936 kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->xen.vcpu_time_info_cache, 0); 1937 #endif 1938 return 0; 1939 } 1940 1941 /* 1942 * The pvclock_wall_clock ABI tells the guest the wall clock time at 1943 * which it started (i.e. its epoch, when its kvmclock was zero). 1944 * 1945 * In fact those clocks are subtly different; wall clock frequency is 1946 * adjusted by NTP and has leap seconds, while the kvmclock is a 1947 * simple function of the TSC without any such adjustment. 1948 * 1949 * Perhaps the ABI should have exposed CLOCK_TAI and a ratio between 1950 * that and kvmclock, but even that would be subject to change over 1951 * time. 1952 * 1953 * Attempt to calculate the epoch at a given moment using the *same* 1954 * TSC reading via kvm_get_walltime_and_clockread() to obtain both 1955 * wallclock and kvmclock times, and subtracting one from the other. 1956 * 1957 * Fall back to using their values at slightly different moments by 1958 * calling ktime_get_real_ns() and get_kvmclock() separately. 1959 */ 1960 uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm) 1961 { 1962 struct kvm_clock_data data; 1963 1964 get_kvmclock(kvm, &data); 1965 1966 /* 1967 * If get_kvmclock() captured both wallclock and kvmclock from the 1968 * same TSC reading, use them for a precise epoch calculation. 1969 */ 1970 if (data.flags & KVM_CLOCK_REALTIME) 1971 return data.realtime - data.clock; 1972 1973 return ktime_get_real_ns() - data.clock; 1974 } 1975 1976 /* 1977 * kvmclock updates which are isolated to a given vcpu, such as 1978 * vcpu->cpu migration, should not allow system_timestamp from 1979 * the rest of the vcpus to remain static. 1980 * 1981 * So in those cases, request a kvmclock update for all vcpus. 1982 * The worst case for a remote vcpu to update its kvmclock 1983 * is then bounded by maximum nohz sleep latency. 1984 */ 1985 static void kvm_gen_kvmclock_update(struct kvm_vcpu *v) 1986 { 1987 unsigned long i; 1988 struct kvm_vcpu *vcpu; 1989 struct kvm *kvm = v->kvm; 1990 1991 kvm_for_each_vcpu(i, vcpu, kvm) { 1992 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 1993 kvm_vcpu_kick(vcpu); 1994 } 1995 } 1996 1997 static void kvmclock_reset(struct kvm_vcpu *vcpu) 1998 { 1999 kvm_gpc_deactivate(&vcpu->arch.pv_time); 2000 vcpu->arch.time = 0; 2001 } 2002 2003 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu) 2004 { 2005 ++vcpu->stat.tlb_flush; 2006 kvm_x86_call(flush_tlb_all)(vcpu); 2007 2008 /* Flushing all ASIDs flushes the current ASID... */ 2009 kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 2010 } 2011 2012 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu) 2013 { 2014 ++vcpu->stat.tlb_flush; 2015 2016 if (!tdp_enabled) { 2017 /* 2018 * A TLB flush on behalf of the guest is equivalent to 2019 * INVPCID(all), toggling CR4.PGE, etc., which requires 2020 * a forced sync of the shadow page tables. Ensure all the 2021 * roots are synced and the guest TLB in hardware is clean. 2022 */ 2023 kvm_mmu_sync_roots(vcpu); 2024 kvm_mmu_sync_prev_roots(vcpu); 2025 } 2026 2027 kvm_x86_call(flush_tlb_guest)(vcpu); 2028 2029 /* 2030 * Flushing all "guest" TLB is always a superset of Hyper-V's fine 2031 * grained flushing. 2032 */ 2033 kvm_hv_vcpu_purge_flush_tlb(vcpu); 2034 } 2035 2036 2037 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu) 2038 { 2039 ++vcpu->stat.tlb_flush; 2040 kvm_x86_call(flush_tlb_current)(vcpu); 2041 } 2042 2043 /* 2044 * Service "local" TLB flush requests, which are specific to the current MMU 2045 * context. In addition to the generic event handling in vcpu_enter_guest(), 2046 * TLB flushes that are targeted at an MMU context also need to be serviced 2047 * prior before nested VM-Enter/VM-Exit. 2048 */ 2049 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu) 2050 { 2051 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) 2052 kvm_vcpu_flush_tlb_current(vcpu); 2053 2054 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu)) 2055 kvm_vcpu_flush_tlb_guest(vcpu); 2056 } 2057 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_service_local_tlb_flush_requests); 2058 2059 static void record_steal_time(struct kvm_vcpu *vcpu) 2060 { 2061 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache; 2062 struct kvm_steal_time __user *st; 2063 struct kvm_memslots *slots; 2064 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS; 2065 u64 steal; 2066 u32 version; 2067 2068 if (kvm_xen_msr_enabled(vcpu->kvm)) { 2069 kvm_xen_runstate_set_running(vcpu); 2070 return; 2071 } 2072 2073 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 2074 return; 2075 2076 if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm)) 2077 return; 2078 2079 slots = kvm_memslots(vcpu->kvm); 2080 2081 if (unlikely(slots->generation != ghc->generation || 2082 gpa != ghc->gpa || 2083 kvm_is_error_hva(ghc->hva) || !ghc->memslot)) { 2084 /* We rely on the fact that it fits in a single page. */ 2085 BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS); 2086 2087 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) || 2088 kvm_is_error_hva(ghc->hva) || !ghc->memslot) 2089 return; 2090 } 2091 2092 st = (struct kvm_steal_time __user *)ghc->hva; 2093 /* 2094 * Doing a TLB flush here, on the guest's behalf, can avoid 2095 * expensive IPIs. 2096 */ 2097 if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) { 2098 u8 st_preempted = 0; 2099 int err = -EFAULT; 2100 2101 if (!user_access_begin(st, sizeof(*st))) 2102 return; 2103 2104 asm volatile("1: xchgb %0, %2\n" 2105 "xor %1, %1\n" 2106 "2:\n" 2107 _ASM_EXTABLE_UA(1b, 2b) 2108 : "+q" (st_preempted), 2109 "+&r" (err), 2110 "+m" (st->preempted)); 2111 if (err) 2112 goto out; 2113 2114 user_access_end(); 2115 2116 vcpu->arch.st.preempted = 0; 2117 2118 trace_kvm_pv_tlb_flush(vcpu->vcpu_id, 2119 st_preempted & KVM_VCPU_FLUSH_TLB); 2120 if (st_preempted & KVM_VCPU_FLUSH_TLB) 2121 kvm_vcpu_flush_tlb_guest(vcpu); 2122 2123 if (!user_access_begin(st, sizeof(*st))) 2124 goto dirty; 2125 } else { 2126 if (!user_access_begin(st, sizeof(*st))) 2127 return; 2128 2129 unsafe_put_user(0, &st->preempted, out); 2130 vcpu->arch.st.preempted = 0; 2131 } 2132 2133 unsafe_get_user(version, &st->version, out); 2134 if (version & 1) 2135 version += 1; /* first time write, random junk */ 2136 2137 version += 1; 2138 unsafe_put_user(version, &st->version, out); 2139 2140 smp_wmb(); 2141 2142 unsafe_get_user(steal, &st->steal, out); 2143 steal += current->sched_info.run_delay - 2144 vcpu->arch.st.last_steal; 2145 vcpu->arch.st.last_steal = current->sched_info.run_delay; 2146 unsafe_put_user(steal, &st->steal, out); 2147 2148 version += 1; 2149 unsafe_put_user(version, &st->version, out); 2150 2151 out: 2152 user_access_end(); 2153 dirty: 2154 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 2155 } 2156 2157 static inline bool kvm_can_mwait_in_guest(void) 2158 { 2159 return boot_cpu_has(X86_FEATURE_MWAIT) && 2160 !boot_cpu_has_bug(X86_BUG_MONITOR) && 2161 boot_cpu_has(X86_FEATURE_ARAT); 2162 } 2163 2164 static u64 kvm_get_allowed_disable_exits(void) 2165 { 2166 u64 r = KVM_X86_DISABLE_EXITS_PAUSE; 2167 2168 if (boot_cpu_has(X86_FEATURE_APERFMPERF)) 2169 r |= KVM_X86_DISABLE_EXITS_APERFMPERF; 2170 2171 if (!mitigate_smt_rsb) { 2172 r |= KVM_X86_DISABLE_EXITS_HLT | 2173 KVM_X86_DISABLE_EXITS_CSTATE; 2174 2175 if (kvm_can_mwait_in_guest()) 2176 r |= KVM_X86_DISABLE_EXITS_MWAIT; 2177 } 2178 return r; 2179 } 2180 2181 #ifdef CONFIG_KVM_HYPERV 2182 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu, 2183 struct kvm_cpuid2 __user *cpuid_arg) 2184 { 2185 struct kvm_cpuid2 cpuid; 2186 int r; 2187 2188 r = -EFAULT; 2189 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 2190 return r; 2191 2192 r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries); 2193 if (r) 2194 return r; 2195 2196 r = -EFAULT; 2197 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid))) 2198 return r; 2199 2200 return 0; 2201 } 2202 #endif 2203 2204 static bool kvm_is_vm_type_supported(unsigned long type) 2205 { 2206 return type < 32 && (kvm_caps.supported_vm_types & BIT(type)); 2207 } 2208 2209 static inline u64 kvm_sync_valid_fields(struct kvm *kvm) 2210 { 2211 return kvm && kvm->arch.has_protected_state ? 0 : KVM_SYNC_X86_VALID_FIELDS; 2212 } 2213 2214 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 2215 { 2216 int r = 0; 2217 2218 switch (ext) { 2219 case KVM_CAP_IRQCHIP: 2220 case KVM_CAP_HLT: 2221 case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: 2222 case KVM_CAP_SET_TSS_ADDR: 2223 case KVM_CAP_EXT_CPUID: 2224 case KVM_CAP_EXT_EMUL_CPUID: 2225 case KVM_CAP_CLOCKSOURCE: 2226 #ifdef CONFIG_KVM_IOAPIC 2227 case KVM_CAP_PIT: 2228 case KVM_CAP_PIT2: 2229 case KVM_CAP_PIT_STATE2: 2230 case KVM_CAP_REINJECT_CONTROL: 2231 #endif 2232 case KVM_CAP_NOP_IO_DELAY: 2233 case KVM_CAP_MP_STATE: 2234 case KVM_CAP_USER_NMI: 2235 case KVM_CAP_IRQ_INJECT_STATUS: 2236 case KVM_CAP_IOEVENTFD: 2237 case KVM_CAP_IOEVENTFD_NO_LENGTH: 2238 2239 case KVM_CAP_SET_IDENTITY_MAP_ADDR: 2240 case KVM_CAP_VCPU_EVENTS: 2241 #ifdef CONFIG_KVM_HYPERV 2242 case KVM_CAP_HYPERV: 2243 case KVM_CAP_HYPERV_VAPIC: 2244 case KVM_CAP_HYPERV_SPIN: 2245 case KVM_CAP_HYPERV_TIME: 2246 case KVM_CAP_HYPERV_SYNIC: 2247 case KVM_CAP_HYPERV_SYNIC2: 2248 case KVM_CAP_HYPERV_VP_INDEX: 2249 case KVM_CAP_HYPERV_EVENTFD: 2250 case KVM_CAP_HYPERV_TLBFLUSH: 2251 case KVM_CAP_HYPERV_SEND_IPI: 2252 case KVM_CAP_HYPERV_CPUID: 2253 case KVM_CAP_HYPERV_ENFORCE_CPUID: 2254 case KVM_CAP_SYS_HYPERV_CPUID: 2255 #endif 2256 case KVM_CAP_PCI_SEGMENT: 2257 case KVM_CAP_DEBUGREGS: 2258 case KVM_CAP_X86_ROBUST_SINGLESTEP: 2259 case KVM_CAP_XSAVE: 2260 case KVM_CAP_ASYNC_PF: 2261 case KVM_CAP_ASYNC_PF_INT: 2262 case KVM_CAP_GET_TSC_KHZ: 2263 case KVM_CAP_KVMCLOCK_CTRL: 2264 case KVM_CAP_IOAPIC_POLARITY_IGNORED: 2265 case KVM_CAP_TSC_DEADLINE_TIMER: 2266 case KVM_CAP_DISABLE_QUIRKS: 2267 case KVM_CAP_SET_BOOT_CPU_ID: 2268 case KVM_CAP_SPLIT_IRQCHIP: 2269 case KVM_CAP_IMMEDIATE_EXIT: 2270 case KVM_CAP_PMU_EVENT_FILTER: 2271 case KVM_CAP_PMU_EVENT_MASKED_EVENTS: 2272 case KVM_CAP_GET_MSR_FEATURES: 2273 case KVM_CAP_MSR_PLATFORM_INFO: 2274 case KVM_CAP_EXCEPTION_PAYLOAD: 2275 case KVM_CAP_X86_TRIPLE_FAULT_EVENT: 2276 case KVM_CAP_SET_GUEST_DEBUG: 2277 case KVM_CAP_LAST_CPU: 2278 case KVM_CAP_X86_USER_SPACE_MSR: 2279 case KVM_CAP_X86_MSR_FILTER: 2280 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID: 2281 #ifdef CONFIG_X86_SGX_KVM 2282 case KVM_CAP_SGX_ATTRIBUTE: 2283 #endif 2284 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: 2285 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM: 2286 case KVM_CAP_SREGS2: 2287 case KVM_CAP_EXIT_ON_EMULATION_FAILURE: 2288 case KVM_CAP_VCPU_ATTRIBUTES: 2289 case KVM_CAP_SYS_ATTRIBUTES: 2290 case KVM_CAP_VAPIC: 2291 case KVM_CAP_ENABLE_CAP: 2292 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES: 2293 case KVM_CAP_IRQFD_RESAMPLE: 2294 case KVM_CAP_MEMORY_FAULT_INFO: 2295 case KVM_CAP_X86_GUEST_MODE: 2296 case KVM_CAP_ONE_REG: 2297 r = 1; 2298 break; 2299 case KVM_CAP_PRE_FAULT_MEMORY: 2300 r = tdp_enabled; 2301 break; 2302 case KVM_CAP_X86_APIC_BUS_CYCLES_NS: 2303 r = kvm ? kvm->arch.apic_bus_cycle_ns : APIC_BUS_CYCLE_NS_DEFAULT; 2304 break; 2305 case KVM_CAP_EXIT_HYPERCALL: 2306 r = KVM_EXIT_HYPERCALL_VALID_MASK; 2307 break; 2308 case KVM_CAP_SET_GUEST_DEBUG2: 2309 return KVM_GUESTDBG_VALID_MASK; 2310 #ifdef CONFIG_KVM_XEN 2311 case KVM_CAP_XEN_HVM: 2312 r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR | 2313 KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL | 2314 KVM_XEN_HVM_CONFIG_SHARED_INFO | 2315 KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL | 2316 KVM_XEN_HVM_CONFIG_EVTCHN_SEND | 2317 KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE | 2318 KVM_XEN_HVM_CONFIG_SHARED_INFO_HVA; 2319 if (sched_info_on()) 2320 r |= KVM_XEN_HVM_CONFIG_RUNSTATE | 2321 KVM_XEN_HVM_CONFIG_RUNSTATE_UPDATE_FLAG; 2322 break; 2323 #endif 2324 case KVM_CAP_SYNC_REGS: 2325 r = kvm_sync_valid_fields(kvm); 2326 break; 2327 case KVM_CAP_ADJUST_CLOCK: 2328 r = KVM_CLOCK_VALID_FLAGS; 2329 break; 2330 case KVM_CAP_X86_DISABLE_EXITS: 2331 r = kvm_get_allowed_disable_exits(); 2332 break; 2333 case KVM_CAP_X86_SMM: 2334 if (!IS_ENABLED(CONFIG_KVM_SMM)) 2335 break; 2336 2337 /* SMBASE is usually relocated above 1M on modern chipsets, 2338 * and SMM handlers might indeed rely on 4G segment limits, 2339 * so do not report SMM to be available if real mode is 2340 * emulated via vm86 mode. Still, do not go to great lengths 2341 * to avoid userspace's usage of the feature, because it is a 2342 * fringe case that is not enabled except via specific settings 2343 * of the module parameters. 2344 */ 2345 r = kvm_x86_call(has_emulated_msr)(kvm, MSR_IA32_SMBASE); 2346 break; 2347 case KVM_CAP_NR_VCPUS: 2348 r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS); 2349 break; 2350 case KVM_CAP_MAX_VCPUS: 2351 r = KVM_MAX_VCPUS; 2352 if (kvm) 2353 r = kvm->max_vcpus; 2354 break; 2355 case KVM_CAP_MAX_VCPU_ID: 2356 r = KVM_MAX_VCPU_IDS; 2357 break; 2358 case KVM_CAP_PV_MMU: /* obsolete */ 2359 r = 0; 2360 break; 2361 case KVM_CAP_MCE: 2362 r = KVM_MAX_MCE_BANKS; 2363 break; 2364 case KVM_CAP_XCRS: 2365 r = boot_cpu_has(X86_FEATURE_XSAVE); 2366 break; 2367 case KVM_CAP_TSC_CONTROL: 2368 case KVM_CAP_VM_TSC_CONTROL: 2369 r = kvm_caps.has_tsc_control; 2370 break; 2371 case KVM_CAP_X2APIC_API: 2372 r = KVM_X2APIC_API_VALID_FLAGS; 2373 if (kvm && !irqchip_split(kvm)) 2374 r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST; 2375 break; 2376 case KVM_CAP_NESTED_STATE: 2377 r = kvm_nested_ops.enabled ? kvm_nested_call(get_state)(NULL, NULL, 0) : 0; 2378 break; 2379 #ifdef CONFIG_KVM_HYPERV 2380 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: 2381 r = kvm_x86_ops.enable_l2_tlb_flush != NULL; 2382 break; 2383 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: 2384 r = kvm_nested_ops.enabled && kvm_nested_ops.enable_evmcs != NULL; 2385 break; 2386 #endif 2387 case KVM_CAP_SMALLER_MAXPHYADDR: 2388 r = (int) allow_smaller_maxphyaddr; 2389 break; 2390 case KVM_CAP_STEAL_TIME: 2391 r = sched_info_on(); 2392 break; 2393 case KVM_CAP_X86_BUS_LOCK_EXIT: 2394 if (kvm_caps.has_bus_lock_exit) 2395 r = KVM_BUS_LOCK_DETECTION_OFF | 2396 KVM_BUS_LOCK_DETECTION_EXIT; 2397 else 2398 r = 0; 2399 break; 2400 case KVM_CAP_XSAVE2: { 2401 r = xstate_required_size(kvm_get_filtered_xcr0(), false); 2402 if (r < sizeof(struct kvm_xsave)) 2403 r = sizeof(struct kvm_xsave); 2404 break; 2405 } 2406 case KVM_CAP_PMU_CAPABILITY: 2407 r = enable_pmu ? KVM_CAP_PMU_VALID_MASK : 0; 2408 break; 2409 case KVM_CAP_DISABLE_QUIRKS2: 2410 r = kvm_caps.supported_quirks; 2411 break; 2412 case KVM_CAP_X86_NOTIFY_VMEXIT: 2413 r = kvm_caps.has_notify_vmexit; 2414 break; 2415 case KVM_CAP_VM_TYPES: 2416 r = kvm_caps.supported_vm_types; 2417 break; 2418 case KVM_CAP_READONLY_MEM: 2419 r = kvm ? kvm_arch_has_readonly_mem(kvm) : 1; 2420 break; 2421 default: 2422 break; 2423 } 2424 return r; 2425 } 2426 2427 static int __kvm_x86_dev_get_attr(struct kvm_device_attr *attr, u64 *val) 2428 { 2429 if (attr->group) { 2430 if (kvm_x86_ops.dev_get_attr) 2431 return kvm_x86_call(dev_get_attr)(attr->group, attr->attr, val); 2432 return -ENXIO; 2433 } 2434 2435 switch (attr->attr) { 2436 case KVM_X86_XCOMP_GUEST_SUPP: 2437 *val = kvm_caps.supported_xcr0; 2438 return 0; 2439 default: 2440 return -ENXIO; 2441 } 2442 } 2443 2444 static int kvm_x86_dev_get_attr(struct kvm_device_attr *attr) 2445 { 2446 u64 __user *uaddr = u64_to_user_ptr(attr->addr); 2447 int r; 2448 u64 val; 2449 2450 r = __kvm_x86_dev_get_attr(attr, &val); 2451 if (r < 0) 2452 return r; 2453 2454 if (put_user(val, uaddr)) 2455 return -EFAULT; 2456 2457 return 0; 2458 } 2459 2460 static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr) 2461 { 2462 u64 val; 2463 2464 return __kvm_x86_dev_get_attr(attr, &val); 2465 } 2466 2467 long kvm_arch_dev_ioctl(struct file *filp, 2468 unsigned int ioctl, unsigned long arg) 2469 { 2470 void __user *argp = (void __user *)arg; 2471 long r; 2472 2473 switch (ioctl) { 2474 case KVM_GET_MSR_INDEX_LIST: 2475 r = kvm_get_msr_index_list(argp); 2476 break; 2477 case KVM_GET_SUPPORTED_CPUID: 2478 case KVM_GET_EMULATED_CPUID: { 2479 struct kvm_cpuid2 __user *cpuid_arg = argp; 2480 struct kvm_cpuid2 cpuid; 2481 2482 r = -EFAULT; 2483 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 2484 goto out; 2485 2486 r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries, 2487 ioctl); 2488 if (r) 2489 goto out; 2490 2491 r = -EFAULT; 2492 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid))) 2493 goto out; 2494 r = 0; 2495 break; 2496 } 2497 case KVM_X86_GET_MCE_CAP_SUPPORTED: 2498 r = -EFAULT; 2499 if (copy_to_user(argp, &kvm_caps.supported_mce_cap, 2500 sizeof(kvm_caps.supported_mce_cap))) 2501 goto out; 2502 r = 0; 2503 break; 2504 case KVM_GET_MSR_FEATURE_INDEX_LIST: 2505 r = kvm_get_feature_msr_index_list(argp); 2506 break; 2507 case KVM_GET_MSRS: 2508 r = kvm_get_feature_msrs(argp); 2509 break; 2510 #ifdef CONFIG_KVM_HYPERV 2511 case KVM_GET_SUPPORTED_HV_CPUID: 2512 r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp); 2513 break; 2514 #endif 2515 case KVM_GET_DEVICE_ATTR: { 2516 struct kvm_device_attr attr; 2517 r = -EFAULT; 2518 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2519 break; 2520 r = kvm_x86_dev_get_attr(&attr); 2521 break; 2522 } 2523 case KVM_HAS_DEVICE_ATTR: { 2524 struct kvm_device_attr attr; 2525 r = -EFAULT; 2526 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2527 break; 2528 r = kvm_x86_dev_has_attr(&attr); 2529 break; 2530 } 2531 default: 2532 r = -EINVAL; 2533 break; 2534 } 2535 out: 2536 return r; 2537 } 2538 2539 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu) 2540 { 2541 return kvm_arch_has_noncoherent_dma(vcpu->kvm); 2542 } 2543 2544 static DEFINE_PER_CPU(struct kvm_vcpu *, last_vcpu); 2545 2546 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 2547 { 2548 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 2549 2550 kvm_request_l1tf_flush_l1d(); 2551 2552 if (vcpu->scheduled_out && pmu->version && pmu->event_count) { 2553 pmu->need_cleanup = true; 2554 kvm_make_request(KVM_REQ_PMU, vcpu); 2555 } 2556 2557 /* Address WBINVD may be executed by guest */ 2558 if (need_emulate_wbinvd(vcpu)) { 2559 if (kvm_x86_call(has_wbinvd_exit)()) 2560 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 2561 else if (vcpu->cpu != -1 && vcpu->cpu != cpu) 2562 wbinvd_on_cpu(vcpu->cpu); 2563 } 2564 2565 kvm_x86_call(vcpu_load)(vcpu, cpu); 2566 2567 if (vcpu != per_cpu(last_vcpu, cpu)) { 2568 /* 2569 * Flush the branch predictor when switching vCPUs on the same 2570 * physical CPU, as each vCPU needs its own branch prediction 2571 * domain. No IBPB is needed when switching between L1 and L2 2572 * on the same vCPU unless IBRS is advertised to the vCPU; that 2573 * is handled on the nested VM-Exit path. 2574 */ 2575 if (static_branch_likely(&switch_vcpu_ibpb)) 2576 indirect_branch_prediction_barrier(); 2577 per_cpu(last_vcpu, cpu) = vcpu; 2578 } 2579 2580 /* Save host pkru register if supported */ 2581 vcpu->arch.host_pkru = read_pkru(); 2582 2583 /* Apply any externally detected TSC adjustments (due to suspend) */ 2584 if (unlikely(vcpu->arch.tsc_offset_adjustment)) { 2585 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment); 2586 vcpu->arch.tsc_offset_adjustment = 0; 2587 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 2588 } 2589 2590 if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) { 2591 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 : 2592 rdtsc() - vcpu->arch.last_host_tsc; 2593 if (tsc_delta < 0) 2594 mark_tsc_unstable("KVM discovered backwards TSC"); 2595 2596 if (kvm_check_tsc_unstable()) { 2597 u64 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), 2598 vcpu->arch.last_guest_tsc); 2599 kvm_vcpu_write_tsc_offset(vcpu, offset); 2600 if (!vcpu->arch.guest_tsc_protected) 2601 vcpu->arch.tsc_catchup = 1; 2602 } 2603 2604 if (kvm_lapic_hv_timer_in_use(vcpu)) 2605 kvm_lapic_restart_hv_timer(vcpu); 2606 2607 /* 2608 * On a host with synchronized TSC, there is no need to update 2609 * kvmclock on vcpu->cpu migration 2610 */ 2611 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1) { 2612 if (__ratelimit(&vcpu->kvm->arch.kvmclock_update_rs)) 2613 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); 2614 else 2615 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 2616 } 2617 2618 if (vcpu->cpu != cpu) 2619 kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu); 2620 vcpu->cpu = cpu; 2621 } 2622 2623 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 2624 } 2625 2626 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) 2627 { 2628 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache; 2629 struct kvm_steal_time __user *st; 2630 struct kvm_memslots *slots; 2631 static const u8 preempted = KVM_VCPU_PREEMPTED; 2632 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS; 2633 2634 /* 2635 * The vCPU can be marked preempted if and only if the VM-Exit was on 2636 * an instruction boundary and will not trigger guest emulation of any 2637 * kind (see vcpu_run). Vendor specific code controls (conservatively) 2638 * when this is true, for example allowing the vCPU to be marked 2639 * preempted if and only if the VM-Exit was due to a host interrupt. 2640 */ 2641 if (!vcpu->arch.at_instruction_boundary) { 2642 vcpu->stat.preemption_other++; 2643 return; 2644 } 2645 2646 vcpu->stat.preemption_reported++; 2647 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 2648 return; 2649 2650 if (vcpu->arch.st.preempted) 2651 return; 2652 2653 /* This happens on process exit */ 2654 if (unlikely(current->mm != vcpu->kvm->mm)) 2655 return; 2656 2657 slots = kvm_memslots(vcpu->kvm); 2658 2659 if (unlikely(slots->generation != ghc->generation || 2660 gpa != ghc->gpa || 2661 kvm_is_error_hva(ghc->hva) || !ghc->memslot)) 2662 return; 2663 2664 st = (struct kvm_steal_time __user *)ghc->hva; 2665 BUILD_BUG_ON(sizeof(st->preempted) != sizeof(preempted)); 2666 2667 if (!copy_to_user_nofault(&st->preempted, &preempted, sizeof(preempted))) 2668 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED; 2669 2670 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 2671 } 2672 2673 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 2674 { 2675 int idx; 2676 2677 if (vcpu->preempted) { 2678 /* 2679 * Assume protected guests are in-kernel. Inefficient yielding 2680 * due to false positives is preferable to never yielding due 2681 * to false negatives. 2682 */ 2683 vcpu->arch.preempted_in_kernel = vcpu->arch.guest_state_protected || 2684 !kvm_x86_call(get_cpl_no_cache)(vcpu); 2685 2686 /* 2687 * Take the srcu lock as memslots will be accessed to check the gfn 2688 * cache generation against the memslots generation. 2689 */ 2690 idx = srcu_read_lock(&vcpu->kvm->srcu); 2691 if (kvm_xen_msr_enabled(vcpu->kvm)) 2692 kvm_xen_runstate_set_preempted(vcpu); 2693 else 2694 kvm_steal_time_set_preempted(vcpu); 2695 srcu_read_unlock(&vcpu->kvm->srcu, idx); 2696 } 2697 2698 kvm_x86_call(vcpu_put)(vcpu); 2699 vcpu->arch.last_host_tsc = rdtsc(); 2700 } 2701 2702 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, 2703 struct kvm_lapic_state *s) 2704 { 2705 if (vcpu->arch.apic->guest_apic_protected) 2706 return -EINVAL; 2707 2708 kvm_x86_call(sync_pir_to_irr)(vcpu); 2709 2710 return kvm_apic_get_state(vcpu, s); 2711 } 2712 2713 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, 2714 struct kvm_lapic_state *s) 2715 { 2716 int r; 2717 2718 if (vcpu->arch.apic->guest_apic_protected) 2719 return -EINVAL; 2720 2721 r = kvm_apic_set_state(vcpu, s); 2722 if (r) 2723 return r; 2724 2725 return 0; 2726 } 2727 2728 static bool kvm_is_interrupt_allowed(struct kvm_vcpu *vcpu) 2729 { 2730 /* 2731 * Note, .interrupt_allowed() returns -EBUSY if interrupts are allowed 2732 * based on CPU state, but can't be immediately delivered due to a 2733 * pending nested VM-Enter. Treat that case as "allowed", because 2734 * the goal here is just to check if interrupts are architecturally 2735 * allowed, not to check if they can be injected. 2736 */ 2737 return kvm_x86_call(interrupt_allowed)(vcpu, false); 2738 } 2739 2740 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu) 2741 { 2742 /* 2743 * We can accept userspace's request for interrupt injection 2744 * as long as we have a place to store the interrupt number. 2745 * The actual injection will happen when the CPU is able to 2746 * deliver the interrupt. 2747 */ 2748 if (kvm_cpu_has_extint(vcpu)) 2749 return false; 2750 2751 /* Acknowledging ExtINT does not happen if LINT0 is masked. */ 2752 return (!lapic_in_kernel(vcpu) || 2753 kvm_apic_accept_pic_intr(vcpu)); 2754 } 2755 2756 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu) 2757 { 2758 /* 2759 * Do not cause an interrupt window exit if an exception 2760 * is pending or an event needs reinjection; userspace 2761 * might want to inject the interrupt manually using KVM_SET_REGS 2762 * or KVM_SET_SREGS. For that to work, we must be at an 2763 * instruction boundary and with no events half-injected. 2764 */ 2765 return (kvm_is_interrupt_allowed(vcpu) && 2766 kvm_cpu_accept_dm_intr(vcpu) && 2767 !kvm_event_needs_reinjection(vcpu) && 2768 !kvm_is_exception_pending(vcpu)); 2769 } 2770 2771 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, 2772 struct kvm_interrupt *irq) 2773 { 2774 if (irq->irq >= KVM_NR_INTERRUPTS) 2775 return -EINVAL; 2776 2777 if (!irqchip_in_kernel(vcpu->kvm)) { 2778 kvm_queue_interrupt(vcpu, irq->irq, false); 2779 kvm_make_request(KVM_REQ_EVENT, vcpu); 2780 return 0; 2781 } 2782 2783 /* 2784 * With in-kernel LAPIC, we only use this to inject EXTINT, so 2785 * fail for in-kernel 8259. 2786 */ 2787 if (pic_in_kernel(vcpu->kvm)) 2788 return -ENXIO; 2789 2790 if (vcpu->arch.pending_external_vector != -1) 2791 return -EEXIST; 2792 2793 vcpu->arch.pending_external_vector = irq->irq; 2794 kvm_make_request(KVM_REQ_EVENT, vcpu); 2795 return 0; 2796 } 2797 2798 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu) 2799 { 2800 kvm_inject_nmi(vcpu); 2801 2802 return 0; 2803 } 2804 2805 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu, 2806 struct kvm_tpr_access_ctl *tac) 2807 { 2808 if (tac->flags) 2809 return -EINVAL; 2810 vcpu->arch.tpr_access_reporting = !!tac->enabled; 2811 return 0; 2812 } 2813 2814 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, 2815 u64 mcg_cap) 2816 { 2817 int r; 2818 unsigned bank_num = mcg_cap & 0xff, bank; 2819 2820 r = -EINVAL; 2821 if (!bank_num || bank_num > KVM_MAX_MCE_BANKS) 2822 goto out; 2823 if (mcg_cap & ~(kvm_caps.supported_mce_cap | 0xff | 0xff0000)) 2824 goto out; 2825 r = 0; 2826 vcpu->arch.mcg_cap = mcg_cap; 2827 /* Init IA32_MCG_CTL to all 1s */ 2828 if (mcg_cap & MCG_CTL_P) 2829 vcpu->arch.mcg_ctl = ~(u64)0; 2830 /* Init IA32_MCi_CTL to all 1s, IA32_MCi_CTL2 to all 0s */ 2831 for (bank = 0; bank < bank_num; bank++) { 2832 vcpu->arch.mce_banks[bank*4] = ~(u64)0; 2833 if (mcg_cap & MCG_CMCI_P) 2834 vcpu->arch.mci_ctl2_banks[bank] = 0; 2835 } 2836 2837 kvm_apic_after_set_mcg_cap(vcpu); 2838 2839 kvm_x86_call(setup_mce)(vcpu); 2840 out: 2841 return r; 2842 } 2843 2844 /* 2845 * Validate this is an UCNA (uncorrectable no action) error by checking the 2846 * MCG_STATUS and MCi_STATUS registers: 2847 * - none of the bits for Machine Check Exceptions are set 2848 * - both the VAL (valid) and UC (uncorrectable) bits are set 2849 * MCI_STATUS_PCC - Processor Context Corrupted 2850 * MCI_STATUS_S - Signaled as a Machine Check Exception 2851 * MCI_STATUS_AR - Software recoverable Action Required 2852 */ 2853 static bool is_ucna(struct kvm_x86_mce *mce) 2854 { 2855 return !mce->mcg_status && 2856 !(mce->status & (MCI_STATUS_PCC | MCI_STATUS_S | MCI_STATUS_AR)) && 2857 (mce->status & MCI_STATUS_VAL) && 2858 (mce->status & MCI_STATUS_UC); 2859 } 2860 2861 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks) 2862 { 2863 u64 mcg_cap = vcpu->arch.mcg_cap; 2864 2865 banks[1] = mce->status; 2866 banks[2] = mce->addr; 2867 banks[3] = mce->misc; 2868 vcpu->arch.mcg_status = mce->mcg_status; 2869 2870 if (!(mcg_cap & MCG_CMCI_P) || 2871 !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN)) 2872 return 0; 2873 2874 if (lapic_in_kernel(vcpu)) 2875 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI); 2876 2877 return 0; 2878 } 2879 2880 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, 2881 struct kvm_x86_mce *mce) 2882 { 2883 u64 mcg_cap = vcpu->arch.mcg_cap; 2884 unsigned bank_num = mcg_cap & 0xff; 2885 u64 *banks = vcpu->arch.mce_banks; 2886 2887 if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL)) 2888 return -EINVAL; 2889 2890 mce->bank = array_index_nospec(mce->bank, bank_num); 2891 banks += 4 * mce->bank; 2892 2893 if (is_ucna(mce)) 2894 return kvm_vcpu_x86_set_ucna(vcpu, mce, banks); 2895 2896 /* 2897 * if IA32_MCG_CTL is not all 1s, the uncorrected error 2898 * reporting is disabled 2899 */ 2900 if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) && 2901 vcpu->arch.mcg_ctl != ~(u64)0) 2902 return 0; 2903 /* 2904 * if IA32_MCi_CTL is not all 1s, the uncorrected error 2905 * reporting is disabled for the bank 2906 */ 2907 if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0) 2908 return 0; 2909 if (mce->status & MCI_STATUS_UC) { 2910 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) || 2911 !kvm_is_cr4_bit_set(vcpu, X86_CR4_MCE)) { 2912 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 2913 return 0; 2914 } 2915 if (banks[1] & MCI_STATUS_VAL) 2916 mce->status |= MCI_STATUS_OVER; 2917 banks[2] = mce->addr; 2918 banks[3] = mce->misc; 2919 vcpu->arch.mcg_status = mce->mcg_status; 2920 banks[1] = mce->status; 2921 kvm_queue_exception(vcpu, MC_VECTOR); 2922 } else if (!(banks[1] & MCI_STATUS_VAL) 2923 || !(banks[1] & MCI_STATUS_UC)) { 2924 if (banks[1] & MCI_STATUS_VAL) 2925 mce->status |= MCI_STATUS_OVER; 2926 banks[2] = mce->addr; 2927 banks[3] = mce->misc; 2928 banks[1] = mce->status; 2929 } else 2930 banks[1] |= MCI_STATUS_OVER; 2931 return 0; 2932 } 2933 2934 static struct kvm_queued_exception *kvm_get_exception_to_save(struct kvm_vcpu *vcpu) 2935 { 2936 /* 2937 * KVM's ABI only allows for one exception to be migrated. Luckily, 2938 * the only time there can be two queued exceptions is if there's a 2939 * non-exiting _injected_ exception, and a pending exiting exception. 2940 * In that case, ignore the VM-Exiting exception as it's an extension 2941 * of the injected exception. 2942 */ 2943 if (vcpu->arch.exception_vmexit.pending && 2944 !vcpu->arch.exception.pending && 2945 !vcpu->arch.exception.injected) 2946 return &vcpu->arch.exception_vmexit; 2947 2948 return &vcpu->arch.exception; 2949 } 2950 2951 void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu) 2952 { 2953 struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu); 2954 2955 /* 2956 * If KVM_CAP_EXCEPTION_PAYLOAD is disabled, then (prematurely) deliver 2957 * the pending exception payload when userspace saves *any* vCPU state 2958 * that interacts with exception payloads to avoid breaking userspace. 2959 * 2960 * Architecturally, KVM must not deliver an exception payload until the 2961 * exception is actually injected, e.g. to avoid losing pending #DB 2962 * information (which VMX tracks in the VMCS), and to avoid clobbering 2963 * state if the exception is never injected for whatever reason. But 2964 * if KVM_CAP_EXCEPTION_PAYLOAD isn't enabled, then userspace may or 2965 * may not propagate the payload across save+restore, and so KVM can't 2966 * safely defer delivery of the payload. 2967 */ 2968 if (!vcpu->kvm->arch.exception_payload_enabled && 2969 ex->pending && ex->has_payload) 2970 kvm_deliver_exception_payload(vcpu, ex); 2971 } 2972 2973 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, 2974 struct kvm_vcpu_events *events) 2975 { 2976 struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu); 2977 2978 process_nmi(vcpu); 2979 2980 #ifdef CONFIG_KVM_SMM 2981 if (kvm_check_request(KVM_REQ_SMI, vcpu)) 2982 process_smi(vcpu); 2983 #endif 2984 2985 kvm_handle_exception_payload_quirk(vcpu); 2986 2987 memset(events, 0, sizeof(*events)); 2988 2989 /* 2990 * The API doesn't provide the instruction length for software 2991 * exceptions, so don't report them. As long as the guest RIP 2992 * isn't advanced, we should expect to encounter the exception 2993 * again. 2994 */ 2995 if (!kvm_exception_is_soft(ex->vector)) { 2996 events->exception.injected = ex->injected; 2997 events->exception.pending = ex->pending; 2998 /* 2999 * For ABI compatibility, deliberately conflate 3000 * pending and injected exceptions when 3001 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled. 3002 */ 3003 if (!vcpu->kvm->arch.exception_payload_enabled) 3004 events->exception.injected |= ex->pending; 3005 } 3006 events->exception.nr = ex->vector; 3007 events->exception.has_error_code = ex->has_error_code; 3008 events->exception.error_code = ex->error_code; 3009 events->exception_has_payload = ex->has_payload; 3010 events->exception_payload = ex->payload; 3011 3012 events->interrupt.injected = 3013 vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft; 3014 events->interrupt.nr = vcpu->arch.interrupt.nr; 3015 events->interrupt.shadow = kvm_x86_call(get_interrupt_shadow)(vcpu); 3016 3017 events->nmi.injected = vcpu->arch.nmi_injected; 3018 events->nmi.pending = kvm_get_nr_pending_nmis(vcpu); 3019 events->nmi.masked = kvm_x86_call(get_nmi_mask)(vcpu); 3020 3021 /* events->sipi_vector is never valid when reporting to user space */ 3022 3023 #ifdef CONFIG_KVM_SMM 3024 events->smi.smm = is_smm(vcpu); 3025 events->smi.pending = vcpu->arch.smi_pending; 3026 events->smi.smm_inside_nmi = 3027 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK); 3028 #endif 3029 events->smi.latched_init = kvm_lapic_latched_init(vcpu); 3030 3031 events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING 3032 | KVM_VCPUEVENT_VALID_SHADOW 3033 | KVM_VCPUEVENT_VALID_SMM); 3034 if (vcpu->kvm->arch.exception_payload_enabled) 3035 events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD; 3036 if (vcpu->kvm->arch.triple_fault_event) { 3037 events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu); 3038 events->flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT; 3039 } 3040 } 3041 3042 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, 3043 struct kvm_vcpu_events *events) 3044 { 3045 if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING 3046 | KVM_VCPUEVENT_VALID_SIPI_VECTOR 3047 | KVM_VCPUEVENT_VALID_SHADOW 3048 | KVM_VCPUEVENT_VALID_SMM 3049 | KVM_VCPUEVENT_VALID_PAYLOAD 3050 | KVM_VCPUEVENT_VALID_TRIPLE_FAULT)) 3051 return -EINVAL; 3052 3053 if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) { 3054 if (!vcpu->kvm->arch.exception_payload_enabled) 3055 return -EINVAL; 3056 if (events->exception.pending) 3057 events->exception.injected = 0; 3058 else 3059 events->exception_has_payload = 0; 3060 } else { 3061 events->exception.pending = 0; 3062 events->exception_has_payload = 0; 3063 } 3064 3065 if ((events->exception.injected || events->exception.pending) && 3066 (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR)) 3067 return -EINVAL; 3068 3069 process_nmi(vcpu); 3070 3071 /* 3072 * Flag that userspace is stuffing an exception, the next KVM_RUN will 3073 * morph the exception to a VM-Exit if appropriate. Do this only for 3074 * pending exceptions, already-injected exceptions are not subject to 3075 * intercpetion. Note, userspace that conflates pending and injected 3076 * is hosed, and will incorrectly convert an injected exception into a 3077 * pending exception, which in turn may cause a spurious VM-Exit. 3078 */ 3079 vcpu->arch.exception_from_userspace = events->exception.pending; 3080 3081 vcpu->arch.exception_vmexit.pending = false; 3082 3083 vcpu->arch.exception.injected = events->exception.injected; 3084 vcpu->arch.exception.pending = events->exception.pending; 3085 vcpu->arch.exception.vector = events->exception.nr; 3086 vcpu->arch.exception.has_error_code = events->exception.has_error_code; 3087 vcpu->arch.exception.error_code = events->exception.error_code; 3088 vcpu->arch.exception.has_payload = events->exception_has_payload; 3089 vcpu->arch.exception.payload = events->exception_payload; 3090 3091 vcpu->arch.interrupt.injected = events->interrupt.injected; 3092 vcpu->arch.interrupt.nr = events->interrupt.nr; 3093 vcpu->arch.interrupt.soft = events->interrupt.soft; 3094 if (events->flags & KVM_VCPUEVENT_VALID_SHADOW) 3095 kvm_x86_call(set_interrupt_shadow)(vcpu, 3096 events->interrupt.shadow); 3097 3098 vcpu->arch.nmi_injected = events->nmi.injected; 3099 if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING) { 3100 vcpu->arch.nmi_pending = 0; 3101 atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending); 3102 if (events->nmi.pending) 3103 kvm_make_request(KVM_REQ_NMI, vcpu); 3104 } 3105 kvm_x86_call(set_nmi_mask)(vcpu, events->nmi.masked); 3106 3107 if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR && 3108 lapic_in_kernel(vcpu)) 3109 vcpu->arch.apic->sipi_vector = events->sipi_vector; 3110 3111 if (events->flags & KVM_VCPUEVENT_VALID_SMM) { 3112 #ifdef CONFIG_KVM_SMM 3113 if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) { 3114 kvm_leave_nested(vcpu); 3115 kvm_smm_changed(vcpu, events->smi.smm); 3116 } 3117 3118 vcpu->arch.smi_pending = events->smi.pending; 3119 3120 if (events->smi.smm) { 3121 if (events->smi.smm_inside_nmi) 3122 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK; 3123 else 3124 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK; 3125 } 3126 3127 #else 3128 if (events->smi.smm || events->smi.pending || 3129 events->smi.smm_inside_nmi) 3130 return -EINVAL; 3131 #endif 3132 3133 if (lapic_in_kernel(vcpu)) { 3134 if (events->smi.latched_init) 3135 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); 3136 else 3137 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); 3138 } 3139 } 3140 3141 if (events->flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) { 3142 if (!vcpu->kvm->arch.triple_fault_event) 3143 return -EINVAL; 3144 if (events->triple_fault.pending) 3145 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 3146 else 3147 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 3148 } 3149 3150 kvm_make_request(KVM_REQ_EVENT, vcpu); 3151 3152 return 0; 3153 } 3154 3155 static int kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu, 3156 u8 *state, unsigned int size) 3157 { 3158 /* 3159 * Only copy state for features that are enabled for the guest. The 3160 * state itself isn't problematic, but setting bits in the header for 3161 * features that are supported in *this* host but not exposed to the 3162 * guest can result in KVM_SET_XSAVE failing when live migrating to a 3163 * compatible host without the features that are NOT exposed to the 3164 * guest. 3165 * 3166 * FP+SSE can always be saved/restored via KVM_{G,S}ET_XSAVE, even if 3167 * XSAVE/XCRO are not exposed to the guest, and even if XSAVE isn't 3168 * supported by the host. 3169 */ 3170 u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 | 3171 XFEATURE_MASK_FPSSE; 3172 3173 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 3174 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 3175 3176 fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size, 3177 supported_xcr0, vcpu->arch.pkru); 3178 return 0; 3179 } 3180 3181 static int kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, 3182 struct kvm_xsave *guest_xsave) 3183 { 3184 return kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region, 3185 sizeof(guest_xsave->region)); 3186 } 3187 3188 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, 3189 struct kvm_xsave *guest_xsave) 3190 { 3191 union fpregs_state *xstate = (union fpregs_state *)guest_xsave->region; 3192 3193 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 3194 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 3195 3196 /* 3197 * For backwards compatibility, do not expect disabled features to be in 3198 * their initial state. XSTATE_BV[i] must still be cleared whenever 3199 * XFD[i]=1, or XRSTOR would cause a #NM. 3200 */ 3201 xstate->xsave.header.xfeatures &= ~vcpu->arch.guest_fpu.fpstate->xfd; 3202 3203 return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu, 3204 guest_xsave->region, 3205 kvm_caps.supported_xcr0, 3206 &vcpu->arch.pkru); 3207 } 3208 3209 static int kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu, 3210 struct kvm_xcrs *guest_xcrs) 3211 { 3212 if (vcpu->kvm->arch.has_protected_state && 3213 vcpu->arch.guest_state_protected) 3214 return -EINVAL; 3215 3216 if (!boot_cpu_has(X86_FEATURE_XSAVE)) { 3217 guest_xcrs->nr_xcrs = 0; 3218 return 0; 3219 } 3220 3221 guest_xcrs->nr_xcrs = 1; 3222 guest_xcrs->flags = 0; 3223 guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK; 3224 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0; 3225 return 0; 3226 } 3227 3228 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu, 3229 struct kvm_xcrs *guest_xcrs) 3230 { 3231 int i, r = 0; 3232 3233 if (vcpu->kvm->arch.has_protected_state && 3234 vcpu->arch.guest_state_protected) 3235 return -EINVAL; 3236 3237 if (!boot_cpu_has(X86_FEATURE_XSAVE)) 3238 return -EINVAL; 3239 3240 if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags) 3241 return -EINVAL; 3242 3243 for (i = 0; i < guest_xcrs->nr_xcrs; i++) 3244 /* Only support XCR0 currently */ 3245 if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) { 3246 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK, 3247 guest_xcrs->xcrs[i].value); 3248 break; 3249 } 3250 if (r) 3251 r = -EINVAL; 3252 return r; 3253 } 3254 3255 /* 3256 * kvm_set_guest_paused() indicates to the guest kernel that it has been 3257 * stopped by the hypervisor. This function will be called from the host only. 3258 * EINVAL is returned when the host attempts to set the flag for a guest that 3259 * does not support pv clocks. 3260 */ 3261 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) 3262 { 3263 if (!vcpu->arch.pv_time.active) 3264 return -EINVAL; 3265 vcpu->arch.pvclock_set_guest_stopped_request = true; 3266 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 3267 return 0; 3268 } 3269 3270 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu, 3271 struct kvm_device_attr *attr) 3272 { 3273 int r; 3274 3275 switch (attr->attr) { 3276 case KVM_VCPU_TSC_OFFSET: 3277 r = 0; 3278 break; 3279 default: 3280 r = -ENXIO; 3281 } 3282 3283 return r; 3284 } 3285 3286 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu, 3287 struct kvm_device_attr *attr) 3288 { 3289 u64 __user *uaddr = u64_to_user_ptr(attr->addr); 3290 int r; 3291 3292 switch (attr->attr) { 3293 case KVM_VCPU_TSC_OFFSET: 3294 r = -EFAULT; 3295 if (put_user(vcpu->arch.l1_tsc_offset, uaddr)) 3296 break; 3297 r = 0; 3298 break; 3299 default: 3300 r = -ENXIO; 3301 } 3302 3303 return r; 3304 } 3305 3306 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu, 3307 struct kvm_device_attr *attr) 3308 { 3309 u64 __user *uaddr = u64_to_user_ptr(attr->addr); 3310 struct kvm *kvm = vcpu->kvm; 3311 int r; 3312 3313 switch (attr->attr) { 3314 case KVM_VCPU_TSC_OFFSET: { 3315 u64 offset, tsc, ns; 3316 unsigned long flags; 3317 bool matched; 3318 3319 r = -EFAULT; 3320 if (get_user(offset, uaddr)) 3321 break; 3322 3323 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 3324 3325 matched = (vcpu->arch.virtual_tsc_khz && 3326 kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz && 3327 kvm->arch.last_tsc_offset == offset); 3328 3329 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset; 3330 ns = get_kvmclock_base_ns(); 3331 3332 __kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched, true); 3333 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 3334 3335 r = 0; 3336 break; 3337 } 3338 default: 3339 r = -ENXIO; 3340 } 3341 3342 return r; 3343 } 3344 3345 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu, 3346 unsigned int ioctl, 3347 void __user *argp) 3348 { 3349 struct kvm_device_attr attr; 3350 int r; 3351 3352 if (copy_from_user(&attr, argp, sizeof(attr))) 3353 return -EFAULT; 3354 3355 if (attr.group != KVM_VCPU_TSC_CTRL) 3356 return -ENXIO; 3357 3358 switch (ioctl) { 3359 case KVM_HAS_DEVICE_ATTR: 3360 r = kvm_arch_tsc_has_attr(vcpu, &attr); 3361 break; 3362 case KVM_GET_DEVICE_ATTR: 3363 r = kvm_arch_tsc_get_attr(vcpu, &attr); 3364 break; 3365 case KVM_SET_DEVICE_ATTR: 3366 r = kvm_arch_tsc_set_attr(vcpu, &attr); 3367 break; 3368 } 3369 3370 return r; 3371 } 3372 3373 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 3374 struct kvm_enable_cap *cap) 3375 { 3376 if (cap->flags) 3377 return -EINVAL; 3378 3379 switch (cap->cap) { 3380 #ifdef CONFIG_KVM_HYPERV 3381 case KVM_CAP_HYPERV_SYNIC2: 3382 if (cap->args[0]) 3383 return -EINVAL; 3384 fallthrough; 3385 3386 case KVM_CAP_HYPERV_SYNIC: 3387 if (!irqchip_in_kernel(vcpu->kvm)) 3388 return -EINVAL; 3389 return kvm_hv_activate_synic(vcpu, cap->cap == 3390 KVM_CAP_HYPERV_SYNIC2); 3391 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: 3392 { 3393 int r; 3394 uint16_t vmcs_version; 3395 void __user *user_ptr; 3396 3397 if (!kvm_nested_ops.enabled || 3398 !kvm_nested_ops.enable_evmcs) 3399 return -ENOTTY; 3400 r = kvm_nested_call(enable_evmcs)(vcpu, &vmcs_version); 3401 if (!r) { 3402 user_ptr = (void __user *)(uintptr_t)cap->args[0]; 3403 if (copy_to_user(user_ptr, &vmcs_version, 3404 sizeof(vmcs_version))) 3405 r = -EFAULT; 3406 } 3407 return r; 3408 } 3409 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: 3410 if (!kvm_x86_ops.enable_l2_tlb_flush) 3411 return -ENOTTY; 3412 3413 return kvm_x86_call(enable_l2_tlb_flush)(vcpu); 3414 3415 case KVM_CAP_HYPERV_ENFORCE_CPUID: 3416 return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]); 3417 #endif 3418 3419 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID: 3420 vcpu->arch.pv_cpuid.enforce = cap->args[0]; 3421 return 0; 3422 default: 3423 return -EINVAL; 3424 } 3425 } 3426 3427 long kvm_arch_vcpu_ioctl(struct file *filp, 3428 unsigned int ioctl, unsigned long arg) 3429 { 3430 struct kvm_vcpu *vcpu = filp->private_data; 3431 void __user *argp = (void __user *)arg; 3432 int r; 3433 union { 3434 struct kvm_sregs2 *sregs2; 3435 struct kvm_lapic_state *lapic; 3436 struct kvm_xsave *xsave; 3437 struct kvm_xcrs *xcrs; 3438 void *buffer; 3439 } u; 3440 3441 vcpu_load(vcpu); 3442 3443 u.buffer = NULL; 3444 switch (ioctl) { 3445 case KVM_GET_LAPIC: { 3446 r = -EINVAL; 3447 if (!lapic_in_kernel(vcpu)) 3448 goto out; 3449 u.lapic = kzalloc_obj(struct kvm_lapic_state); 3450 3451 r = -ENOMEM; 3452 if (!u.lapic) 3453 goto out; 3454 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic); 3455 if (r) 3456 goto out; 3457 r = -EFAULT; 3458 if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state))) 3459 goto out; 3460 r = 0; 3461 break; 3462 } 3463 case KVM_SET_LAPIC: { 3464 r = -EINVAL; 3465 if (!lapic_in_kernel(vcpu)) 3466 goto out; 3467 u.lapic = memdup_user(argp, sizeof(*u.lapic)); 3468 if (IS_ERR(u.lapic)) { 3469 r = PTR_ERR(u.lapic); 3470 goto out_nofree; 3471 } 3472 3473 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic); 3474 break; 3475 } 3476 case KVM_INTERRUPT: { 3477 struct kvm_interrupt irq; 3478 3479 r = -EFAULT; 3480 if (copy_from_user(&irq, argp, sizeof(irq))) 3481 goto out; 3482 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); 3483 break; 3484 } 3485 case KVM_NMI: { 3486 r = kvm_vcpu_ioctl_nmi(vcpu); 3487 break; 3488 } 3489 case KVM_SMI: { 3490 r = kvm_inject_smi(vcpu); 3491 break; 3492 } 3493 case KVM_SET_CPUID: { 3494 struct kvm_cpuid __user *cpuid_arg = argp; 3495 struct kvm_cpuid cpuid; 3496 3497 r = -EFAULT; 3498 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 3499 goto out; 3500 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); 3501 break; 3502 } 3503 case KVM_SET_CPUID2: { 3504 struct kvm_cpuid2 __user *cpuid_arg = argp; 3505 struct kvm_cpuid2 cpuid; 3506 3507 r = -EFAULT; 3508 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 3509 goto out; 3510 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, 3511 cpuid_arg->entries); 3512 break; 3513 } 3514 case KVM_GET_CPUID2: { 3515 struct kvm_cpuid2 __user *cpuid_arg = argp; 3516 struct kvm_cpuid2 cpuid; 3517 3518 r = -EFAULT; 3519 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 3520 goto out; 3521 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, 3522 cpuid_arg->entries); 3523 if (r) 3524 goto out; 3525 r = -EFAULT; 3526 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid))) 3527 goto out; 3528 r = 0; 3529 break; 3530 } 3531 case KVM_GET_MSRS: 3532 r = kvm_get_msrs(vcpu, argp); 3533 break; 3534 case KVM_SET_MSRS: 3535 r = kvm_set_msrs(vcpu, argp); 3536 break; 3537 case KVM_GET_ONE_REG: 3538 case KVM_SET_ONE_REG: 3539 r = kvm_get_set_one_reg(vcpu, ioctl, argp); 3540 break; 3541 case KVM_GET_REG_LIST: 3542 r = kvm_get_reg_list(vcpu, argp); 3543 break; 3544 case KVM_TPR_ACCESS_REPORTING: { 3545 struct kvm_tpr_access_ctl tac; 3546 3547 r = -EFAULT; 3548 if (copy_from_user(&tac, argp, sizeof(tac))) 3549 goto out; 3550 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac); 3551 if (r) 3552 goto out; 3553 r = -EFAULT; 3554 if (copy_to_user(argp, &tac, sizeof(tac))) 3555 goto out; 3556 r = 0; 3557 break; 3558 }; 3559 case KVM_SET_VAPIC_ADDR: { 3560 struct kvm_vapic_addr va; 3561 int idx; 3562 3563 r = -EINVAL; 3564 if (!lapic_in_kernel(vcpu)) 3565 goto out; 3566 r = -EFAULT; 3567 if (copy_from_user(&va, argp, sizeof(va))) 3568 goto out; 3569 idx = srcu_read_lock(&vcpu->kvm->srcu); 3570 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr); 3571 srcu_read_unlock(&vcpu->kvm->srcu, idx); 3572 break; 3573 } 3574 case KVM_X86_SETUP_MCE: { 3575 u64 mcg_cap; 3576 3577 r = -EFAULT; 3578 if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap))) 3579 goto out; 3580 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap); 3581 break; 3582 } 3583 case KVM_X86_SET_MCE: { 3584 struct kvm_x86_mce mce; 3585 3586 r = -EFAULT; 3587 if (copy_from_user(&mce, argp, sizeof(mce))) 3588 goto out; 3589 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce); 3590 break; 3591 } 3592 case KVM_GET_VCPU_EVENTS: { 3593 struct kvm_vcpu_events events; 3594 3595 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events); 3596 3597 r = -EFAULT; 3598 if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events))) 3599 break; 3600 r = 0; 3601 break; 3602 } 3603 case KVM_SET_VCPU_EVENTS: { 3604 struct kvm_vcpu_events events; 3605 3606 r = -EFAULT; 3607 if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events))) 3608 break; 3609 3610 kvm_vcpu_srcu_read_lock(vcpu); 3611 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events); 3612 kvm_vcpu_srcu_read_unlock(vcpu); 3613 break; 3614 } 3615 case KVM_GET_DEBUGREGS: { 3616 struct kvm_debugregs dbgregs; 3617 3618 r = kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs); 3619 if (r < 0) 3620 break; 3621 3622 r = -EFAULT; 3623 if (copy_to_user(argp, &dbgregs, 3624 sizeof(struct kvm_debugregs))) 3625 break; 3626 r = 0; 3627 break; 3628 } 3629 case KVM_SET_DEBUGREGS: { 3630 struct kvm_debugregs dbgregs; 3631 3632 r = -EFAULT; 3633 if (copy_from_user(&dbgregs, argp, 3634 sizeof(struct kvm_debugregs))) 3635 break; 3636 3637 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs); 3638 break; 3639 } 3640 case KVM_GET_XSAVE: { 3641 r = -EINVAL; 3642 if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave)) 3643 break; 3644 3645 u.xsave = kzalloc_obj(struct kvm_xsave); 3646 r = -ENOMEM; 3647 if (!u.xsave) 3648 break; 3649 3650 r = kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave); 3651 if (r < 0) 3652 break; 3653 3654 r = -EFAULT; 3655 if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave))) 3656 break; 3657 r = 0; 3658 break; 3659 } 3660 case KVM_SET_XSAVE: { 3661 int size = vcpu->arch.guest_fpu.uabi_size; 3662 3663 u.xsave = memdup_user(argp, size); 3664 if (IS_ERR(u.xsave)) { 3665 r = PTR_ERR(u.xsave); 3666 goto out_nofree; 3667 } 3668 3669 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave); 3670 break; 3671 } 3672 3673 case KVM_GET_XSAVE2: { 3674 int size = vcpu->arch.guest_fpu.uabi_size; 3675 3676 u.xsave = kzalloc(size, GFP_KERNEL); 3677 r = -ENOMEM; 3678 if (!u.xsave) 3679 break; 3680 3681 r = kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size); 3682 if (r < 0) 3683 break; 3684 3685 r = -EFAULT; 3686 if (copy_to_user(argp, u.xsave, size)) 3687 break; 3688 3689 r = 0; 3690 break; 3691 } 3692 3693 case KVM_GET_XCRS: { 3694 u.xcrs = kzalloc_obj(struct kvm_xcrs); 3695 r = -ENOMEM; 3696 if (!u.xcrs) 3697 break; 3698 3699 r = kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs); 3700 if (r < 0) 3701 break; 3702 3703 r = -EFAULT; 3704 if (copy_to_user(argp, u.xcrs, 3705 sizeof(struct kvm_xcrs))) 3706 break; 3707 r = 0; 3708 break; 3709 } 3710 case KVM_SET_XCRS: { 3711 u.xcrs = memdup_user(argp, sizeof(*u.xcrs)); 3712 if (IS_ERR(u.xcrs)) { 3713 r = PTR_ERR(u.xcrs); 3714 goto out_nofree; 3715 } 3716 3717 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs); 3718 break; 3719 } 3720 case KVM_SET_TSC_KHZ: { 3721 u32 user_tsc_khz; 3722 3723 r = -EINVAL; 3724 3725 if (vcpu->arch.guest_tsc_protected) 3726 goto out; 3727 3728 user_tsc_khz = (u32)arg; 3729 3730 if (kvm_caps.has_tsc_control && 3731 user_tsc_khz > kvm_caps.max_guest_tsc_khz) 3732 goto out; 3733 3734 if (user_tsc_khz == 0) 3735 user_tsc_khz = tsc_khz; 3736 3737 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz)) 3738 r = 0; 3739 3740 goto out; 3741 } 3742 case KVM_GET_TSC_KHZ: { 3743 r = vcpu->arch.virtual_tsc_khz; 3744 goto out; 3745 } 3746 case KVM_KVMCLOCK_CTRL: { 3747 r = kvm_set_guest_paused(vcpu); 3748 goto out; 3749 } 3750 case KVM_ENABLE_CAP: { 3751 struct kvm_enable_cap cap; 3752 3753 r = -EFAULT; 3754 if (copy_from_user(&cap, argp, sizeof(cap))) 3755 goto out; 3756 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 3757 break; 3758 } 3759 case KVM_GET_NESTED_STATE: { 3760 struct kvm_nested_state __user *user_kvm_nested_state = argp; 3761 u32 user_data_size; 3762 3763 r = -EINVAL; 3764 if (!kvm_nested_ops.enabled) 3765 break; 3766 3767 BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size)); 3768 r = -EFAULT; 3769 if (get_user(user_data_size, &user_kvm_nested_state->size)) 3770 break; 3771 3772 r = kvm_nested_call(get_state)(vcpu, user_kvm_nested_state, user_data_size); 3773 if (r < 0) 3774 break; 3775 3776 if (r > user_data_size) { 3777 if (put_user(r, &user_kvm_nested_state->size)) 3778 r = -EFAULT; 3779 else 3780 r = -E2BIG; 3781 break; 3782 } 3783 3784 r = 0; 3785 break; 3786 } 3787 case KVM_SET_NESTED_STATE: { 3788 struct kvm_nested_state __user *user_kvm_nested_state = argp; 3789 struct kvm_nested_state kvm_state; 3790 int idx; 3791 3792 r = -EINVAL; 3793 if (!kvm_nested_ops.enabled) 3794 break; 3795 3796 r = -EFAULT; 3797 if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state))) 3798 break; 3799 3800 r = -EINVAL; 3801 if (kvm_state.size < sizeof(kvm_state)) 3802 break; 3803 3804 if (kvm_state.flags & 3805 ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE 3806 | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING 3807 | KVM_STATE_NESTED_GIF_SET)) 3808 break; 3809 3810 /* nested_run_pending implies guest_mode. */ 3811 if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING) 3812 && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE)) 3813 break; 3814 3815 idx = srcu_read_lock(&vcpu->kvm->srcu); 3816 r = kvm_nested_call(set_state)(vcpu, user_kvm_nested_state, &kvm_state); 3817 srcu_read_unlock(&vcpu->kvm->srcu, idx); 3818 break; 3819 } 3820 #ifdef CONFIG_KVM_HYPERV 3821 case KVM_GET_SUPPORTED_HV_CPUID: 3822 r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp); 3823 break; 3824 #endif 3825 #ifdef CONFIG_KVM_XEN 3826 case KVM_XEN_VCPU_GET_ATTR: { 3827 struct kvm_xen_vcpu_attr xva; 3828 3829 r = -EFAULT; 3830 if (copy_from_user(&xva, argp, sizeof(xva))) 3831 goto out; 3832 r = kvm_xen_vcpu_get_attr(vcpu, &xva); 3833 if (!r && copy_to_user(argp, &xva, sizeof(xva))) 3834 r = -EFAULT; 3835 break; 3836 } 3837 case KVM_XEN_VCPU_SET_ATTR: { 3838 struct kvm_xen_vcpu_attr xva; 3839 3840 r = -EFAULT; 3841 if (copy_from_user(&xva, argp, sizeof(xva))) 3842 goto out; 3843 r = kvm_xen_vcpu_set_attr(vcpu, &xva); 3844 break; 3845 } 3846 #endif 3847 case KVM_GET_SREGS2: { 3848 r = -EINVAL; 3849 if (vcpu->kvm->arch.has_protected_state && 3850 vcpu->arch.guest_state_protected) 3851 goto out; 3852 3853 u.sregs2 = kzalloc_obj(struct kvm_sregs2); 3854 r = -ENOMEM; 3855 if (!u.sregs2) 3856 goto out; 3857 kvm_vcpu_ioctl_x86_get_sregs2(vcpu, u.sregs2); 3858 r = -EFAULT; 3859 if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2))) 3860 goto out; 3861 r = 0; 3862 break; 3863 } 3864 case KVM_SET_SREGS2: { 3865 r = -EINVAL; 3866 if (vcpu->kvm->arch.has_protected_state && 3867 vcpu->arch.guest_state_protected) 3868 goto out; 3869 3870 u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2)); 3871 if (IS_ERR(u.sregs2)) { 3872 r = PTR_ERR(u.sregs2); 3873 u.sregs2 = NULL; 3874 goto out; 3875 } 3876 r = kvm_vcpu_ioctl_x86_set_sregs2(vcpu, u.sregs2); 3877 break; 3878 } 3879 case KVM_HAS_DEVICE_ATTR: 3880 case KVM_GET_DEVICE_ATTR: 3881 case KVM_SET_DEVICE_ATTR: 3882 r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp); 3883 break; 3884 case KVM_MEMORY_ENCRYPT_OP: 3885 r = -ENOTTY; 3886 if (!kvm_x86_ops.vcpu_mem_enc_ioctl) 3887 goto out; 3888 r = kvm_x86_ops.vcpu_mem_enc_ioctl(vcpu, argp); 3889 break; 3890 default: 3891 r = -EINVAL; 3892 } 3893 out: 3894 kfree(u.buffer); 3895 out_nofree: 3896 vcpu_put(vcpu); 3897 return r; 3898 } 3899 3900 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 3901 { 3902 return VM_FAULT_SIGBUS; 3903 } 3904 3905 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) 3906 { 3907 int ret; 3908 3909 if (addr > (unsigned int)(-3 * PAGE_SIZE)) 3910 return -EINVAL; 3911 ret = kvm_x86_call(set_tss_addr)(kvm, addr); 3912 return ret; 3913 } 3914 3915 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm, 3916 u64 ident_addr) 3917 { 3918 return kvm_x86_call(set_identity_map_addr)(kvm, ident_addr); 3919 } 3920 3921 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, 3922 unsigned long kvm_nr_mmu_pages) 3923 { 3924 if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) 3925 return -EINVAL; 3926 3927 mutex_lock(&kvm->slots_lock); 3928 3929 kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); 3930 kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; 3931 3932 mutex_unlock(&kvm->slots_lock); 3933 return 0; 3934 } 3935 3936 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 3937 { 3938 3939 /* 3940 * Flush all CPUs' dirty log buffers to the dirty_bitmap. Called 3941 * before reporting dirty_bitmap to userspace. KVM flushes the buffers 3942 * on all VM-Exits, thus we only need to kick running vCPUs to force a 3943 * VM-Exit. 3944 */ 3945 struct kvm_vcpu *vcpu; 3946 unsigned long i; 3947 3948 if (!kvm->arch.cpu_dirty_log_size) 3949 return; 3950 3951 kvm_for_each_vcpu(i, vcpu, kvm) 3952 kvm_vcpu_kick(vcpu); 3953 } 3954 3955 void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx) 3956 { 3957 u16 pml_tail_index; 3958 u64 *pml_buf; 3959 int i; 3960 3961 /* 3962 * PML index always points to the next available PML buffer entity 3963 * unless PML log has just overflowed. 3964 */ 3965 pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; 3966 3967 /* 3968 * PML log is written backwards: the CPU first writes the entry 511 3969 * then the entry 510, and so on. 3970 * 3971 * Read the entries in the same order they were written, to ensure that 3972 * the dirty ring is filled in the same order the CPU wrote them. 3973 */ 3974 pml_buf = page_address(vcpu->arch.pml_page); 3975 3976 for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { 3977 u64 gpa; 3978 3979 gpa = pml_buf[i]; 3980 WARN_ON_ONCE(gpa & (PAGE_SIZE - 1)); 3981 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); 3982 } 3983 } 3984 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_flush_pml_buffer); 3985 3986 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 3987 struct kvm_enable_cap *cap) 3988 { 3989 int r; 3990 3991 if (cap->flags) 3992 return -EINVAL; 3993 3994 switch (cap->cap) { 3995 case KVM_CAP_DISABLE_QUIRKS2: 3996 r = -EINVAL; 3997 if (cap->args[0] & ~kvm_caps.supported_quirks) 3998 break; 3999 fallthrough; 4000 case KVM_CAP_DISABLE_QUIRKS: 4001 mutex_lock(&kvm->lock); 4002 WRITE_ONCE(kvm->arch.disabled_quirks, 4003 kvm->arch.disabled_quirks | (cap->args[0] & kvm_caps.supported_quirks)); 4004 mutex_unlock(&kvm->lock); 4005 r = 0; 4006 break; 4007 case KVM_CAP_SPLIT_IRQCHIP: { 4008 mutex_lock(&kvm->lock); 4009 r = -EINVAL; 4010 if (cap->args[0] > KVM_MAX_IRQ_ROUTES) 4011 goto split_irqchip_unlock; 4012 r = -EEXIST; 4013 if (irqchip_in_kernel(kvm)) 4014 goto split_irqchip_unlock; 4015 if (kvm->created_vcpus) 4016 goto split_irqchip_unlock; 4017 /* Pairs with irqchip_in_kernel. */ 4018 smp_wmb(); 4019 kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT; 4020 kvm->arch.nr_reserved_ioapic_pins = cap->args[0]; 4021 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT); 4022 r = 0; 4023 split_irqchip_unlock: 4024 mutex_unlock(&kvm->lock); 4025 break; 4026 } 4027 case KVM_CAP_X2APIC_API: 4028 r = -EINVAL; 4029 if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS) 4030 break; 4031 4032 if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && 4033 (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)) 4034 break; 4035 4036 if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && 4037 !irqchip_split(kvm)) 4038 break; 4039 4040 if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS) 4041 kvm->arch.x2apic_format = true; 4042 if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) 4043 kvm->arch.x2apic_broadcast_quirk_disabled = true; 4044 4045 if (cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) 4046 kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_ENABLED; 4047 if (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) 4048 kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_DISABLED; 4049 4050 r = 0; 4051 break; 4052 case KVM_CAP_X86_DISABLE_EXITS: 4053 r = -EINVAL; 4054 if (cap->args[0] & ~kvm_get_allowed_disable_exits()) 4055 break; 4056 4057 mutex_lock(&kvm->lock); 4058 if (kvm->created_vcpus) 4059 goto disable_exits_unlock; 4060 4061 #define SMT_RSB_MSG "This processor is affected by the Cross-Thread Return Predictions vulnerability. " \ 4062 "KVM_CAP_X86_DISABLE_EXITS should only be used with SMT disabled or trusted guests." 4063 4064 if (!mitigate_smt_rsb && boot_cpu_has_bug(X86_BUG_SMT_RSB) && 4065 cpu_smt_possible() && 4066 (cap->args[0] & ~(KVM_X86_DISABLE_EXITS_PAUSE | 4067 KVM_X86_DISABLE_EXITS_APERFMPERF))) 4068 pr_warn_once(SMT_RSB_MSG); 4069 4070 kvm_disable_exits(kvm, cap->args[0]); 4071 r = 0; 4072 disable_exits_unlock: 4073 mutex_unlock(&kvm->lock); 4074 break; 4075 case KVM_CAP_MSR_PLATFORM_INFO: 4076 kvm->arch.guest_can_read_msr_platform_info = cap->args[0]; 4077 r = 0; 4078 break; 4079 case KVM_CAP_EXCEPTION_PAYLOAD: 4080 kvm->arch.exception_payload_enabled = cap->args[0]; 4081 r = 0; 4082 break; 4083 case KVM_CAP_X86_TRIPLE_FAULT_EVENT: 4084 kvm->arch.triple_fault_event = cap->args[0]; 4085 r = 0; 4086 break; 4087 case KVM_CAP_X86_USER_SPACE_MSR: 4088 r = -EINVAL; 4089 if (cap->args[0] & ~KVM_MSR_EXIT_REASON_VALID_MASK) 4090 break; 4091 kvm->arch.user_space_msr_mask = cap->args[0]; 4092 r = 0; 4093 break; 4094 case KVM_CAP_X86_BUS_LOCK_EXIT: 4095 r = -EINVAL; 4096 if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE) 4097 break; 4098 4099 if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) && 4100 (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)) 4101 break; 4102 4103 if (kvm_caps.has_bus_lock_exit && 4104 cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT) 4105 kvm->arch.bus_lock_detection_enabled = true; 4106 r = 0; 4107 break; 4108 #ifdef CONFIG_X86_SGX_KVM 4109 case KVM_CAP_SGX_ATTRIBUTE: { 4110 unsigned long allowed_attributes = 0; 4111 4112 r = sgx_set_attribute(&allowed_attributes, cap->args[0]); 4113 if (r) 4114 break; 4115 4116 /* KVM only supports the PROVISIONKEY privileged attribute. */ 4117 if ((allowed_attributes & SGX_ATTR_PROVISIONKEY) && 4118 !(allowed_attributes & ~SGX_ATTR_PROVISIONKEY)) 4119 kvm->arch.sgx_provisioning_allowed = true; 4120 else 4121 r = -EINVAL; 4122 break; 4123 } 4124 #endif 4125 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: 4126 r = -EINVAL; 4127 if (!kvm_x86_ops.vm_copy_enc_context_from) 4128 break; 4129 4130 r = kvm_x86_call(vm_copy_enc_context_from)(kvm, cap->args[0]); 4131 break; 4132 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM: 4133 r = -EINVAL; 4134 if (!kvm_x86_ops.vm_move_enc_context_from) 4135 break; 4136 4137 r = kvm_x86_call(vm_move_enc_context_from)(kvm, cap->args[0]); 4138 break; 4139 case KVM_CAP_EXIT_HYPERCALL: 4140 if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) { 4141 r = -EINVAL; 4142 break; 4143 } 4144 kvm->arch.hypercall_exit_enabled = cap->args[0]; 4145 r = 0; 4146 break; 4147 case KVM_CAP_EXIT_ON_EMULATION_FAILURE: 4148 r = -EINVAL; 4149 if (cap->args[0] & ~1) 4150 break; 4151 kvm->arch.exit_on_emulation_error = cap->args[0]; 4152 r = 0; 4153 break; 4154 case KVM_CAP_PMU_CAPABILITY: 4155 r = -EINVAL; 4156 if (!enable_pmu || (cap->args[0] & ~KVM_CAP_PMU_VALID_MASK)) 4157 break; 4158 4159 if (kvm->arch.has_protected_pmu && 4160 cap->args[0] != KVM_PMU_CAP_DISABLE) 4161 break; 4162 4163 mutex_lock(&kvm->lock); 4164 if (!kvm->created_vcpus && !kvm->arch.created_mediated_pmu) { 4165 kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE); 4166 r = 0; 4167 } 4168 mutex_unlock(&kvm->lock); 4169 break; 4170 case KVM_CAP_MAX_VCPU_ID: 4171 r = -EINVAL; 4172 if (cap->args[0] > KVM_MAX_VCPU_IDS) 4173 break; 4174 4175 mutex_lock(&kvm->lock); 4176 if (kvm->arch.bsp_vcpu_id > cap->args[0]) { 4177 ; 4178 } else if (kvm->arch.max_vcpu_ids == cap->args[0]) { 4179 r = 0; 4180 } else if (!kvm->arch.max_vcpu_ids) { 4181 kvm->arch.max_vcpu_ids = cap->args[0]; 4182 r = 0; 4183 } 4184 mutex_unlock(&kvm->lock); 4185 break; 4186 case KVM_CAP_X86_NOTIFY_VMEXIT: 4187 r = -EINVAL; 4188 if ((u32)cap->args[0] & ~KVM_X86_NOTIFY_VMEXIT_VALID_BITS) 4189 break; 4190 if (!kvm_caps.has_notify_vmexit) 4191 break; 4192 if (!((u32)cap->args[0] & KVM_X86_NOTIFY_VMEXIT_ENABLED)) 4193 break; 4194 mutex_lock(&kvm->lock); 4195 if (!kvm->created_vcpus) { 4196 kvm->arch.notify_window = cap->args[0] >> 32; 4197 kvm->arch.notify_vmexit_flags = (u32)cap->args[0]; 4198 r = 0; 4199 } 4200 mutex_unlock(&kvm->lock); 4201 break; 4202 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES: 4203 r = -EINVAL; 4204 4205 /* 4206 * Since the risk of disabling NX hugepages is a guest crashing 4207 * the system, ensure the userspace process has permission to 4208 * reboot the system. 4209 * 4210 * Note that unlike the reboot() syscall, the process must have 4211 * this capability in the root namespace because exposing 4212 * /dev/kvm into a container does not limit the scope of the 4213 * iTLB multihit bug to that container. In other words, 4214 * this must use capable(), not ns_capable(). 4215 */ 4216 if (!capable(CAP_SYS_BOOT)) { 4217 r = -EPERM; 4218 break; 4219 } 4220 4221 if (cap->args[0]) 4222 break; 4223 4224 mutex_lock(&kvm->lock); 4225 if (!kvm->created_vcpus) { 4226 kvm->arch.disable_nx_huge_pages = true; 4227 r = 0; 4228 } 4229 mutex_unlock(&kvm->lock); 4230 break; 4231 case KVM_CAP_X86_APIC_BUS_CYCLES_NS: { 4232 u64 bus_cycle_ns = cap->args[0]; 4233 u64 unused; 4234 4235 /* 4236 * Guard against overflow in tmict_to_ns(). 128 is the highest 4237 * divide value that can be programmed in APIC_TDCR. 4238 */ 4239 r = -EINVAL; 4240 if (!bus_cycle_ns || 4241 check_mul_overflow((u64)U32_MAX * 128, bus_cycle_ns, &unused)) 4242 break; 4243 4244 r = 0; 4245 mutex_lock(&kvm->lock); 4246 if (!irqchip_in_kernel(kvm)) 4247 r = -ENXIO; 4248 else if (kvm->created_vcpus) 4249 r = -EINVAL; 4250 else 4251 kvm->arch.apic_bus_cycle_ns = bus_cycle_ns; 4252 mutex_unlock(&kvm->lock); 4253 break; 4254 } 4255 default: 4256 r = -EINVAL; 4257 break; 4258 } 4259 return r; 4260 } 4261 4262 #ifdef CONFIG_KVM_COMPAT 4263 /* for KVM_X86_SET_MSR_FILTER */ 4264 struct kvm_msr_filter_range_compat { 4265 __u32 flags; 4266 __u32 nmsrs; 4267 __u32 base; 4268 __u32 bitmap; 4269 }; 4270 4271 struct kvm_msr_filter_compat { 4272 __u32 flags; 4273 struct kvm_msr_filter_range_compat ranges[KVM_MSR_FILTER_MAX_RANGES]; 4274 }; 4275 4276 #define KVM_X86_SET_MSR_FILTER_COMPAT _IOW(KVMIO, 0xc6, struct kvm_msr_filter_compat) 4277 4278 long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl, 4279 unsigned long arg) 4280 { 4281 void __user *argp = (void __user *)arg; 4282 struct kvm *kvm = filp->private_data; 4283 long r = -ENOTTY; 4284 4285 switch (ioctl) { 4286 case KVM_X86_SET_MSR_FILTER_COMPAT: { 4287 struct kvm_msr_filter __user *user_msr_filter = argp; 4288 struct kvm_msr_filter_compat filter_compat; 4289 struct kvm_msr_filter filter; 4290 int i; 4291 4292 if (copy_from_user(&filter_compat, user_msr_filter, 4293 sizeof(filter_compat))) 4294 return -EFAULT; 4295 4296 filter.flags = filter_compat.flags; 4297 for (i = 0; i < ARRAY_SIZE(filter.ranges); i++) { 4298 struct kvm_msr_filter_range_compat *cr; 4299 4300 cr = &filter_compat.ranges[i]; 4301 filter.ranges[i] = (struct kvm_msr_filter_range) { 4302 .flags = cr->flags, 4303 .nmsrs = cr->nmsrs, 4304 .base = cr->base, 4305 .bitmap = (__u8 *)(ulong)cr->bitmap, 4306 }; 4307 } 4308 4309 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter); 4310 break; 4311 } 4312 } 4313 4314 return r; 4315 } 4316 #endif 4317 4318 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER 4319 static int kvm_arch_suspend_notifier(struct kvm *kvm) 4320 { 4321 struct kvm_vcpu *vcpu; 4322 unsigned long i; 4323 4324 /* 4325 * Ignore the return, marking the guest paused only "fails" if the vCPU 4326 * isn't using kvmclock; continuing on is correct and desirable. 4327 */ 4328 kvm_for_each_vcpu(i, vcpu, kvm) 4329 (void)kvm_set_guest_paused(vcpu); 4330 4331 return NOTIFY_DONE; 4332 } 4333 4334 int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state) 4335 { 4336 switch (state) { 4337 case PM_HIBERNATION_PREPARE: 4338 case PM_SUSPEND_PREPARE: 4339 return kvm_arch_suspend_notifier(kvm); 4340 } 4341 4342 return NOTIFY_DONE; 4343 } 4344 #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */ 4345 4346 static int kvm_vm_ioctl_get_clock(struct kvm *kvm, void __user *argp) 4347 { 4348 struct kvm_clock_data data = { 0 }; 4349 4350 get_kvmclock(kvm, &data); 4351 if (copy_to_user(argp, &data, sizeof(data))) 4352 return -EFAULT; 4353 4354 return 0; 4355 } 4356 4357 static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp) 4358 { 4359 struct kvm_arch *ka = &kvm->arch; 4360 struct kvm_clock_data data; 4361 u64 now_raw_ns; 4362 4363 if (copy_from_user(&data, argp, sizeof(data))) 4364 return -EFAULT; 4365 4366 /* 4367 * Only KVM_CLOCK_REALTIME is used, but allow passing the 4368 * result of KVM_GET_CLOCK back to KVM_SET_CLOCK. 4369 */ 4370 if (data.flags & ~KVM_CLOCK_VALID_FLAGS) 4371 return -EINVAL; 4372 4373 kvm_hv_request_tsc_page_update(kvm); 4374 kvm_start_pvclock_update(kvm); 4375 pvclock_update_vm_gtod_copy(kvm); 4376 4377 /* 4378 * This pairs with kvm_guest_time_update(): when masterclock is 4379 * in use, we use master_kernel_ns + kvmclock_offset to set 4380 * unsigned 'system_time' so if we use get_kvmclock_ns() (which 4381 * is slightly ahead) here we risk going negative on unsigned 4382 * 'system_time' when 'data.clock' is very small. 4383 */ 4384 if (data.flags & KVM_CLOCK_REALTIME) { 4385 u64 now_real_ns = ktime_get_real_ns(); 4386 4387 /* 4388 * Avoid stepping the kvmclock backwards. 4389 */ 4390 if (now_real_ns > data.realtime) 4391 data.clock += now_real_ns - data.realtime; 4392 } 4393 4394 if (ka->use_master_clock) 4395 now_raw_ns = ka->master_kernel_ns; 4396 else 4397 now_raw_ns = get_kvmclock_base_ns(); 4398 ka->kvmclock_offset = data.clock - now_raw_ns; 4399 kvm_end_pvclock_update(kvm); 4400 return 0; 4401 } 4402 4403 long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, 4404 unsigned long arg) 4405 { 4406 struct kvm_vcpu *vcpu = filp->private_data; 4407 void __user *argp = (void __user *)arg; 4408 4409 if (ioctl == KVM_MEMORY_ENCRYPT_OP && 4410 kvm_x86_ops.vcpu_mem_enc_unlocked_ioctl) 4411 return kvm_x86_call(vcpu_mem_enc_unlocked_ioctl)(vcpu, argp); 4412 4413 return -ENOIOCTLCMD; 4414 } 4415 4416 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) 4417 { 4418 struct kvm *kvm = filp->private_data; 4419 void __user *argp = (void __user *)arg; 4420 int r = -ENOTTY; 4421 4422 #ifdef CONFIG_KVM_IOAPIC 4423 /* 4424 * This union makes it completely explicit to gcc-3.x 4425 * that these three variables' stack usage should be 4426 * combined, not added together. 4427 */ 4428 union { 4429 struct kvm_pit_state ps; 4430 struct kvm_pit_state2 ps2; 4431 struct kvm_pit_config pit_config; 4432 } u; 4433 #endif 4434 4435 switch (ioctl) { 4436 case KVM_SET_TSS_ADDR: 4437 r = kvm_vm_ioctl_set_tss_addr(kvm, arg); 4438 break; 4439 case KVM_SET_IDENTITY_MAP_ADDR: { 4440 u64 ident_addr; 4441 4442 mutex_lock(&kvm->lock); 4443 r = -EINVAL; 4444 if (kvm->created_vcpus) 4445 goto set_identity_unlock; 4446 r = -EFAULT; 4447 if (copy_from_user(&ident_addr, argp, sizeof(ident_addr))) 4448 goto set_identity_unlock; 4449 r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr); 4450 set_identity_unlock: 4451 mutex_unlock(&kvm->lock); 4452 break; 4453 } 4454 case KVM_SET_NR_MMU_PAGES: 4455 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); 4456 break; 4457 #ifdef CONFIG_KVM_IOAPIC 4458 case KVM_CREATE_IRQCHIP: { 4459 mutex_lock(&kvm->lock); 4460 4461 r = -EEXIST; 4462 if (irqchip_in_kernel(kvm)) 4463 goto create_irqchip_unlock; 4464 4465 /* 4466 * Disallow an in-kernel I/O APIC if the VM has protected EOIs, 4467 * i.e. if KVM can't intercept EOIs and thus can't properly 4468 * emulate level-triggered interrupts. 4469 */ 4470 r = -ENOTTY; 4471 if (kvm->arch.has_protected_eoi) 4472 goto create_irqchip_unlock; 4473 4474 r = -EINVAL; 4475 if (kvm->created_vcpus) 4476 goto create_irqchip_unlock; 4477 4478 r = kvm_pic_init(kvm); 4479 if (r) 4480 goto create_irqchip_unlock; 4481 4482 r = kvm_ioapic_init(kvm); 4483 if (r) { 4484 kvm_pic_destroy(kvm); 4485 goto create_irqchip_unlock; 4486 } 4487 4488 r = kvm_setup_default_ioapic_and_pic_routing(kvm); 4489 if (r) { 4490 kvm_ioapic_destroy(kvm); 4491 kvm_pic_destroy(kvm); 4492 goto create_irqchip_unlock; 4493 } 4494 /* Write kvm->irq_routing before enabling irqchip_in_kernel. */ 4495 smp_wmb(); 4496 kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL; 4497 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT); 4498 create_irqchip_unlock: 4499 mutex_unlock(&kvm->lock); 4500 break; 4501 } 4502 case KVM_CREATE_PIT: 4503 u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY; 4504 goto create_pit; 4505 case KVM_CREATE_PIT2: 4506 r = -EFAULT; 4507 if (copy_from_user(&u.pit_config, argp, 4508 sizeof(struct kvm_pit_config))) 4509 goto out; 4510 create_pit: 4511 mutex_lock(&kvm->lock); 4512 r = -EEXIST; 4513 if (kvm->arch.vpit) 4514 goto create_pit_unlock; 4515 r = -ENOENT; 4516 if (!pic_in_kernel(kvm)) 4517 goto create_pit_unlock; 4518 r = -ENOMEM; 4519 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags); 4520 if (kvm->arch.vpit) 4521 r = 0; 4522 create_pit_unlock: 4523 mutex_unlock(&kvm->lock); 4524 break; 4525 case KVM_GET_IRQCHIP: { 4526 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 4527 struct kvm_irqchip *chip; 4528 4529 chip = memdup_user(argp, sizeof(*chip)); 4530 if (IS_ERR(chip)) { 4531 r = PTR_ERR(chip); 4532 goto out; 4533 } 4534 4535 r = -ENXIO; 4536 if (!irqchip_full(kvm)) 4537 goto get_irqchip_out; 4538 r = kvm_vm_ioctl_get_irqchip(kvm, chip); 4539 if (r) 4540 goto get_irqchip_out; 4541 r = -EFAULT; 4542 if (copy_to_user(argp, chip, sizeof(*chip))) 4543 goto get_irqchip_out; 4544 r = 0; 4545 get_irqchip_out: 4546 kfree(chip); 4547 break; 4548 } 4549 case KVM_SET_IRQCHIP: { 4550 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 4551 struct kvm_irqchip *chip; 4552 4553 chip = memdup_user(argp, sizeof(*chip)); 4554 if (IS_ERR(chip)) { 4555 r = PTR_ERR(chip); 4556 goto out; 4557 } 4558 4559 r = -ENXIO; 4560 if (!irqchip_full(kvm)) 4561 goto set_irqchip_out; 4562 r = kvm_vm_ioctl_set_irqchip(kvm, chip); 4563 set_irqchip_out: 4564 kfree(chip); 4565 break; 4566 } 4567 case KVM_GET_PIT: { 4568 r = -EFAULT; 4569 if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state))) 4570 goto out; 4571 r = -ENXIO; 4572 if (!kvm->arch.vpit) 4573 goto out; 4574 r = kvm_vm_ioctl_get_pit(kvm, &u.ps); 4575 if (r) 4576 goto out; 4577 r = -EFAULT; 4578 if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state))) 4579 goto out; 4580 r = 0; 4581 break; 4582 } 4583 case KVM_SET_PIT: { 4584 r = -EFAULT; 4585 if (copy_from_user(&u.ps, argp, sizeof(u.ps))) 4586 goto out; 4587 mutex_lock(&kvm->lock); 4588 r = -ENXIO; 4589 if (!kvm->arch.vpit) 4590 goto set_pit_out; 4591 r = kvm_vm_ioctl_set_pit(kvm, &u.ps); 4592 set_pit_out: 4593 mutex_unlock(&kvm->lock); 4594 break; 4595 } 4596 case KVM_GET_PIT2: { 4597 r = -ENXIO; 4598 if (!kvm->arch.vpit) 4599 goto out; 4600 r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2); 4601 if (r) 4602 goto out; 4603 r = -EFAULT; 4604 if (copy_to_user(argp, &u.ps2, sizeof(u.ps2))) 4605 goto out; 4606 r = 0; 4607 break; 4608 } 4609 case KVM_SET_PIT2: { 4610 r = -EFAULT; 4611 if (copy_from_user(&u.ps2, argp, sizeof(u.ps2))) 4612 goto out; 4613 mutex_lock(&kvm->lock); 4614 r = -ENXIO; 4615 if (!kvm->arch.vpit) 4616 goto set_pit2_out; 4617 r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2); 4618 set_pit2_out: 4619 mutex_unlock(&kvm->lock); 4620 break; 4621 } 4622 case KVM_REINJECT_CONTROL: { 4623 struct kvm_reinject_control control; 4624 r = -EFAULT; 4625 if (copy_from_user(&control, argp, sizeof(control))) 4626 goto out; 4627 r = -ENXIO; 4628 if (!kvm->arch.vpit) 4629 goto out; 4630 r = kvm_vm_ioctl_reinject(kvm, &control); 4631 break; 4632 } 4633 #endif 4634 case KVM_SET_BOOT_CPU_ID: 4635 r = 0; 4636 mutex_lock(&kvm->lock); 4637 if (kvm->created_vcpus) 4638 r = -EBUSY; 4639 else if (arg > KVM_MAX_VCPU_IDS || 4640 (kvm->arch.max_vcpu_ids && arg > kvm->arch.max_vcpu_ids)) 4641 r = -EINVAL; 4642 else 4643 kvm->arch.bsp_vcpu_id = arg; 4644 mutex_unlock(&kvm->lock); 4645 break; 4646 #ifdef CONFIG_KVM_XEN 4647 case KVM_XEN_HVM_CONFIG: { 4648 struct kvm_xen_hvm_config xhc; 4649 r = -EFAULT; 4650 if (copy_from_user(&xhc, argp, sizeof(xhc))) 4651 goto out; 4652 r = kvm_xen_hvm_config(kvm, &xhc); 4653 break; 4654 } 4655 case KVM_XEN_HVM_GET_ATTR: { 4656 struct kvm_xen_hvm_attr xha; 4657 4658 r = -EFAULT; 4659 if (copy_from_user(&xha, argp, sizeof(xha))) 4660 goto out; 4661 r = kvm_xen_hvm_get_attr(kvm, &xha); 4662 if (!r && copy_to_user(argp, &xha, sizeof(xha))) 4663 r = -EFAULT; 4664 break; 4665 } 4666 case KVM_XEN_HVM_SET_ATTR: { 4667 struct kvm_xen_hvm_attr xha; 4668 4669 r = -EFAULT; 4670 if (copy_from_user(&xha, argp, sizeof(xha))) 4671 goto out; 4672 r = kvm_xen_hvm_set_attr(kvm, &xha); 4673 break; 4674 } 4675 case KVM_XEN_HVM_EVTCHN_SEND: { 4676 struct kvm_irq_routing_xen_evtchn uxe; 4677 4678 r = -EFAULT; 4679 if (copy_from_user(&uxe, argp, sizeof(uxe))) 4680 goto out; 4681 r = kvm_xen_hvm_evtchn_send(kvm, &uxe); 4682 break; 4683 } 4684 #endif 4685 case KVM_SET_CLOCK: 4686 r = kvm_vm_ioctl_set_clock(kvm, argp); 4687 break; 4688 case KVM_GET_CLOCK: 4689 r = kvm_vm_ioctl_get_clock(kvm, argp); 4690 break; 4691 case KVM_SET_TSC_KHZ: { 4692 u32 user_tsc_khz; 4693 4694 r = -EINVAL; 4695 user_tsc_khz = (u32)arg; 4696 4697 if (kvm_caps.has_tsc_control && 4698 user_tsc_khz > kvm_caps.max_guest_tsc_khz) 4699 goto out; 4700 4701 if (user_tsc_khz == 0) 4702 user_tsc_khz = tsc_khz; 4703 4704 mutex_lock(&kvm->lock); 4705 if (!kvm->created_vcpus) { 4706 WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz); 4707 r = 0; 4708 } 4709 mutex_unlock(&kvm->lock); 4710 goto out; 4711 } 4712 case KVM_GET_TSC_KHZ: { 4713 r = READ_ONCE(kvm->arch.default_tsc_khz); 4714 goto out; 4715 } 4716 case KVM_MEMORY_ENCRYPT_OP: 4717 r = -ENOTTY; 4718 if (!kvm_x86_ops.mem_enc_ioctl) 4719 goto out; 4720 4721 r = kvm_x86_call(mem_enc_ioctl)(kvm, argp); 4722 break; 4723 case KVM_MEMORY_ENCRYPT_REG_REGION: { 4724 struct kvm_enc_region region; 4725 4726 r = -EFAULT; 4727 if (copy_from_user(®ion, argp, sizeof(region))) 4728 goto out; 4729 4730 r = -ENOTTY; 4731 if (!kvm_x86_ops.mem_enc_register_region) 4732 goto out; 4733 4734 r = kvm_x86_call(mem_enc_register_region)(kvm, ®ion); 4735 break; 4736 } 4737 case KVM_MEMORY_ENCRYPT_UNREG_REGION: { 4738 struct kvm_enc_region region; 4739 4740 r = -EFAULT; 4741 if (copy_from_user(®ion, argp, sizeof(region))) 4742 goto out; 4743 4744 r = -ENOTTY; 4745 if (!kvm_x86_ops.mem_enc_unregister_region) 4746 goto out; 4747 4748 r = kvm_x86_call(mem_enc_unregister_region)(kvm, ®ion); 4749 break; 4750 } 4751 #ifdef CONFIG_KVM_HYPERV 4752 case KVM_HYPERV_EVENTFD: { 4753 struct kvm_hyperv_eventfd hvevfd; 4754 4755 r = -EFAULT; 4756 if (copy_from_user(&hvevfd, argp, sizeof(hvevfd))) 4757 goto out; 4758 r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd); 4759 break; 4760 } 4761 #endif 4762 case KVM_SET_PMU_EVENT_FILTER: 4763 r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp); 4764 break; 4765 case KVM_X86_SET_MSR_FILTER: { 4766 struct kvm_msr_filter __user *user_msr_filter = argp; 4767 struct kvm_msr_filter filter; 4768 4769 if (copy_from_user(&filter, user_msr_filter, sizeof(filter))) 4770 return -EFAULT; 4771 4772 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter); 4773 break; 4774 } 4775 default: 4776 r = -ENOTTY; 4777 } 4778 out: 4779 return r; 4780 } 4781 4782 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len, 4783 void *__v) 4784 { 4785 const void *v = __v; 4786 int handled = 0; 4787 int n; 4788 4789 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, addr, __v); 4790 4791 do { 4792 n = min(len, 8); 4793 if (!(lapic_in_kernel(vcpu) && 4794 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v)) 4795 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v)) 4796 break; 4797 handled += n; 4798 addr += n; 4799 len -= n; 4800 v += n; 4801 } while (len); 4802 4803 return handled; 4804 } 4805 4806 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v) 4807 { 4808 int handled = 0; 4809 int n; 4810 4811 do { 4812 n = min(len, 8); 4813 if (!(lapic_in_kernel(vcpu) && 4814 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev, 4815 addr, n, v)) 4816 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v)) 4817 break; 4818 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v); 4819 handled += n; 4820 addr += n; 4821 len -= n; 4822 v += n; 4823 } while (len); 4824 4825 if (len) 4826 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len, addr, NULL); 4827 4828 return handled; 4829 } 4830 4831 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, 4832 struct x86_exception *exception) 4833 { 4834 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4835 4836 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 4837 return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception); 4838 } 4839 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_read); 4840 4841 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, 4842 struct x86_exception *exception) 4843 { 4844 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4845 4846 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 4847 access |= PFERR_WRITE_MASK; 4848 return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception); 4849 } 4850 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_write); 4851 4852 /* uses this to access any guest's mapped memory without checking CPL */ 4853 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, 4854 struct x86_exception *exception) 4855 { 4856 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4857 4858 return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, 0, exception); 4859 } 4860 4861 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, 4862 struct kvm_vcpu *vcpu, u64 access, 4863 struct x86_exception *exception) 4864 { 4865 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4866 void *data = val; 4867 int r = X86EMUL_CONTINUE; 4868 4869 while (bytes) { 4870 gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access, exception); 4871 unsigned offset = addr & (PAGE_SIZE-1); 4872 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset); 4873 int ret; 4874 4875 if (gpa == INVALID_GPA) 4876 return X86EMUL_PROPAGATE_FAULT; 4877 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data, 4878 offset, toread); 4879 if (ret < 0) { 4880 r = X86EMUL_IO_NEEDED; 4881 goto out; 4882 } 4883 4884 bytes -= toread; 4885 data += toread; 4886 addr += toread; 4887 } 4888 out: 4889 return r; 4890 } 4891 4892 /* used for instruction fetching */ 4893 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, 4894 gva_t addr, void *val, unsigned int bytes, 4895 struct x86_exception *exception) 4896 { 4897 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4898 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4899 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 4900 unsigned offset; 4901 int ret; 4902 4903 /* Inline kvm_read_guest_virt_helper for speed. */ 4904 gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access|PFERR_FETCH_MASK, 4905 exception); 4906 if (unlikely(gpa == INVALID_GPA)) 4907 return X86EMUL_PROPAGATE_FAULT; 4908 4909 offset = addr & (PAGE_SIZE-1); 4910 if (WARN_ON(offset + bytes > PAGE_SIZE)) 4911 bytes = (unsigned)PAGE_SIZE - offset; 4912 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val, 4913 offset, bytes); 4914 if (unlikely(ret < 0)) 4915 return X86EMUL_IO_NEEDED; 4916 4917 return X86EMUL_CONTINUE; 4918 } 4919 4920 int kvm_read_guest_virt(struct kvm_vcpu *vcpu, 4921 gva_t addr, void *val, unsigned int bytes, 4922 struct x86_exception *exception) 4923 { 4924 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 4925 4926 /* 4927 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED 4928 * is returned, but our callers are not ready for that and they blindly 4929 * call kvm_inject_page_fault. Ensure that they at least do not leak 4930 * uninitialized kernel stack memory into cr2 and error code. 4931 */ 4932 memset(exception, 0, sizeof(*exception)); 4933 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, 4934 exception); 4935 } 4936 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_guest_virt); 4937 4938 static int emulator_read_std(struct x86_emulate_ctxt *ctxt, 4939 gva_t addr, void *val, unsigned int bytes, 4940 struct x86_exception *exception, bool system) 4941 { 4942 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4943 u64 access = 0; 4944 4945 if (system) 4946 access |= PFERR_IMPLICIT_ACCESS; 4947 else if (kvm_x86_call(get_cpl)(vcpu) == 3) 4948 access |= PFERR_USER_MASK; 4949 4950 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception); 4951 } 4952 4953 static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, 4954 struct kvm_vcpu *vcpu, u64 access, 4955 struct x86_exception *exception) 4956 { 4957 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 4958 void *data = val; 4959 int r = X86EMUL_CONTINUE; 4960 4961 while (bytes) { 4962 gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access, exception); 4963 unsigned offset = addr & (PAGE_SIZE-1); 4964 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset); 4965 int ret; 4966 4967 if (gpa == INVALID_GPA) 4968 return X86EMUL_PROPAGATE_FAULT; 4969 ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite); 4970 if (ret < 0) { 4971 r = X86EMUL_IO_NEEDED; 4972 goto out; 4973 } 4974 4975 bytes -= towrite; 4976 data += towrite; 4977 addr += towrite; 4978 } 4979 out: 4980 return r; 4981 } 4982 4983 static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val, 4984 unsigned int bytes, struct x86_exception *exception, 4985 bool system) 4986 { 4987 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4988 u64 access = PFERR_WRITE_MASK; 4989 4990 if (system) 4991 access |= PFERR_IMPLICIT_ACCESS; 4992 else if (kvm_x86_call(get_cpl)(vcpu) == 3) 4993 access |= PFERR_USER_MASK; 4994 4995 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, 4996 access, exception); 4997 } 4998 4999 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val, 5000 unsigned int bytes, struct x86_exception *exception) 5001 { 5002 /* kvm_write_guest_virt_system can pull in tons of pages. */ 5003 kvm_request_l1tf_flush_l1d(); 5004 5005 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, 5006 PFERR_WRITE_MASK, exception); 5007 } 5008 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_write_guest_virt_system); 5009 5010 static int kvm_check_emulate_insn(struct kvm_vcpu *vcpu, int emul_type, 5011 void *insn, int insn_len) 5012 { 5013 return kvm_x86_call(check_emulate_instruction)(vcpu, emul_type, 5014 insn, insn_len); 5015 } 5016 5017 int handle_ud(struct kvm_vcpu *vcpu) 5018 { 5019 static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX }; 5020 int fep_flags = READ_ONCE(force_emulation_prefix); 5021 int emul_type = EMULTYPE_TRAP_UD; 5022 char sig[5]; /* ud2; .ascii "kvm" */ 5023 struct x86_exception e; 5024 int r; 5025 5026 r = kvm_check_emulate_insn(vcpu, emul_type, NULL, 0); 5027 if (r != X86EMUL_CONTINUE) 5028 return 1; 5029 5030 if (fep_flags && 5031 kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu), 5032 sig, sizeof(sig), &e) == 0 && 5033 memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) { 5034 if (fep_flags & KVM_FEP_CLEAR_RFLAGS_RF) 5035 kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF); 5036 kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig)); 5037 emul_type = EMULTYPE_TRAP_UD_FORCED; 5038 } 5039 5040 return kvm_emulate_instruction(vcpu, emul_type); 5041 } 5042 EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_ud); 5043 5044 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 5045 gpa_t gpa, bool write) 5046 { 5047 /* For APIC access vmexit */ 5048 if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 5049 return 1; 5050 5051 if (vcpu_match_mmio_gpa(vcpu, gpa)) { 5052 trace_vcpu_match_mmio(gva, gpa, write, true); 5053 return 1; 5054 } 5055 5056 return 0; 5057 } 5058 5059 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 5060 gpa_t *gpa, struct x86_exception *exception, 5061 bool write) 5062 { 5063 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 5064 u64 access = ((kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0) 5065 | (write ? PFERR_WRITE_MASK : 0); 5066 5067 /* 5068 * currently PKRU is only applied to ept enabled guest so 5069 * there is no pkey in EPT page table for L1 guest or EPT 5070 * shadow page table for L2 guest. 5071 */ 5072 if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) || 5073 !permission_fault(vcpu, gva_walk, 5074 vcpu->arch.mmio_access, 0, access))) { 5075 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT | 5076 (gva & (PAGE_SIZE - 1)); 5077 trace_vcpu_match_mmio(gva, *gpa, write, false); 5078 return 1; 5079 } 5080 5081 *gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception); 5082 5083 if (*gpa == INVALID_GPA) 5084 return -1; 5085 5086 return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write); 5087 } 5088 5089 struct read_write_emulator_ops { 5090 int (*read_write_guest)(struct kvm_vcpu *vcpu, gpa_t gpa, 5091 void *val, int bytes); 5092 int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa, 5093 int bytes, void *val); 5094 bool write; 5095 }; 5096 5097 static int emulator_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, 5098 void *val, int bytes) 5099 { 5100 return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes); 5101 } 5102 5103 static int emulator_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, 5104 void *val, int bytes) 5105 { 5106 int ret; 5107 5108 ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes); 5109 if (ret < 0) 5110 return 0; 5111 kvm_page_track_write(vcpu, gpa, val, bytes); 5112 return 1; 5113 } 5114 5115 static int emulator_read_write_onepage(unsigned long addr, void *val, 5116 unsigned int bytes, 5117 struct x86_exception *exception, 5118 struct kvm_vcpu *vcpu, 5119 const struct read_write_emulator_ops *ops) 5120 { 5121 gpa_t gpa; 5122 int handled, ret; 5123 bool write = ops->write; 5124 struct kvm_mmio_fragment *frag; 5125 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 5126 5127 /* 5128 * If the exit was due to a NPF we may already have a GPA. 5129 * If the GPA is present, use it to avoid the GVA to GPA table walk. 5130 * Note, this cannot be used on string operations since string 5131 * operation using rep will only have the initial GPA from the NPF 5132 * occurred. 5133 */ 5134 if ((ctxt->gpa_access & (write ? ACC_WRITE_MASK : ACC_READ_MASK)) && 5135 emulator_can_use_gpa(ctxt) && 5136 (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) { 5137 gpa = ctxt->gpa_val; 5138 ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); 5139 } else { 5140 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write); 5141 if (ret < 0) 5142 return X86EMUL_PROPAGATE_FAULT; 5143 } 5144 5145 /* 5146 * If the memory is not _known_ to be emulated MMIO, attempt to access 5147 * guest memory. If accessing guest memory fails, e.g. because there's 5148 * no memslot, then handle the access as MMIO. Note, treating the 5149 * access as emulated MMIO is technically wrong if there is a memslot, 5150 * i.e. if accessing host user memory failed, but this has been KVM's 5151 * historical ABI for decades. 5152 */ 5153 if (!ret && ops->read_write_guest(vcpu, gpa, val, bytes)) 5154 return X86EMUL_CONTINUE; 5155 5156 /* 5157 * Attempt to handle emulated MMIO within the kernel, e.g. for accesses 5158 * to an in-kernel local or I/O APIC, or to an ioeventfd range attached 5159 * to MMIO bus. If the access isn't fully resolved, insert an MMIO 5160 * fragment with the relevant details. 5161 */ 5162 handled = ops->read_write_mmio(vcpu, gpa, bytes, val); 5163 if (handled == bytes) 5164 return X86EMUL_CONTINUE; 5165 5166 gpa += handled; 5167 bytes -= handled; 5168 val += handled; 5169 5170 WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS); 5171 frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++]; 5172 frag->gpa = gpa; 5173 if (write && bytes <= 8u) { 5174 frag->val = 0; 5175 frag->data = &frag->val; 5176 memcpy(&frag->val, val, bytes); 5177 } else { 5178 frag->data = val; 5179 } 5180 frag->len = bytes; 5181 5182 /* 5183 * Continue emulating, even though KVM needs to (eventually) do an MMIO 5184 * exit to userspace. If the access splits multiple pages, then KVM 5185 * needs to exit to userspace only after emulating both parts of the 5186 * access. 5187 */ 5188 return X86EMUL_CONTINUE; 5189 } 5190 5191 static int emulator_read_write(struct x86_emulate_ctxt *ctxt, 5192 unsigned long addr, 5193 void *val, unsigned int bytes, 5194 struct x86_exception *exception, 5195 const struct read_write_emulator_ops *ops) 5196 { 5197 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5198 int rc; 5199 5200 if (WARN_ON_ONCE((bytes > 8u || !ops->write) && object_is_on_stack(val))) 5201 return X86EMUL_UNHANDLEABLE; 5202 5203 /* 5204 * If the read was already completed via a userspace MMIO exit, there's 5205 * nothing left to do except trace the MMIO read. When completing MMIO 5206 * reads, KVM re-emulates the instruction to propagate the value into 5207 * the correct destination, e.g. into the correct register, but the 5208 * value itself has already been copied to the read cache. 5209 * 5210 * Note! This is *tightly* coupled to read_emulated() satisfying reads 5211 * from the emulator's mem_read cache, so that the MMIO fragment data 5212 * is copied to the correct chunk of the correct operand. 5213 */ 5214 if (!ops->write && vcpu->mmio_read_completed) { 5215 /* 5216 * For simplicity, trace the entire MMIO read in one shot, even 5217 * though the GPA might be incorrect if there are two fragments 5218 * that aren't contiguous in the GPA space. 5219 */ 5220 trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, 5221 vcpu->mmio_fragments[0].gpa, val); 5222 vcpu->mmio_read_completed = 0; 5223 return X86EMUL_CONTINUE; 5224 } 5225 5226 vcpu->mmio_nr_fragments = 0; 5227 5228 /* Crossing a page boundary? */ 5229 if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { 5230 int now; 5231 5232 now = -addr & ~PAGE_MASK; 5233 rc = emulator_read_write_onepage(addr, val, now, exception, 5234 vcpu, ops); 5235 5236 if (rc != X86EMUL_CONTINUE) 5237 return rc; 5238 addr += now; 5239 if (ctxt->mode != X86EMUL_MODE_PROT64) 5240 addr = (u32)addr; 5241 val += now; 5242 bytes -= now; 5243 } 5244 5245 rc = emulator_read_write_onepage(addr, val, bytes, exception, 5246 vcpu, ops); 5247 if (rc != X86EMUL_CONTINUE) 5248 return rc; 5249 5250 if (!vcpu->mmio_nr_fragments) 5251 return X86EMUL_CONTINUE; 5252 5253 vcpu->mmio_needed = 1; 5254 vcpu->mmio_cur_fragment = 0; 5255 vcpu->mmio_is_write = ops->write; 5256 5257 kvm_prepare_emulated_mmio_exit(vcpu, &vcpu->mmio_fragments[0]); 5258 5259 /* 5260 * For MMIO reads, stop emulating and immediately exit to userspace, as 5261 * KVM needs the value to correctly emulate the instruction. For MMIO 5262 * writes, continue emulating as the write to MMIO is a side effect for 5263 * all intents and purposes. KVM will still exit to userspace, but 5264 * after completing emulation (see the check on vcpu->mmio_needed in 5265 * x86_emulate_instruction()). 5266 */ 5267 return ops->write ? X86EMUL_CONTINUE : X86EMUL_IO_NEEDED; 5268 } 5269 5270 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt, 5271 unsigned long addr, 5272 void *val, 5273 unsigned int bytes, 5274 struct x86_exception *exception) 5275 { 5276 static const struct read_write_emulator_ops ops = { 5277 .read_write_guest = emulator_read_guest, 5278 .read_write_mmio = vcpu_mmio_read, 5279 .write = false, 5280 }; 5281 5282 return emulator_read_write(ctxt, addr, val, bytes, exception, &ops); 5283 } 5284 5285 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt, 5286 unsigned long addr, 5287 const void *val, 5288 unsigned int bytes, 5289 struct x86_exception *exception) 5290 { 5291 static const struct read_write_emulator_ops ops = { 5292 .read_write_guest = emulator_write_guest, 5293 .read_write_mmio = vcpu_mmio_write, 5294 .write = true, 5295 }; 5296 5297 return emulator_read_write(ctxt, addr, (void *)val, bytes, exception, &ops); 5298 } 5299 5300 #define emulator_try_cmpxchg_user(t, ptr, old, new) \ 5301 (__try_cmpxchg_user((t __user *)(ptr), (t *)(old), *(t *)(new), efault ## t)) 5302 5303 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, 5304 unsigned long addr, 5305 const void *old, 5306 const void *new, 5307 unsigned int bytes, 5308 struct x86_exception *exception) 5309 { 5310 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5311 u64 page_line_mask; 5312 unsigned long hva; 5313 gpa_t gpa; 5314 int r; 5315 5316 /* guests cmpxchg8b have to be emulated atomically */ 5317 if (bytes > 8 || (bytes & (bytes - 1))) 5318 goto emul_write; 5319 5320 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL); 5321 5322 if (gpa == INVALID_GPA || 5323 (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 5324 goto emul_write; 5325 5326 /* 5327 * Emulate the atomic as a straight write to avoid #AC if SLD is 5328 * enabled in the host and the access splits a cache line. 5329 */ 5330 if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 5331 page_line_mask = ~(cache_line_size() - 1); 5332 else 5333 page_line_mask = PAGE_MASK; 5334 5335 if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask)) 5336 goto emul_write; 5337 5338 hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa)); 5339 if (kvm_is_error_hva(hva)) 5340 goto emul_write; 5341 5342 hva += offset_in_page(gpa); 5343 5344 switch (bytes) { 5345 case 1: 5346 r = emulator_try_cmpxchg_user(u8, hva, old, new); 5347 break; 5348 case 2: 5349 r = emulator_try_cmpxchg_user(u16, hva, old, new); 5350 break; 5351 case 4: 5352 r = emulator_try_cmpxchg_user(u32, hva, old, new); 5353 break; 5354 case 8: 5355 r = emulator_try_cmpxchg_user(u64, hva, old, new); 5356 break; 5357 default: 5358 BUG(); 5359 } 5360 5361 if (r < 0) 5362 return X86EMUL_UNHANDLEABLE; 5363 5364 /* 5365 * Mark the page dirty _before_ checking whether or not the CMPXCHG was 5366 * successful, as the old value is written back on failure. Note, for 5367 * live migration, this is unnecessarily conservative as CMPXCHG writes 5368 * back the original value and the access is atomic, but KVM's ABI is 5369 * that all writes are dirty logged, regardless of the value written. 5370 */ 5371 kvm_vcpu_mark_page_dirty(vcpu, gpa_to_gfn(gpa)); 5372 5373 if (r) 5374 return X86EMUL_CMPXCHG_FAILED; 5375 5376 kvm_page_track_write(vcpu, gpa, new, bytes); 5377 5378 return X86EMUL_CONTINUE; 5379 5380 emul_write: 5381 pr_warn_once("emulating exchange as write\n"); 5382 5383 return emulator_write_emulated(ctxt, addr, new, bytes, exception); 5384 } 5385 5386 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size, 5387 unsigned short port, void *data, 5388 unsigned int count, bool in) 5389 { 5390 unsigned i; 5391 int r; 5392 5393 WARN_ON_ONCE(vcpu->arch.pio.count); 5394 for (i = 0; i < count; i++) { 5395 if (in) 5396 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data); 5397 else 5398 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data); 5399 5400 if (r) { 5401 if (i == 0) 5402 goto userspace_io; 5403 5404 /* 5405 * Userspace must have unregistered the device while PIO 5406 * was running. Drop writes / read as 0. 5407 */ 5408 if (in) 5409 memset(data, 0, size * (count - i)); 5410 break; 5411 } 5412 5413 data += size; 5414 } 5415 return 1; 5416 5417 userspace_io: 5418 vcpu->arch.pio.port = port; 5419 vcpu->arch.pio.in = in; 5420 vcpu->arch.pio.count = count; 5421 vcpu->arch.pio.size = size; 5422 5423 if (in) 5424 memset(vcpu->arch.pio_data, 0, size * count); 5425 else 5426 memcpy(vcpu->arch.pio_data, data, size * count); 5427 5428 vcpu->run->exit_reason = KVM_EXIT_IO; 5429 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; 5430 vcpu->run->io.size = size; 5431 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; 5432 vcpu->run->io.count = count; 5433 vcpu->run->io.port = port; 5434 return 0; 5435 } 5436 5437 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size, 5438 unsigned short port, void *val, unsigned int count) 5439 { 5440 int r = emulator_pio_in_out(vcpu, size, port, val, count, true); 5441 if (r) 5442 trace_kvm_pio(KVM_PIO_IN, port, size, count, val); 5443 5444 return r; 5445 } 5446 5447 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val) 5448 { 5449 int size = vcpu->arch.pio.size; 5450 unsigned int count = vcpu->arch.pio.count; 5451 memcpy(val, vcpu->arch.pio_data, size * count); 5452 trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data); 5453 vcpu->arch.pio.count = 0; 5454 } 5455 5456 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt, 5457 int size, unsigned short port, void *val, 5458 unsigned int count) 5459 { 5460 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5461 if (vcpu->arch.pio.count) { 5462 /* 5463 * Complete a previous iteration that required userspace I/O. 5464 * Note, @count isn't guaranteed to match pio.count as userspace 5465 * can modify ECX before rerunning the vCPU. Ignore any such 5466 * shenanigans as KVM doesn't support modifying the rep count, 5467 * and the emulator ensures @count doesn't overflow the buffer. 5468 */ 5469 complete_emulator_pio_in(vcpu, val); 5470 return 1; 5471 } 5472 5473 return emulator_pio_in(vcpu, size, port, val, count); 5474 } 5475 5476 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size, 5477 unsigned short port, const void *val, 5478 unsigned int count) 5479 { 5480 trace_kvm_pio(KVM_PIO_OUT, port, size, count, val); 5481 return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false); 5482 } 5483 5484 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, 5485 int size, unsigned short port, 5486 const void *val, unsigned int count) 5487 { 5488 return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count); 5489 } 5490 5491 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address) 5492 { 5493 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); 5494 } 5495 5496 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) 5497 { 5498 if (!need_emulate_wbinvd(vcpu)) 5499 return X86EMUL_CONTINUE; 5500 5501 if (kvm_x86_call(has_wbinvd_exit)()) { 5502 int cpu = get_cpu(); 5503 5504 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 5505 wbinvd_on_cpus_mask(vcpu->arch.wbinvd_dirty_mask); 5506 put_cpu(); 5507 cpumask_clear(vcpu->arch.wbinvd_dirty_mask); 5508 } else 5509 wbinvd(); 5510 return X86EMUL_CONTINUE; 5511 } 5512 5513 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) 5514 { 5515 kvm_emulate_wbinvd_noskip(vcpu); 5516 return kvm_skip_emulated_instruction(vcpu); 5517 } 5518 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wbinvd); 5519 5520 5521 5522 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) 5523 { 5524 kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt)); 5525 } 5526 5527 static unsigned long emulator_get_effective_dr7(struct x86_emulate_ctxt *ctxt) 5528 { 5529 return kvm_get_effective_dr7(emul_to_vcpu(ctxt)); 5530 } 5531 5532 static unsigned long emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr) 5533 { 5534 return kvm_get_dr(emul_to_vcpu(ctxt), dr); 5535 } 5536 5537 static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, 5538 unsigned long value) 5539 { 5540 5541 return kvm_set_dr(emul_to_vcpu(ctxt), dr, value); 5542 } 5543 5544 static u64 mk_cr_64(u64 curr_cr, u32 new_val) 5545 { 5546 return (curr_cr & ~((1ULL << 32) - 1)) | new_val; 5547 } 5548 5549 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr) 5550 { 5551 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5552 unsigned long value; 5553 5554 switch (cr) { 5555 case 0: 5556 value = kvm_read_cr0(vcpu); 5557 break; 5558 case 2: 5559 value = vcpu->arch.cr2; 5560 break; 5561 case 3: 5562 value = kvm_read_cr3(vcpu); 5563 break; 5564 case 4: 5565 value = kvm_read_cr4(vcpu); 5566 break; 5567 case 8: 5568 value = kvm_get_cr8(vcpu); 5569 break; 5570 default: 5571 kvm_err("%s: unexpected cr %u\n", __func__, cr); 5572 return 0; 5573 } 5574 5575 return value; 5576 } 5577 5578 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val) 5579 { 5580 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5581 int res = 0; 5582 5583 switch (cr) { 5584 case 0: 5585 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val)); 5586 break; 5587 case 2: 5588 vcpu->arch.cr2 = val; 5589 break; 5590 case 3: 5591 res = kvm_set_cr3(vcpu, val); 5592 break; 5593 case 4: 5594 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val)); 5595 break; 5596 case 8: 5597 res = kvm_set_cr8(vcpu, val); 5598 break; 5599 default: 5600 kvm_err("%s: unexpected cr %u\n", __func__, cr); 5601 res = -1; 5602 } 5603 5604 return res; 5605 } 5606 5607 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt) 5608 { 5609 return kvm_x86_call(get_cpl)(emul_to_vcpu(ctxt)); 5610 } 5611 5612 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 5613 { 5614 kvm_x86_call(get_gdt)(emul_to_vcpu(ctxt), dt); 5615 } 5616 5617 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 5618 { 5619 kvm_x86_call(get_idt)(emul_to_vcpu(ctxt), dt); 5620 } 5621 5622 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 5623 { 5624 kvm_x86_call(set_gdt)(emul_to_vcpu(ctxt), dt); 5625 } 5626 5627 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 5628 { 5629 kvm_x86_call(set_idt)(emul_to_vcpu(ctxt), dt); 5630 } 5631 5632 static unsigned long emulator_get_cached_segment_base( 5633 struct x86_emulate_ctxt *ctxt, int seg) 5634 { 5635 return kvm_get_segment_base(emul_to_vcpu(ctxt), seg); 5636 } 5637 5638 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector, 5639 struct desc_struct *desc, u32 *base3, 5640 int seg) 5641 { 5642 struct kvm_segment var; 5643 5644 kvm_get_segment(emul_to_vcpu(ctxt), &var, seg); 5645 *selector = var.selector; 5646 5647 if (var.unusable) { 5648 memset(desc, 0, sizeof(*desc)); 5649 if (base3) 5650 *base3 = 0; 5651 return false; 5652 } 5653 5654 if (var.g) 5655 var.limit >>= 12; 5656 set_desc_limit(desc, var.limit); 5657 set_desc_base(desc, (unsigned long)var.base); 5658 #ifdef CONFIG_X86_64 5659 if (base3) 5660 *base3 = var.base >> 32; 5661 #endif 5662 desc->type = var.type; 5663 desc->s = var.s; 5664 desc->dpl = var.dpl; 5665 desc->p = var.present; 5666 desc->avl = var.avl; 5667 desc->l = var.l; 5668 desc->d = var.db; 5669 desc->g = var.g; 5670 5671 return true; 5672 } 5673 5674 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector, 5675 struct desc_struct *desc, u32 base3, 5676 int seg) 5677 { 5678 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5679 struct kvm_segment var; 5680 5681 var.selector = selector; 5682 var.base = get_desc_base(desc); 5683 #ifdef CONFIG_X86_64 5684 var.base |= ((u64)base3) << 32; 5685 #endif 5686 var.limit = get_desc_limit(desc); 5687 if (desc->g) 5688 var.limit = (var.limit << 12) | 0xfff; 5689 var.type = desc->type; 5690 var.dpl = desc->dpl; 5691 var.db = desc->d; 5692 var.s = desc->s; 5693 var.l = desc->l; 5694 var.g = desc->g; 5695 var.avl = desc->avl; 5696 var.present = desc->p; 5697 var.unusable = !var.present; 5698 var.padding = 0; 5699 5700 kvm_set_segment(vcpu, &var, seg); 5701 return; 5702 } 5703 5704 static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt, 5705 u32 msr_index, u64 *pdata) 5706 { 5707 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5708 5709 return kvm_emulator_get_msr_with_filter(vcpu, msr_index, pdata); 5710 } 5711 5712 static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt, 5713 u32 msr_index, u64 data) 5714 { 5715 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5716 5717 return kvm_emulator_set_msr_with_filter(vcpu, msr_index, data); 5718 } 5719 5720 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, 5721 u32 msr_index, u64 *pdata) 5722 { 5723 return kvm_emulator_get_msr(emul_to_vcpu(ctxt), msr_index, pdata); 5724 } 5725 5726 static int emulator_check_rdpmc_early(struct x86_emulate_ctxt *ctxt, u32 pmc) 5727 { 5728 return kvm_pmu_check_rdpmc_early(emul_to_vcpu(ctxt), pmc); 5729 } 5730 5731 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt, 5732 u32 pmc, u64 *pdata) 5733 { 5734 return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata); 5735 } 5736 5737 static void emulator_halt(struct x86_emulate_ctxt *ctxt) 5738 { 5739 emul_to_vcpu(ctxt)->arch.halt_request = 1; 5740 } 5741 5742 static int emulator_intercept(struct x86_emulate_ctxt *ctxt, 5743 struct x86_instruction_info *info, 5744 enum x86_intercept_stage stage) 5745 { 5746 return kvm_x86_call(check_intercept)(emul_to_vcpu(ctxt), info, stage, 5747 &ctxt->exception); 5748 } 5749 5750 static bool emulator_is_cpuid_allowed(struct x86_emulate_ctxt *ctxt) 5751 { 5752 return kvm_is_cpuid_allowed(emul_to_vcpu(ctxt)); 5753 } 5754 5755 static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt, 5756 u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, 5757 bool exact_only) 5758 { 5759 return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only); 5760 } 5761 5762 static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt) 5763 { 5764 return guest_cpu_cap_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE); 5765 } 5766 5767 static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt) 5768 { 5769 return guest_cpu_cap_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR); 5770 } 5771 5772 static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt) 5773 { 5774 return guest_cpu_cap_has(emul_to_vcpu(ctxt), X86_FEATURE_RDPID); 5775 } 5776 5777 static bool emulator_guest_cpuid_is_intel_compatible(struct x86_emulate_ctxt *ctxt) 5778 { 5779 return guest_cpuid_is_intel_compatible(emul_to_vcpu(ctxt)); 5780 } 5781 5782 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg) 5783 { 5784 return kvm_register_read_raw(emul_to_vcpu(ctxt), reg); 5785 } 5786 5787 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val) 5788 { 5789 kvm_register_write_raw(emul_to_vcpu(ctxt), reg, val); 5790 } 5791 5792 static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked) 5793 { 5794 kvm_x86_call(set_nmi_mask)(emul_to_vcpu(ctxt), masked); 5795 } 5796 5797 static bool emulator_is_smm(struct x86_emulate_ctxt *ctxt) 5798 { 5799 return is_smm(emul_to_vcpu(ctxt)); 5800 } 5801 5802 #ifndef CONFIG_KVM_SMM 5803 static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt) 5804 { 5805 WARN_ON_ONCE(1); 5806 return X86EMUL_UNHANDLEABLE; 5807 } 5808 #endif 5809 5810 static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt) 5811 { 5812 kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt)); 5813 } 5814 5815 static int emulator_get_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 *xcr) 5816 { 5817 if (index != XCR_XFEATURE_ENABLED_MASK) 5818 return 1; 5819 *xcr = emul_to_vcpu(ctxt)->arch.xcr0; 5820 return 0; 5821 } 5822 5823 static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr) 5824 { 5825 return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr); 5826 } 5827 5828 static void emulator_vm_bugged(struct x86_emulate_ctxt *ctxt) 5829 { 5830 struct kvm *kvm = emul_to_vcpu(ctxt)->kvm; 5831 5832 if (!kvm->vm_bugged) 5833 kvm_vm_bugged(kvm); 5834 } 5835 5836 static gva_t emulator_get_untagged_addr(struct x86_emulate_ctxt *ctxt, 5837 gva_t addr, unsigned int flags) 5838 { 5839 if (!kvm_x86_ops.get_untagged_addr) 5840 return addr; 5841 5842 return kvm_x86_call(get_untagged_addr)(emul_to_vcpu(ctxt), 5843 addr, flags); 5844 } 5845 5846 static bool emulator_is_canonical_addr(struct x86_emulate_ctxt *ctxt, 5847 gva_t addr, unsigned int flags) 5848 { 5849 return !is_noncanonical_address(addr, emul_to_vcpu(ctxt), flags); 5850 } 5851 5852 static bool emulator_page_address_valid(struct x86_emulate_ctxt *ctxt, gpa_t gpa) 5853 { 5854 return page_address_valid(emul_to_vcpu(ctxt), gpa); 5855 } 5856 5857 static const struct x86_emulate_ops emulate_ops = { 5858 .vm_bugged = emulator_vm_bugged, 5859 .read_gpr = emulator_read_gpr, 5860 .write_gpr = emulator_write_gpr, 5861 .read_std = emulator_read_std, 5862 .write_std = emulator_write_std, 5863 .fetch = kvm_fetch_guest_virt, 5864 .read_emulated = emulator_read_emulated, 5865 .write_emulated = emulator_write_emulated, 5866 .cmpxchg_emulated = emulator_cmpxchg_emulated, 5867 .invlpg = emulator_invlpg, 5868 .pio_in_emulated = emulator_pio_in_emulated, 5869 .pio_out_emulated = emulator_pio_out_emulated, 5870 .get_segment = emulator_get_segment, 5871 .set_segment = emulator_set_segment, 5872 .get_cached_segment_base = emulator_get_cached_segment_base, 5873 .get_gdt = emulator_get_gdt, 5874 .get_idt = emulator_get_idt, 5875 .set_gdt = emulator_set_gdt, 5876 .set_idt = emulator_set_idt, 5877 .get_cr = emulator_get_cr, 5878 .set_cr = emulator_set_cr, 5879 .cpl = emulator_get_cpl, 5880 .get_effective_dr7 = emulator_get_effective_dr7, 5881 .get_dr = emulator_get_dr, 5882 .set_dr = emulator_set_dr, 5883 .set_msr_with_filter = emulator_set_msr_with_filter, 5884 .get_msr_with_filter = emulator_get_msr_with_filter, 5885 .get_msr = emulator_get_msr, 5886 .check_rdpmc_early = emulator_check_rdpmc_early, 5887 .read_pmc = emulator_read_pmc, 5888 .halt = emulator_halt, 5889 .wbinvd = emulator_wbinvd, 5890 .fix_hypercall = emulator_fix_hypercall, 5891 .intercept = emulator_intercept, 5892 .is_cpuid_allowed = emulator_is_cpuid_allowed, 5893 .get_cpuid = emulator_get_cpuid, 5894 .guest_has_movbe = emulator_guest_has_movbe, 5895 .guest_has_fxsr = emulator_guest_has_fxsr, 5896 .guest_has_rdpid = emulator_guest_has_rdpid, 5897 .guest_cpuid_is_intel_compatible = emulator_guest_cpuid_is_intel_compatible, 5898 .set_nmi_mask = emulator_set_nmi_mask, 5899 .is_smm = emulator_is_smm, 5900 .leave_smm = emulator_leave_smm, 5901 .triple_fault = emulator_triple_fault, 5902 .get_xcr = emulator_get_xcr, 5903 .set_xcr = emulator_set_xcr, 5904 .get_untagged_addr = emulator_get_untagged_addr, 5905 .is_canonical_addr = emulator_is_canonical_addr, 5906 .page_address_valid = emulator_page_address_valid, 5907 }; 5908 5909 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) 5910 { 5911 u32 int_shadow = kvm_x86_call(get_interrupt_shadow)(vcpu); 5912 /* 5913 * an sti; sti; sequence only disable interrupts for the first 5914 * instruction. So, if the last instruction, be it emulated or 5915 * not, left the system with the INT_STI flag enabled, it 5916 * means that the last instruction is an sti. We should not 5917 * leave the flag on in this case. The same goes for mov ss 5918 */ 5919 if (int_shadow & mask) 5920 mask = 0; 5921 if (unlikely(int_shadow || mask)) { 5922 kvm_x86_call(set_interrupt_shadow)(vcpu, mask); 5923 if (!mask) 5924 kvm_make_request(KVM_REQ_EVENT, vcpu); 5925 } 5926 } 5927 5928 static int kvm_inject_emulated_db(struct kvm_vcpu *vcpu, unsigned long dr6) 5929 { 5930 struct kvm_run *kvm_run = vcpu->run; 5931 5932 if (vcpu->guest_debug & (KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_SINGLESTEP)) { 5933 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; 5934 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5935 kvm_run->debug.arch.exception = DB_VECTOR; 5936 kvm_run->exit_reason = KVM_EXIT_DEBUG; 5937 return 0; 5938 } 5939 5940 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 5941 return 1; 5942 } 5943 5944 static int inject_emulated_exception(struct kvm_vcpu *vcpu) 5945 { 5946 struct x86_exception *ex = &vcpu->arch.emulate_ctxt->exception; 5947 5948 if (ex->vector == DB_VECTOR) 5949 return kvm_inject_emulated_db(vcpu, ex->dr6); 5950 5951 if (ex->vector == PF_VECTOR) 5952 kvm_inject_emulated_page_fault(vcpu, ex); 5953 else if (ex->error_code_valid) 5954 kvm_queue_exception_e(vcpu, ex->vector, ex->error_code); 5955 else 5956 kvm_queue_exception(vcpu, ex->vector); 5957 return 1; 5958 } 5959 5960 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu) 5961 { 5962 struct x86_emulate_ctxt *ctxt; 5963 5964 ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT); 5965 if (!ctxt) { 5966 pr_err("failed to allocate vcpu's emulator\n"); 5967 return NULL; 5968 } 5969 5970 ctxt->vcpu = vcpu; 5971 ctxt->ops = &emulate_ops; 5972 vcpu->arch.emulate_ctxt = ctxt; 5973 5974 return ctxt; 5975 } 5976 5977 static void init_emulate_ctxt(struct kvm_vcpu *vcpu) 5978 { 5979 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 5980 int cs_db, cs_l; 5981 5982 kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); 5983 5984 ctxt->gpa_access = 0; 5985 ctxt->eflags = kvm_get_rflags(vcpu); 5986 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; 5987 5988 ctxt->eip = kvm_rip_read(vcpu); 5989 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL : 5990 (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 : 5991 (cs_l && is_long_mode(vcpu)) ? X86EMUL_MODE_PROT64 : 5992 cs_db ? X86EMUL_MODE_PROT32 : 5993 X86EMUL_MODE_PROT16; 5994 ctxt->interruptibility = 0; 5995 ctxt->have_exception = false; 5996 ctxt->exception.vector = -1; 5997 ctxt->exception.payload = 0; 5998 ctxt->perm_ok = false; 5999 6000 init_decode_cache(ctxt); 6001 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 6002 } 6003 6004 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) 6005 { 6006 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 6007 int ret; 6008 6009 init_emulate_ctxt(vcpu); 6010 6011 ctxt->op_bytes = 2; 6012 ctxt->ad_bytes = 2; 6013 ctxt->_eip = ctxt->eip + inc_eip; 6014 ret = emulate_int_real(ctxt, irq); 6015 6016 if (ret != X86EMUL_CONTINUE) { 6017 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 6018 } else { 6019 ctxt->eip = ctxt->_eip; 6020 kvm_rip_write(vcpu, ctxt->eip); 6021 kvm_set_rflags(vcpu, ctxt->eflags); 6022 } 6023 } 6024 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_inject_realmode_interrupt); 6025 6026 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, 6027 u8 ndata, u8 *insn_bytes, u8 insn_size) 6028 { 6029 struct kvm_run *run = vcpu->run; 6030 u64 info[5]; 6031 u8 info_start; 6032 6033 /* 6034 * Zero the whole array used to retrieve the exit info, as casting to 6035 * u32 for select entries will leave some chunks uninitialized. 6036 */ 6037 memset(&info, 0, sizeof(info)); 6038 6039 kvm_x86_call(get_exit_info)(vcpu, (u32 *)&info[0], &info[1], &info[2], 6040 (u32 *)&info[3], (u32 *)&info[4]); 6041 6042 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6043 run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION; 6044 6045 /* 6046 * There's currently space for 13 entries, but 5 are used for the exit 6047 * reason and info. Restrict to 4 to reduce the maintenance burden 6048 * when expanding kvm_run.emulation_failure in the future. 6049 */ 6050 if (WARN_ON_ONCE(ndata > 4)) 6051 ndata = 4; 6052 6053 /* Always include the flags as a 'data' entry. */ 6054 info_start = 1; 6055 run->emulation_failure.flags = 0; 6056 6057 if (insn_size) { 6058 BUILD_BUG_ON((sizeof(run->emulation_failure.insn_size) + 6059 sizeof(run->emulation_failure.insn_bytes) != 16)); 6060 info_start += 2; 6061 run->emulation_failure.flags |= 6062 KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES; 6063 run->emulation_failure.insn_size = insn_size; 6064 memset(run->emulation_failure.insn_bytes, 0x90, 6065 sizeof(run->emulation_failure.insn_bytes)); 6066 memcpy(run->emulation_failure.insn_bytes, insn_bytes, insn_size); 6067 } 6068 6069 memcpy(&run->internal.data[info_start], info, sizeof(info)); 6070 memcpy(&run->internal.data[info_start + ARRAY_SIZE(info)], data, 6071 ndata * sizeof(data[0])); 6072 6073 run->emulation_failure.ndata = info_start + ARRAY_SIZE(info) + ndata; 6074 } 6075 6076 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu) 6077 { 6078 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 6079 6080 prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data, 6081 ctxt->fetch.end - ctxt->fetch.data); 6082 } 6083 6084 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, 6085 u8 ndata) 6086 { 6087 prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0); 6088 } 6089 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_prepare_emulation_failure_exit); 6090 6091 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu) 6092 { 6093 __kvm_prepare_emulation_failure_exit(vcpu, NULL, 0); 6094 } 6095 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prepare_emulation_failure_exit); 6096 6097 void kvm_prepare_event_vectoring_exit(struct kvm_vcpu *vcpu, gpa_t gpa) 6098 { 6099 u32 reason, intr_info, error_code; 6100 struct kvm_run *run = vcpu->run; 6101 u64 info1, info2; 6102 int ndata = 0; 6103 6104 kvm_x86_call(get_exit_info)(vcpu, &reason, &info1, &info2, 6105 &intr_info, &error_code); 6106 6107 run->internal.data[ndata++] = info2; 6108 run->internal.data[ndata++] = reason; 6109 run->internal.data[ndata++] = info1; 6110 run->internal.data[ndata++] = gpa; 6111 run->internal.data[ndata++] = vcpu->arch.last_vmentry_cpu; 6112 6113 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6114 run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV; 6115 run->internal.ndata = ndata; 6116 } 6117 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prepare_event_vectoring_exit); 6118 6119 void kvm_prepare_unexpected_reason_exit(struct kvm_vcpu *vcpu, u64 exit_reason) 6120 { 6121 vcpu_unimpl(vcpu, "unexpected exit reason 0x%llx\n", exit_reason); 6122 6123 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6124 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 6125 vcpu->run->internal.ndata = 2; 6126 vcpu->run->internal.data[0] = exit_reason; 6127 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 6128 } 6129 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prepare_unexpected_reason_exit); 6130 6131 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) 6132 { 6133 struct kvm *kvm = vcpu->kvm; 6134 6135 ++vcpu->stat.insn_emulation_fail; 6136 trace_kvm_emulate_insn_failed(vcpu); 6137 6138 if (emulation_type & EMULTYPE_VMWARE_GP) { 6139 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 6140 return 1; 6141 } 6142 6143 if (kvm->arch.exit_on_emulation_error || 6144 (emulation_type & EMULTYPE_SKIP)) { 6145 prepare_emulation_ctxt_failure_exit(vcpu); 6146 return 0; 6147 } 6148 6149 kvm_queue_exception(vcpu, UD_VECTOR); 6150 6151 if (!is_guest_mode(vcpu) && kvm_x86_call(get_cpl)(vcpu) == 0) { 6152 prepare_emulation_ctxt_failure_exit(vcpu); 6153 return 0; 6154 } 6155 6156 return 1; 6157 } 6158 6159 static bool kvm_unprotect_and_retry_on_failure(struct kvm_vcpu *vcpu, 6160 gpa_t cr2_or_gpa, 6161 int emulation_type) 6162 { 6163 if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF)) 6164 return false; 6165 6166 /* 6167 * If the failed instruction faulted on an access to page tables that 6168 * are used to translate any part of the instruction, KVM can't resolve 6169 * the issue by unprotecting the gfn, as zapping the shadow page will 6170 * result in the instruction taking a !PRESENT page fault and thus put 6171 * the vCPU into an infinite loop of page faults. E.g. KVM will create 6172 * a SPTE and write-protect the gfn to resolve the !PRESENT fault, and 6173 * then zap the SPTE to unprotect the gfn, and then do it all over 6174 * again. Report the error to userspace. 6175 */ 6176 if (emulation_type & EMULTYPE_WRITE_PF_TO_SP) 6177 return false; 6178 6179 /* 6180 * If emulation may have been triggered by a write to a shadowed page 6181 * table, unprotect the gfn (zap any relevant SPTEs) and re-enter the 6182 * guest to let the CPU re-execute the instruction in the hope that the 6183 * CPU can cleanly execute the instruction that KVM failed to emulate. 6184 */ 6185 __kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa, true); 6186 6187 /* 6188 * Retry even if _this_ vCPU didn't unprotect the gfn, as it's possible 6189 * all SPTEs were already zapped by a different task. The alternative 6190 * is to report the error to userspace and likely terminate the guest, 6191 * and the last_retry_{eip,addr} checks will prevent retrying the page 6192 * fault indefinitely, i.e. there's nothing to lose by retrying. 6193 */ 6194 return true; 6195 } 6196 6197 static int complete_emulated_mmio(struct kvm_vcpu *vcpu); 6198 static int complete_emulated_pio(struct kvm_vcpu *vcpu); 6199 6200 static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7, 6201 unsigned long *db) 6202 { 6203 u32 dr6 = 0; 6204 int i; 6205 u32 enable, rwlen; 6206 6207 enable = dr7; 6208 rwlen = dr7 >> 16; 6209 for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4) 6210 if ((enable & 3) && (rwlen & 15) == type && db[i] == addr) 6211 dr6 |= (1 << i); 6212 return dr6; 6213 } 6214 6215 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) 6216 { 6217 unsigned long rflags = kvm_x86_call(get_rflags)(vcpu); 6218 int r; 6219 6220 r = kvm_x86_call(skip_emulated_instruction)(vcpu); 6221 if (unlikely(!r)) 6222 return 0; 6223 6224 kvm_pmu_instruction_retired(vcpu); 6225 6226 /* 6227 * rflags is the old, "raw" value of the flags. The new value has 6228 * not been saved yet. 6229 * 6230 * This is correct even for TF set by the guest, because "the 6231 * processor will not generate this exception after the instruction 6232 * that sets the TF flag". 6233 */ 6234 if (unlikely(rflags & X86_EFLAGS_TF)) 6235 r = kvm_inject_emulated_db(vcpu, DR6_BS); 6236 return r; 6237 } 6238 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_skip_emulated_instruction); 6239 6240 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu) 6241 { 6242 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 6243 return false; 6244 6245 if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF) 6246 return true; 6247 6248 /* 6249 * Intel compatible CPUs inhibit code #DBs when MOV/POP SS blocking is 6250 * active, but AMD compatible CPUs do not. 6251 */ 6252 if (!guest_cpuid_is_intel_compatible(vcpu)) 6253 return false; 6254 6255 return kvm_x86_call(get_interrupt_shadow)(vcpu) & KVM_X86_SHADOW_INT_MOV_SS; 6256 } 6257 6258 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, 6259 int emulation_type, int *r) 6260 { 6261 unsigned long dr7 = kvm_get_effective_dr7(vcpu); 6262 6263 WARN_ON_ONCE(emulation_type & EMULTYPE_NO_DECODE); 6264 6265 /* 6266 * Do not check for code breakpoints if hardware has already done the 6267 * checks, as inferred from the emulation type. On NO_DECODE and SKIP, 6268 * the instruction has passed all exception checks, and all intercepted 6269 * exceptions that trigger emulation have lower priority than code 6270 * breakpoints, i.e. the fact that the intercepted exception occurred 6271 * means any code breakpoints have already been serviced. 6272 * 6273 * Note, KVM needs to check for code #DBs on EMULTYPE_TRAP_UD_FORCED as 6274 * hardware has checked the RIP of the magic prefix, but not the RIP of 6275 * the instruction being emulated. The intent of forced emulation is 6276 * to behave as if KVM intercepted the instruction without an exception 6277 * and without a prefix. 6278 */ 6279 if (emulation_type & (EMULTYPE_NO_DECODE | EMULTYPE_SKIP | 6280 EMULTYPE_TRAP_UD | EMULTYPE_VMWARE_GP | EMULTYPE_PF)) 6281 return false; 6282 6283 if (unlikely(dr7 & DR7_BP_EN_MASK) && 6284 !kvm_is_code_breakpoint_inhibited(vcpu)) { 6285 unsigned long eip = kvm_get_linear_rip(vcpu); 6286 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, dr7, 6287 vcpu->arch.eff_db); 6288 6289 if (dr6) { 6290 *r = kvm_inject_emulated_db(vcpu, dr6); 6291 return true; 6292 } 6293 } 6294 6295 return false; 6296 } 6297 6298 static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt) 6299 { 6300 switch (ctxt->opcode_len) { 6301 case 1: 6302 switch (ctxt->b) { 6303 case 0xe4: /* IN */ 6304 case 0xe5: 6305 case 0xec: 6306 case 0xed: 6307 case 0xe6: /* OUT */ 6308 case 0xe7: 6309 case 0xee: 6310 case 0xef: 6311 case 0x6c: /* INS */ 6312 case 0x6d: 6313 case 0x6e: /* OUTS */ 6314 case 0x6f: 6315 return true; 6316 } 6317 break; 6318 case 2: 6319 switch (ctxt->b) { 6320 case 0x33: /* RDPMC */ 6321 return true; 6322 } 6323 break; 6324 } 6325 6326 return false; 6327 } 6328 6329 static bool is_soft_int_instruction(struct x86_emulate_ctxt *ctxt, 6330 int emulation_type) 6331 { 6332 u8 vector = EMULTYPE_GET_SOFT_INT_VECTOR(emulation_type); 6333 6334 switch (ctxt->b) { 6335 case 0xcc: 6336 return vector == BP_VECTOR; 6337 case 0xcd: 6338 return vector == ctxt->src.val; 6339 case 0xce: 6340 return vector == OF_VECTOR; 6341 default: 6342 return false; 6343 } 6344 } 6345 6346 /* 6347 * Decode an instruction for emulation. The caller is responsible for handling 6348 * code breakpoints. Note, manually detecting code breakpoints is unnecessary 6349 * (and wrong) when emulating on an intercepted fault-like exception[*], as 6350 * code breakpoints have higher priority and thus have already been done by 6351 * hardware. 6352 * 6353 * [*] Except #MC, which is higher priority, but KVM should never emulate in 6354 * response to a machine check. 6355 */ 6356 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type, 6357 void *insn, int insn_len) 6358 { 6359 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 6360 int r; 6361 6362 init_emulate_ctxt(vcpu); 6363 6364 r = x86_decode_insn(ctxt, insn, insn_len, emulation_type); 6365 6366 trace_kvm_emulate_insn_start(vcpu); 6367 ++vcpu->stat.insn_emulation; 6368 6369 return r; 6370 } 6371 EXPORT_SYMBOL_FOR_KVM_INTERNAL(x86_decode_emulated_instruction); 6372 6373 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 6374 int emulation_type, void *insn, int insn_len) 6375 { 6376 int r; 6377 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 6378 bool writeback = true; 6379 6380 if ((emulation_type & EMULTYPE_ALLOW_RETRY_PF) && 6381 (WARN_ON_ONCE(is_guest_mode(vcpu)) || 6382 WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))) 6383 emulation_type &= ~EMULTYPE_ALLOW_RETRY_PF; 6384 6385 r = kvm_check_emulate_insn(vcpu, emulation_type, insn, insn_len); 6386 if (r != X86EMUL_CONTINUE) { 6387 if (r == X86EMUL_RETRY_INSTR || r == X86EMUL_PROPAGATE_FAULT) 6388 return 1; 6389 6390 if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa, 6391 emulation_type)) 6392 return 1; 6393 6394 if (r == X86EMUL_UNHANDLEABLE_VECTORING) { 6395 kvm_prepare_event_vectoring_exit(vcpu, cr2_or_gpa); 6396 return 0; 6397 } 6398 6399 WARN_ON_ONCE(r != X86EMUL_UNHANDLEABLE); 6400 return handle_emulation_failure(vcpu, emulation_type); 6401 } 6402 6403 kvm_request_l1tf_flush_l1d(); 6404 6405 if (!(emulation_type & EMULTYPE_NO_DECODE)) { 6406 kvm_clear_exception_queue(vcpu); 6407 6408 /* 6409 * Return immediately if RIP hits a code breakpoint, such #DBs 6410 * are fault-like and are higher priority than any faults on 6411 * the code fetch itself. 6412 */ 6413 if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r)) 6414 return r; 6415 6416 r = x86_decode_emulated_instruction(vcpu, emulation_type, 6417 insn, insn_len); 6418 if (r != EMULATION_OK) { 6419 if ((emulation_type & EMULTYPE_TRAP_UD) || 6420 (emulation_type & EMULTYPE_TRAP_UD_FORCED)) { 6421 kvm_queue_exception(vcpu, UD_VECTOR); 6422 return 1; 6423 } 6424 if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa, 6425 emulation_type)) 6426 return 1; 6427 6428 if (ctxt->have_exception && 6429 !(emulation_type & EMULTYPE_SKIP)) { 6430 /* 6431 * #UD should result in just EMULATION_FAILED, and trap-like 6432 * exception should not be encountered during decode. 6433 */ 6434 WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR || 6435 exception_type(ctxt->exception.vector) == EXCPT_TRAP); 6436 return inject_emulated_exception(vcpu); 6437 } 6438 return handle_emulation_failure(vcpu, emulation_type); 6439 } 6440 } 6441 6442 if ((emulation_type & EMULTYPE_VMWARE_GP) && 6443 !is_vmware_backdoor_opcode(ctxt)) { 6444 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 6445 return 1; 6446 } 6447 6448 /* 6449 * EMULTYPE_SKIP without EMULTYPE_COMPLETE_USER_EXIT is intended for 6450 * use *only* by vendor callbacks for kvm_skip_emulated_instruction(). 6451 * The caller is responsible for updating interruptibility state and 6452 * injecting single-step #DBs. 6453 */ 6454 if (emulation_type & EMULTYPE_SKIP) { 6455 if (emulation_type & EMULTYPE_SKIP_SOFT_INT && 6456 !is_soft_int_instruction(ctxt, emulation_type)) 6457 return 0; 6458 6459 if (ctxt->mode != X86EMUL_MODE_PROT64) 6460 ctxt->eip = (u32)ctxt->_eip; 6461 else 6462 ctxt->eip = ctxt->_eip; 6463 6464 if (emulation_type & EMULTYPE_COMPLETE_USER_EXIT) { 6465 r = 1; 6466 goto writeback; 6467 } 6468 6469 kvm_rip_write(vcpu, ctxt->eip); 6470 if (ctxt->eflags & X86_EFLAGS_RF) 6471 kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF); 6472 return 1; 6473 } 6474 6475 /* 6476 * If emulation was caused by a write-protection #PF on a non-page_table 6477 * writing instruction, try to unprotect the gfn, i.e. zap shadow pages, 6478 * and retry the instruction, as the vCPU is likely no longer using the 6479 * gfn as a page table. 6480 */ 6481 if ((emulation_type & EMULTYPE_ALLOW_RETRY_PF) && 6482 !x86_page_table_writing_insn(ctxt) && 6483 kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa)) 6484 return 1; 6485 6486 /* this is needed for vmware backdoor interface to work since it 6487 changes registers values during IO operation */ 6488 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) { 6489 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 6490 emulator_invalidate_register_cache(ctxt); 6491 } 6492 6493 restart: 6494 if (emulation_type & EMULTYPE_PF) { 6495 /* Save the faulting GPA (cr2) in the address field */ 6496 ctxt->exception.address = cr2_or_gpa; 6497 6498 /* With shadow page tables, cr2 contains a GVA or nGPA. */ 6499 if (vcpu->arch.mmu->root_role.direct) { 6500 ctxt->gpa_access = ACC_READ_MASK; 6501 /* 6502 * Always allow writes for guests with protected page 6503 * tables, as KVM can't walk the guest's page tables, 6504 * i.e. KVM can't get the RMW protections for a given 6505 * GVA to see if the write side of a RMW operation 6506 * should be allowed. 6507 */ 6508 if ((emulation_type & EMULTYPE_PF_WRITE) || 6509 vcpu->kvm->arch.has_protected_page_tables) 6510 ctxt->gpa_access |= ACC_WRITE_MASK; 6511 ctxt->gpa_val = cr2_or_gpa; 6512 } 6513 } else { 6514 /* Sanitize the address out of an abundance of paranoia. */ 6515 ctxt->exception.address = 0; 6516 } 6517 6518 /* 6519 * Check L1's instruction intercepts when emulating instructions for 6520 * L2, unless KVM is re-emulating a previously decoded instruction, 6521 * e.g. to complete userspace I/O, in which case KVM has already 6522 * checked the intercepts. 6523 */ 6524 r = x86_emulate_insn(ctxt, is_guest_mode(vcpu) && 6525 !(emulation_type & EMULTYPE_NO_DECODE)); 6526 6527 if (r == EMULATION_INTERCEPTED) 6528 return 1; 6529 6530 if (r == EMULATION_FAILED) { 6531 if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa, 6532 emulation_type)) 6533 return 1; 6534 6535 return handle_emulation_failure(vcpu, emulation_type); 6536 } 6537 6538 if (ctxt->have_exception) { 6539 WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write); 6540 vcpu->mmio_needed = false; 6541 r = inject_emulated_exception(vcpu); 6542 } else if (vcpu->arch.pio.count) { 6543 if (!vcpu->arch.pio.in) { 6544 /* FIXME: return into emulator if single-stepping. */ 6545 vcpu->arch.pio.count = 0; 6546 } else { 6547 writeback = false; 6548 vcpu->arch.complete_userspace_io = complete_emulated_pio; 6549 } 6550 r = 0; 6551 } else if (vcpu->mmio_needed) { 6552 ++vcpu->stat.mmio_exits; 6553 6554 if (!vcpu->mmio_is_write) 6555 writeback = false; 6556 r = 0; 6557 vcpu->arch.complete_userspace_io = complete_emulated_mmio; 6558 } else if (vcpu->arch.complete_userspace_io) { 6559 writeback = false; 6560 r = 0; 6561 } else if (r == EMULATION_RESTART) 6562 goto restart; 6563 else 6564 r = 1; 6565 6566 writeback: 6567 if (writeback) { 6568 unsigned long rflags = kvm_x86_call(get_rflags)(vcpu); 6569 toggle_interruptibility(vcpu, ctxt->interruptibility); 6570 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 6571 6572 /* 6573 * Note, EXCPT_DB is assumed to be fault-like as the emulator 6574 * only supports code breakpoints and general detect #DB, both 6575 * of which are fault-like. 6576 */ 6577 if (!ctxt->have_exception || 6578 exception_type(ctxt->exception.vector) == EXCPT_TRAP) { 6579 kvm_pmu_instruction_retired(vcpu); 6580 if (ctxt->is_branch) 6581 kvm_pmu_branch_retired(vcpu); 6582 kvm_rip_write(vcpu, ctxt->eip); 6583 if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP))) 6584 r = kvm_inject_emulated_db(vcpu, DR6_BS); 6585 kvm_x86_call(update_emulated_instruction)(vcpu); 6586 __kvm_set_rflags(vcpu, ctxt->eflags); 6587 } 6588 6589 /* 6590 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will 6591 * do nothing, and it will be requested again as soon as 6592 * the shadow expires. But we still need to check here, 6593 * because POPF has no interrupt shadow. 6594 */ 6595 if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF)) 6596 kvm_make_request(KVM_REQ_EVENT, vcpu); 6597 } else 6598 vcpu->arch.emulate_regs_need_sync_to_vcpu = true; 6599 6600 return r; 6601 } 6602 6603 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type) 6604 { 6605 return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0); 6606 } 6607 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_instruction); 6608 6609 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, 6610 void *insn, int insn_len) 6611 { 6612 return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len); 6613 } 6614 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_instruction_from_buffer); 6615 6616 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu) 6617 { 6618 vcpu->arch.pio.count = 0; 6619 return 1; 6620 } 6621 6622 static int complete_fast_pio_out(struct kvm_vcpu *vcpu) 6623 { 6624 vcpu->arch.pio.count = 0; 6625 6626 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.cui_linear_rip))) 6627 return 1; 6628 6629 return kvm_skip_emulated_instruction(vcpu); 6630 } 6631 6632 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, 6633 unsigned short port) 6634 { 6635 unsigned long val = kvm_rax_read_raw(vcpu); 6636 int ret = emulator_pio_out(vcpu, size, port, &val, 1); 6637 6638 if (ret) 6639 return ret; 6640 6641 /* 6642 * Workaround userspace that relies on old KVM behavior of %rip being 6643 * incremented prior to exiting to userspace to handle "OUT 0x7e". 6644 */ 6645 if (port == 0x7e && 6646 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) { 6647 vcpu->arch.complete_userspace_io = 6648 complete_fast_pio_out_port_0x7e; 6649 kvm_skip_emulated_instruction(vcpu); 6650 } else { 6651 vcpu->arch.cui_linear_rip = kvm_get_linear_rip(vcpu); 6652 vcpu->arch.complete_userspace_io = complete_fast_pio_out; 6653 } 6654 return 0; 6655 } 6656 6657 static int complete_fast_pio_in(struct kvm_vcpu *vcpu) 6658 { 6659 unsigned long val; 6660 6661 /* We should only ever be called with arch.pio.count equal to 1 */ 6662 if (KVM_BUG_ON(vcpu->arch.pio.count != 1, vcpu->kvm)) 6663 return -EIO; 6664 6665 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.cui_linear_rip))) { 6666 vcpu->arch.pio.count = 0; 6667 return 1; 6668 } 6669 6670 /* For size less than 4 we merge, else we zero extend */ 6671 val = (vcpu->arch.pio.size < 4) ? kvm_rax_read_raw(vcpu) : 0; 6672 6673 complete_emulator_pio_in(vcpu, &val); 6674 kvm_rax_write_raw(vcpu, val); 6675 6676 return kvm_skip_emulated_instruction(vcpu); 6677 } 6678 6679 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, 6680 unsigned short port) 6681 { 6682 unsigned long val; 6683 int ret; 6684 6685 /* For size less than 4 we merge, else we zero extend */ 6686 val = (size < 4) ? kvm_rax_read_raw(vcpu) : 0; 6687 6688 ret = emulator_pio_in(vcpu, size, port, &val, 1); 6689 if (ret) { 6690 kvm_rax_write_raw(vcpu, val); 6691 return ret; 6692 } 6693 6694 vcpu->arch.cui_linear_rip = kvm_get_linear_rip(vcpu); 6695 vcpu->arch.complete_userspace_io = complete_fast_pio_in; 6696 6697 return 0; 6698 } 6699 6700 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in) 6701 { 6702 int ret; 6703 6704 if (in) 6705 ret = kvm_fast_pio_in(vcpu, size, port); 6706 else 6707 ret = kvm_fast_pio_out(vcpu, size, port); 6708 return ret && kvm_skip_emulated_instruction(vcpu); 6709 } 6710 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fast_pio); 6711 6712 static int kvmclock_cpu_down_prep(unsigned int cpu) 6713 { 6714 __this_cpu_write(cpu_tsc_khz, 0); 6715 return 0; 6716 } 6717 6718 static void tsc_khz_changed(void *data) 6719 { 6720 struct cpufreq_freqs *freq = data; 6721 unsigned long khz; 6722 6723 WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_CONSTANT_TSC)); 6724 6725 if (data) 6726 khz = freq->new; 6727 else 6728 khz = cpufreq_quick_get(raw_smp_processor_id()); 6729 if (!khz) 6730 khz = tsc_khz; 6731 __this_cpu_write(cpu_tsc_khz, khz); 6732 } 6733 6734 #ifdef CONFIG_X86_64 6735 static void kvm_hyperv_tsc_notifier(void) 6736 { 6737 struct kvm *kvm; 6738 int cpu; 6739 6740 mutex_lock(&kvm_lock); 6741 list_for_each_entry(kvm, &vm_list, vm_list) 6742 kvm_make_mclock_inprogress_request(kvm); 6743 6744 /* no guest entries from this point */ 6745 hyperv_stop_tsc_emulation(); 6746 6747 /* TSC frequency always matches when on Hyper-V */ 6748 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 6749 for_each_present_cpu(cpu) 6750 per_cpu(cpu_tsc_khz, cpu) = tsc_khz; 6751 } 6752 kvm_caps.max_guest_tsc_khz = tsc_khz; 6753 6754 list_for_each_entry(kvm, &vm_list, vm_list) { 6755 __kvm_start_pvclock_update(kvm); 6756 pvclock_update_vm_gtod_copy(kvm); 6757 kvm_end_pvclock_update(kvm); 6758 } 6759 6760 mutex_unlock(&kvm_lock); 6761 } 6762 #endif 6763 6764 static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu) 6765 { 6766 struct kvm *kvm; 6767 struct kvm_vcpu *vcpu; 6768 int send_ipi = 0; 6769 unsigned long i; 6770 6771 /* 6772 * We allow guests to temporarily run on slowing clocks, 6773 * provided we notify them after, or to run on accelerating 6774 * clocks, provided we notify them before. Thus time never 6775 * goes backwards. 6776 * 6777 * However, we have a problem. We can't atomically update 6778 * the frequency of a given CPU from this function; it is 6779 * merely a notifier, which can be called from any CPU. 6780 * Changing the TSC frequency at arbitrary points in time 6781 * requires a recomputation of local variables related to 6782 * the TSC for each VCPU. We must flag these local variables 6783 * to be updated and be sure the update takes place with the 6784 * new frequency before any guests proceed. 6785 * 6786 * Unfortunately, the combination of hotplug CPU and frequency 6787 * change creates an intractable locking scenario; the order 6788 * of when these callouts happen is undefined with respect to 6789 * CPU hotplug, and they can race with each other. As such, 6790 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is 6791 * undefined; you can actually have a CPU frequency change take 6792 * place in between the computation of X and the setting of the 6793 * variable. To protect against this problem, all updates of 6794 * the per_cpu tsc_khz variable are done in an interrupt 6795 * protected IPI, and all callers wishing to update the value 6796 * must wait for a synchronous IPI to complete (which is trivial 6797 * if the caller is on the CPU already). This establishes the 6798 * necessary total order on variable updates. 6799 * 6800 * Note that because a guest time update may take place 6801 * anytime after the setting of the VCPU's request bit, the 6802 * correct TSC value must be set before the request. However, 6803 * to ensure the update actually makes it to any guest which 6804 * starts running in hardware virtualization between the set 6805 * and the acquisition of the spinlock, we must also ping the 6806 * CPU after setting the request bit. 6807 * 6808 */ 6809 6810 smp_call_function_single(cpu, tsc_khz_changed, freq, 1); 6811 6812 mutex_lock(&kvm_lock); 6813 list_for_each_entry(kvm, &vm_list, vm_list) { 6814 kvm_for_each_vcpu(i, vcpu, kvm) { 6815 if (vcpu->cpu != cpu) 6816 continue; 6817 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 6818 if (vcpu->cpu != raw_smp_processor_id()) 6819 send_ipi = 1; 6820 } 6821 } 6822 mutex_unlock(&kvm_lock); 6823 6824 if (freq->old < freq->new && send_ipi) { 6825 /* 6826 * We upscale the frequency. Must make the guest 6827 * doesn't see old kvmclock values while running with 6828 * the new frequency, otherwise we risk the guest sees 6829 * time go backwards. 6830 * 6831 * In case we update the frequency for another cpu 6832 * (which might be in guest context) send an interrupt 6833 * to kick the cpu out of guest context. Next time 6834 * guest context is entered kvmclock will be updated, 6835 * so the guest will not see stale values. 6836 */ 6837 smp_call_function_single(cpu, tsc_khz_changed, freq, 1); 6838 } 6839 } 6840 6841 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 6842 void *data) 6843 { 6844 struct cpufreq_freqs *freq = data; 6845 int cpu; 6846 6847 if (val == CPUFREQ_PRECHANGE && freq->old > freq->new) 6848 return 0; 6849 if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new) 6850 return 0; 6851 6852 for_each_cpu(cpu, freq->policy->cpus) 6853 __kvmclock_cpufreq_notifier(freq, cpu); 6854 6855 return 0; 6856 } 6857 6858 static struct notifier_block kvmclock_cpufreq_notifier_block = { 6859 .notifier_call = kvmclock_cpufreq_notifier 6860 }; 6861 6862 static int kvmclock_cpu_online(unsigned int cpu) 6863 { 6864 tsc_khz_changed(NULL); 6865 return 0; 6866 } 6867 6868 static void kvm_timer_init(void) 6869 { 6870 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 6871 max_tsc_khz = tsc_khz; 6872 6873 if (IS_ENABLED(CONFIG_CPU_FREQ)) { 6874 struct cpufreq_policy *policy; 6875 int cpu; 6876 6877 cpu = get_cpu(); 6878 policy = cpufreq_cpu_get(cpu); 6879 if (policy) { 6880 if (policy->cpuinfo.max_freq) 6881 max_tsc_khz = policy->cpuinfo.max_freq; 6882 cpufreq_cpu_put(policy); 6883 } 6884 put_cpu(); 6885 } 6886 cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block, 6887 CPUFREQ_TRANSITION_NOTIFIER); 6888 6889 cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online", 6890 kvmclock_cpu_online, kvmclock_cpu_down_prep); 6891 } 6892 } 6893 6894 #ifdef CONFIG_X86_64 6895 static void pvclock_gtod_update_fn(struct work_struct *work) 6896 { 6897 struct kvm *kvm; 6898 struct kvm_vcpu *vcpu; 6899 unsigned long i; 6900 6901 mutex_lock(&kvm_lock); 6902 list_for_each_entry(kvm, &vm_list, vm_list) 6903 kvm_for_each_vcpu(i, vcpu, kvm) 6904 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 6905 atomic_set(&kvm_guest_has_master_clock, 0); 6906 mutex_unlock(&kvm_lock); 6907 } 6908 6909 static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn); 6910 6911 /* 6912 * Indirection to move queue_work() out of the tk_core.seq write held 6913 * region to prevent possible deadlocks against time accessors which 6914 * are invoked with work related locks held. 6915 */ 6916 static void pvclock_irq_work_fn(struct irq_work *w) 6917 { 6918 queue_work(system_long_wq, &pvclock_gtod_work); 6919 } 6920 6921 static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn); 6922 6923 /* 6924 * Notification about pvclock gtod data update. 6925 */ 6926 static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused, 6927 void *priv) 6928 { 6929 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 6930 struct timekeeper *tk = priv; 6931 6932 update_pvclock_gtod(tk); 6933 6934 /* 6935 * Disable master clock if host does not trust, or does not use, 6936 * TSC based clocksource. Delegate queue_work() to irq_work as 6937 * this is invoked with tk_core.seq write held. 6938 */ 6939 if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) && 6940 atomic_read(&kvm_guest_has_master_clock) != 0) 6941 irq_work_queue(&pvclock_irq_work); 6942 return 0; 6943 } 6944 6945 static struct notifier_block pvclock_gtod_notifier = { 6946 .notifier_call = pvclock_gtod_notify, 6947 }; 6948 #endif 6949 6950 void kvm_setup_xss_caps(void) 6951 { 6952 if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) 6953 kvm_caps.supported_xss = 0; 6954 6955 if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && 6956 !kvm_cpu_cap_has(X86_FEATURE_IBT)) 6957 kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; 6958 6959 if ((kvm_caps.supported_xss & XFEATURE_MASK_CET_ALL) != XFEATURE_MASK_CET_ALL) { 6960 kvm_cpu_cap_clear(X86_FEATURE_SHSTK); 6961 kvm_cpu_cap_clear(X86_FEATURE_IBT); 6962 kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL; 6963 } 6964 } 6965 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_setup_xss_caps); 6966 6967 static void kvm_setup_efer_caps(void) 6968 { 6969 /* Enable syscall by default because its emulated by KVM */ 6970 kvm_caps.supported_efer_bits = (u64)EFER_SCE; 6971 6972 if (kvm_cpu_cap_has(X86_FEATURE_LM)) 6973 kvm_caps.supported_efer_bits |= (EFER_LME | EFER_LMA); 6974 6975 if (kvm_cpu_cap_has(X86_FEATURE_NX)) 6976 kvm_caps.supported_efer_bits |= EFER_NX; 6977 6978 if (kvm_cpu_cap_has(X86_FEATURE_FXSR_OPT)) 6979 kvm_caps.supported_efer_bits |= EFER_FFXSR; 6980 6981 if (kvm_cpu_cap_has(X86_FEATURE_AUTOIBRS)) 6982 kvm_caps.supported_efer_bits |= EFER_AUTOIBRS; 6983 6984 if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { 6985 kvm_caps.supported_efer_bits |= EFER_SVME; 6986 6987 /* 6988 * Enumerating EFER_LMSLE_MBZ and allowing EFER.LMSLE=1 6989 * would be nonsensical. Note, vendor code sets the defeature 6990 * if KVM can't support EFER.LMSLE for any reason, i.e. this 6991 * needs to consult KVM's capabilities, not just raw CPUID. 6992 */ 6993 if (!kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) 6994 kvm_caps.supported_efer_bits |= EFER_LMSLE; 6995 } 6996 } 6997 6998 static void kvm_nested_ops_update(const struct kvm_x86_nested_ops *nested_ops) 6999 { 7000 memcpy(&kvm_nested_ops, nested_ops, sizeof(kvm_nested_ops)); 7001 7002 #define __KVM_X86_NESTED_OP(func) \ 7003 static_call_update(kvm_x86_nested_##func, kvm_nested_ops.func); 7004 #define KVM_X86_NESTED_OP(func) \ 7005 WARN_ON(!kvm_nested_ops.func); __KVM_X86_NESTED_OP(func) 7006 #define KVM_X86_NESTED_OP_OPTIONAL __KVM_X86_NESTED_OP 7007 #define KVM_X86_NESTED_OP_OPTIONAL_RET0(func) \ 7008 static_call_update(kvm_x86_nested_##func, (void *)kvm_nested_ops.func ? : \ 7009 (void *)__static_call_return0); 7010 #include <asm/kvm-x86-nested-ops.h> 7011 #undef __KVM_X86_NESTED_OP 7012 } 7013 7014 static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) 7015 { 7016 memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); 7017 7018 #define __KVM_X86_OP(func) \ 7019 static_call_update(kvm_x86_##func, kvm_x86_ops.func); 7020 #define KVM_X86_OP(func) \ 7021 WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func) 7022 #define KVM_X86_OP_OPTIONAL __KVM_X86_OP 7023 #define KVM_X86_OP_OPTIONAL_RET0(func) \ 7024 static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \ 7025 (void *)__static_call_return0); 7026 #include <asm/kvm-x86-ops.h> 7027 #undef __KVM_X86_OP 7028 7029 kvm_nested_ops_update(ops->nested_ops); 7030 7031 kvm_pmu_ops_update(ops->pmu_ops); 7032 } 7033 7034 static int kvm_x86_check_processor_compatibility(void) 7035 { 7036 int cpu = smp_processor_id(); 7037 struct cpuinfo_x86 *c = &cpu_data(cpu); 7038 7039 /* 7040 * Compatibility checks are done when loading KVM and when enabling 7041 * hardware, e.g. during CPU hotplug, to ensure all online CPUs are 7042 * compatible, i.e. KVM should never perform a compatibility check on 7043 * an offline CPU. 7044 */ 7045 WARN_ON(!cpu_online(cpu)); 7046 7047 if (__cr4_reserved_bits(cpu_has, c) != 7048 __cr4_reserved_bits(cpu_has, &boot_cpu_data)) 7049 return -EIO; 7050 7051 return kvm_x86_call(check_processor_compatibility)(); 7052 } 7053 7054 static void kvm_x86_check_cpu_compat(void *ret) 7055 { 7056 *(int *)ret = kvm_x86_check_processor_compatibility(); 7057 } 7058 7059 int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) 7060 { 7061 u64 host_pat; 7062 int r, cpu; 7063 7064 guard(mutex)(&vendor_module_lock); 7065 7066 if (kvm_x86_ops.enable_virtualization_cpu) { 7067 pr_err("already loaded vendor module '%s'\n", kvm_x86_ops.name); 7068 return -EEXIST; 7069 } 7070 7071 /* 7072 * KVM explicitly assumes that the guest has an FPU and 7073 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the 7074 * vCPU's FPU state as a fxregs_state struct. 7075 */ 7076 if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) { 7077 pr_err("inadequate fpu\n"); 7078 return -EOPNOTSUPP; 7079 } 7080 7081 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 7082 pr_err("RT requires X86_FEATURE_CONSTANT_TSC\n"); 7083 return -EOPNOTSUPP; 7084 } 7085 7086 /* 7087 * KVM assumes that PAT entry '0' encodes WB memtype and simply zeroes 7088 * the PAT bits in SPTEs. Bail if PAT[0] is programmed to something 7089 * other than WB. Note, EPT doesn't utilize the PAT, but don't bother 7090 * with an exception. PAT[0] is set to WB on RESET and also by the 7091 * kernel, i.e. failure indicates a kernel bug or broken firmware. 7092 */ 7093 if (rdmsrq_safe(MSR_IA32_CR_PAT, &host_pat) || 7094 (host_pat & GENMASK(2, 0)) != 6) { 7095 pr_err("host PAT[0] is not WB\n"); 7096 return -EIO; 7097 } 7098 7099 if (boot_cpu_has(X86_FEATURE_SHSTK) || boot_cpu_has(X86_FEATURE_IBT)) { 7100 rdmsrq(MSR_IA32_S_CET, kvm_host.s_cet); 7101 /* 7102 * Linux doesn't yet support supervisor shadow stacks (SSS), so 7103 * KVM doesn't save/restore the associated MSRs, i.e. KVM may 7104 * clobber the host values. Yell and refuse to load if SSS is 7105 * unexpectedly enabled, e.g. to avoid crashing the host. 7106 */ 7107 if (WARN_ON_ONCE(kvm_host.s_cet & CET_SHSTK_EN)) 7108 return -EIO; 7109 } 7110 7111 memset(&kvm_caps, 0, sizeof(kvm_caps)); 7112 7113 x86_emulator_cache = kvm_alloc_emulator_cache(); 7114 if (!x86_emulator_cache) { 7115 pr_err("failed to allocate cache for x86 emulator\n"); 7116 return -ENOMEM; 7117 } 7118 7119 r = kvm_mmu_vendor_module_init(); 7120 if (r) 7121 goto out_free_x86_emulator_cache; 7122 7123 kvm_caps.supported_vm_types = BIT(KVM_X86_DEFAULT_VM); 7124 kvm_caps.supported_mce_cap = MCG_CTL_P | MCG_SER_P; 7125 7126 if (boot_cpu_has(X86_FEATURE_XSAVE)) { 7127 kvm_host.xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 7128 kvm_caps.supported_xcr0 = kvm_host.xcr0 & KVM_SUPPORTED_XCR0; 7129 } 7130 7131 if (boot_cpu_has(X86_FEATURE_XSAVES)) { 7132 rdmsrq(MSR_IA32_XSS, kvm_host.xss); 7133 kvm_caps.supported_xss = kvm_host.xss & KVM_SUPPORTED_XSS; 7134 } 7135 7136 kvm_caps.supported_quirks = KVM_X86_VALID_QUIRKS; 7137 kvm_caps.inapplicable_quirks = KVM_X86_CONDITIONAL_QUIRKS; 7138 7139 rdmsrq_safe(MSR_EFER, &kvm_host.efer); 7140 7141 kvm_init_pmu_capability(ops->pmu_ops); 7142 7143 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) 7144 rdmsrq(MSR_IA32_ARCH_CAPABILITIES, kvm_host.arch_capabilities); 7145 7146 WARN_ON_ONCE(kvm_nr_uret_msrs); 7147 7148 r = ops->hardware_setup(); 7149 if (r != 0) 7150 goto out_mmu_exit; 7151 7152 kvm_setup_efer_caps(); 7153 7154 enable_device_posted_irqs &= enable_apicv && 7155 irq_remapping_cap(IRQ_POSTING_CAP); 7156 7157 kvm_ops_update(ops); 7158 7159 for_each_online_cpu(cpu) { 7160 smp_call_function_single(cpu, kvm_x86_check_cpu_compat, &r, 1); 7161 if (r < 0) 7162 goto out_unwind_ops; 7163 } 7164 7165 /* 7166 * Point of no return! DO NOT add error paths below this point unless 7167 * absolutely necessary, as most operations from this point forward 7168 * require unwinding. 7169 */ 7170 kvm_timer_init(); 7171 7172 if (pi_inject_timer == -1) 7173 pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER); 7174 #ifdef CONFIG_X86_64 7175 pvclock_gtod_register_notifier(&pvclock_gtod_notifier); 7176 7177 if (hypervisor_is_type(X86_HYPER_MS_HYPERV)) 7178 set_hv_tscchange_cb(kvm_hyperv_tsc_notifier); 7179 #endif 7180 7181 __kvm_register_perf_callbacks(ops->handle_intel_pt_intr, 7182 enable_mediated_pmu ? kvm_handle_guest_mediated_pmi : NULL); 7183 7184 if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && tdp_mmu_enabled) 7185 kvm_caps.supported_vm_types |= BIT(KVM_X86_SW_PROTECTED_VM); 7186 7187 /* KVM always ignores guest PAT for shadow paging. */ 7188 if (!tdp_enabled) 7189 kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; 7190 7191 if (kvm_caps.has_tsc_control) { 7192 /* 7193 * Make sure the user can only configure tsc_khz values that 7194 * fit into a signed integer, otherwise KVM_GET_TSC_KHZ would 7195 * return a negative value and confuse userspace. 7196 * A min value is not calculated because it will always 7197 * be 1 on all machines. 7198 */ 7199 u64 max = min(0x7fffffffULL, 7200 __scale_tsc(kvm_caps.max_tsc_scaling_ratio, tsc_khz)); 7201 kvm_caps.max_guest_tsc_khz = max; 7202 } 7203 kvm_caps.default_tsc_scaling_ratio = 1ULL << kvm_caps.tsc_scaling_ratio_frac_bits; 7204 kvm_init_msr_lists(); 7205 return 0; 7206 7207 out_unwind_ops: 7208 kvm_x86_ops.enable_virtualization_cpu = NULL; 7209 kvm_x86_call(hardware_unsetup)(); 7210 out_mmu_exit: 7211 kvm_destroy_user_return_msrs(); 7212 kvm_mmu_vendor_module_exit(); 7213 out_free_x86_emulator_cache: 7214 kmem_cache_destroy(x86_emulator_cache); 7215 return r; 7216 } 7217 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_x86_vendor_init); 7218 7219 void kvm_x86_vendor_exit(void) 7220 { 7221 kvm_unregister_perf_callbacks(); 7222 7223 #ifdef CONFIG_X86_64 7224 if (hypervisor_is_type(X86_HYPER_MS_HYPERV)) 7225 clear_hv_tscchange_cb(); 7226 #endif 7227 kvm_lapic_exit(); 7228 7229 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 7230 cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block, 7231 CPUFREQ_TRANSITION_NOTIFIER); 7232 cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE); 7233 } 7234 #ifdef CONFIG_X86_64 7235 pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier); 7236 irq_work_sync(&pvclock_irq_work); 7237 cancel_work_sync(&pvclock_gtod_work); 7238 #endif 7239 kvm_x86_call(hardware_unsetup)(); 7240 kvm_destroy_user_return_msrs(); 7241 kvm_mmu_vendor_module_exit(); 7242 kmem_cache_destroy(x86_emulator_cache); 7243 #ifdef CONFIG_KVM_XEN 7244 static_key_deferred_flush(&kvm_xen_enabled); 7245 WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key)); 7246 #endif 7247 mutex_lock(&vendor_module_lock); 7248 kvm_x86_ops.enable_virtualization_cpu = NULL; 7249 mutex_unlock(&vendor_module_lock); 7250 } 7251 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_x86_vendor_exit); 7252 7253 #ifdef CONFIG_X86_64 7254 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr, 7255 unsigned long clock_type) 7256 { 7257 struct kvm_clock_pairing clock_pairing; 7258 struct timespec64 ts; 7259 u64 cycle; 7260 int ret; 7261 7262 if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK) 7263 return -KVM_EOPNOTSUPP; 7264 7265 /* 7266 * When tsc is in permanent catchup mode guests won't be able to use 7267 * pvclock_read_retry loop to get consistent view of pvclock 7268 */ 7269 if (vcpu->arch.tsc_always_catchup) 7270 return -KVM_EOPNOTSUPP; 7271 7272 if (!kvm_get_walltime_and_clockread(&ts, &cycle)) 7273 return -KVM_EOPNOTSUPP; 7274 7275 clock_pairing.sec = ts.tv_sec; 7276 clock_pairing.nsec = ts.tv_nsec; 7277 clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle); 7278 clock_pairing.flags = 0; 7279 memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad)); 7280 7281 ret = 0; 7282 if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing, 7283 sizeof(struct kvm_clock_pairing))) 7284 ret = -KVM_EFAULT; 7285 7286 return ret; 7287 } 7288 #endif 7289 7290 /* 7291 * kvm_pv_kick_cpu_op: Kick a vcpu. 7292 * 7293 * @apicid - apicid of vcpu to be kicked. 7294 */ 7295 static void kvm_pv_kick_cpu_op(struct kvm *kvm, int apicid) 7296 { 7297 /* 7298 * All other fields are unused for APIC_DM_REMRD, but may be consumed by 7299 * common code, e.g. for tracing. Defer initialization to the compiler. 7300 */ 7301 struct kvm_lapic_irq lapic_irq = { 7302 .delivery_mode = APIC_DM_REMRD, 7303 .dest_mode = APIC_DEST_PHYSICAL, 7304 .shorthand = APIC_DEST_NOSHORT, 7305 .dest_id = apicid, 7306 }; 7307 7308 kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq); 7309 } 7310 7311 bool kvm_apicv_activated(struct kvm *kvm) 7312 { 7313 return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0); 7314 } 7315 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_apicv_activated); 7316 7317 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu) 7318 { 7319 ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons); 7320 ulong vcpu_reasons = 7321 kvm_x86_call(vcpu_get_apicv_inhibit_reasons)(vcpu); 7322 7323 return (vm_reasons | vcpu_reasons) == 0; 7324 } 7325 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_apicv_activated); 7326 7327 static void set_or_clear_apicv_inhibit(unsigned long *inhibits, 7328 enum kvm_apicv_inhibit reason, bool set) 7329 { 7330 const struct trace_print_flags apicv_inhibits[] = { APICV_INHIBIT_REASONS }; 7331 7332 BUILD_BUG_ON(ARRAY_SIZE(apicv_inhibits) != NR_APICV_INHIBIT_REASONS); 7333 7334 if (set) 7335 __set_bit(reason, inhibits); 7336 else 7337 __clear_bit(reason, inhibits); 7338 7339 trace_kvm_apicv_inhibit_changed(reason, set, *inhibits); 7340 } 7341 7342 static void kvm_apicv_init(struct kvm *kvm) 7343 { 7344 enum kvm_apicv_inhibit reason = enable_apicv ? APICV_INHIBIT_REASON_ABSENT : 7345 APICV_INHIBIT_REASON_DISABLED; 7346 7347 set_or_clear_apicv_inhibit(&kvm->arch.apicv_inhibit_reasons, reason, true); 7348 7349 init_rwsem(&kvm->arch.apicv_update_lock); 7350 } 7351 7352 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id) 7353 { 7354 struct kvm_vcpu *target = NULL; 7355 struct kvm_apic_map *map; 7356 7357 vcpu->stat.directed_yield_attempted++; 7358 7359 if (single_task_running()) 7360 goto no_yield; 7361 7362 rcu_read_lock(); 7363 map = rcu_dereference(vcpu->kvm->arch.apic_map); 7364 7365 if (likely(map) && dest_id <= map->max_apic_id) { 7366 dest_id = array_index_nospec(dest_id, map->max_apic_id + 1); 7367 if (map->phys_map[dest_id]) 7368 target = map->phys_map[dest_id]->vcpu; 7369 } 7370 7371 rcu_read_unlock(); 7372 7373 if (!target || !READ_ONCE(target->ready)) 7374 goto no_yield; 7375 7376 /* Ignore requests to yield to self */ 7377 if (vcpu == target) 7378 goto no_yield; 7379 7380 if (kvm_vcpu_yield_to(target) <= 0) 7381 goto no_yield; 7382 7383 vcpu->stat.directed_yield_successful++; 7384 7385 no_yield: 7386 return; 7387 } 7388 7389 static int complete_hypercall_exit(struct kvm_vcpu *vcpu) 7390 { 7391 u64 ret = vcpu->run->hypercall.ret; 7392 7393 if (!is_64_bit_hypercall(vcpu)) 7394 ret = (u32)ret; 7395 kvm_rax_write_raw(vcpu, ret); 7396 return kvm_skip_emulated_instruction(vcpu); 7397 } 7398 7399 int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl, 7400 int (*complete_hypercall)(struct kvm_vcpu *)) 7401 { 7402 int op_64_bit = is_64_bit_hypercall(vcpu); 7403 unsigned long ret, nr, a0, a1, a2, a3; 7404 7405 ++vcpu->stat.hypercalls; 7406 7407 if (op_64_bit) { 7408 nr = kvm_rax_read_raw(vcpu); 7409 a0 = kvm_rbx_read_raw(vcpu); 7410 a1 = kvm_rcx_read_raw(vcpu); 7411 a2 = kvm_rdx_read_raw(vcpu); 7412 a3 = kvm_rsi_read_raw(vcpu); 7413 } else { 7414 nr = kvm_eax_read(vcpu); 7415 a0 = kvm_ebx_read(vcpu); 7416 a1 = kvm_ecx_read(vcpu); 7417 a2 = kvm_edx_read(vcpu); 7418 a3 = kvm_esi_read(vcpu); 7419 } 7420 7421 trace_kvm_hypercall(nr, a0, a1, a2, a3); 7422 7423 if (cpl) { 7424 ret = -KVM_EPERM; 7425 goto out; 7426 } 7427 7428 ret = -KVM_ENOSYS; 7429 7430 switch (nr) { 7431 case KVM_HC_VAPIC_POLL_IRQ: 7432 ret = 0; 7433 break; 7434 case KVM_HC_KICK_CPU: 7435 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT)) 7436 break; 7437 7438 kvm_pv_kick_cpu_op(vcpu->kvm, a1); 7439 kvm_sched_yield(vcpu, a1); 7440 ret = 0; 7441 break; 7442 #ifdef CONFIG_X86_64 7443 case KVM_HC_CLOCK_PAIRING: 7444 ret = kvm_pv_clock_pairing(vcpu, a0, a1); 7445 break; 7446 #endif 7447 case KVM_HC_SEND_IPI: 7448 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI)) 7449 break; 7450 7451 ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit); 7452 break; 7453 case KVM_HC_SCHED_YIELD: 7454 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD)) 7455 break; 7456 7457 kvm_sched_yield(vcpu, a0); 7458 ret = 0; 7459 break; 7460 case KVM_HC_MAP_GPA_RANGE: { 7461 u64 gpa = a0, npages = a1, attrs = a2; 7462 7463 ret = -KVM_ENOSYS; 7464 if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE)) 7465 break; 7466 7467 if (!PAGE_ALIGNED(gpa) || !npages || 7468 gpa_to_gfn(gpa) + npages <= gpa_to_gfn(gpa)) { 7469 ret = -KVM_EINVAL; 7470 break; 7471 } 7472 7473 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; 7474 vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE; 7475 /* 7476 * In principle this should have been -KVM_ENOSYS, but userspace (QEMU <=9.2) 7477 * assumed that vcpu->run->hypercall.ret is never changed by KVM and thus that 7478 * it was always zero on KVM_EXIT_HYPERCALL. Since KVM is now overwriting 7479 * vcpu->run->hypercall.ret, ensuring that it is zero to not break QEMU. 7480 */ 7481 vcpu->run->hypercall.ret = 0; 7482 vcpu->run->hypercall.args[0] = gpa; 7483 vcpu->run->hypercall.args[1] = npages; 7484 vcpu->run->hypercall.args[2] = attrs; 7485 vcpu->run->hypercall.flags = 0; 7486 if (op_64_bit) 7487 vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE; 7488 7489 WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ); 7490 vcpu->arch.complete_userspace_io = complete_hypercall; 7491 return 0; 7492 } 7493 default: 7494 ret = -KVM_ENOSYS; 7495 break; 7496 } 7497 7498 out: 7499 vcpu->run->hypercall.ret = ret; 7500 return 1; 7501 } 7502 EXPORT_SYMBOL_FOR_KVM_INTERNAL(____kvm_emulate_hypercall); 7503 7504 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) 7505 { 7506 if (kvm_xen_hypercall_enabled(vcpu->kvm)) 7507 return kvm_xen_hypercall(vcpu); 7508 7509 if (kvm_hv_hypercall_enabled(vcpu)) 7510 return kvm_hv_hypercall(vcpu); 7511 7512 return __kvm_emulate_hypercall(vcpu, kvm_x86_call(get_cpl)(vcpu), 7513 complete_hypercall_exit); 7514 } 7515 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_hypercall); 7516 7517 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt) 7518 { 7519 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7520 char instruction[3]; 7521 unsigned long rip = kvm_rip_read(vcpu); 7522 7523 /* 7524 * If the quirk is disabled, synthesize a #UD and let the guest pick up 7525 * the pieces. 7526 */ 7527 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) { 7528 ctxt->exception.error_code_valid = false; 7529 ctxt->exception.vector = UD_VECTOR; 7530 ctxt->have_exception = true; 7531 return X86EMUL_PROPAGATE_FAULT; 7532 } 7533 7534 kvm_x86_call(patch_hypercall)(vcpu, instruction); 7535 7536 return emulator_write_emulated(ctxt, rip, instruction, 3, 7537 &ctxt->exception); 7538 } 7539 7540 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu) 7541 { 7542 return vcpu->run->request_interrupt_window && 7543 likely(!pic_in_kernel(vcpu->kvm)); 7544 } 7545 7546 /* Called within kvm->srcu read side. */ 7547 static void post_kvm_run_save(struct kvm_vcpu *vcpu) 7548 { 7549 struct kvm_run *kvm_run = vcpu->run; 7550 7551 kvm_run->if_flag = kvm_x86_call(get_if_flag)(vcpu); 7552 kvm_run->cr8 = kvm_get_cr8(vcpu); 7553 kvm_run->apic_base = vcpu->arch.apic_base; 7554 7555 kvm_run->ready_for_interrupt_injection = 7556 pic_in_kernel(vcpu->kvm) || 7557 kvm_vcpu_ready_for_interrupt_injection(vcpu); 7558 7559 if (is_smm(vcpu)) 7560 kvm_run->flags |= KVM_RUN_X86_SMM; 7561 if (is_guest_mode(vcpu)) 7562 kvm_run->flags |= KVM_RUN_X86_GUEST_MODE; 7563 } 7564 7565 int kvm_check_nested_events(struct kvm_vcpu *vcpu) 7566 { 7567 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 7568 kvm_nested_call(triple_fault)(vcpu); 7569 return 1; 7570 } 7571 7572 return kvm_nested_call(check_events)(vcpu); 7573 } 7574 7575 static void kvm_inject_exception(struct kvm_vcpu *vcpu) 7576 { 7577 /* 7578 * Suppress the error code if the vCPU is in Real Mode, as Real Mode 7579 * exceptions don't report error codes. The presence of an error code 7580 * is carried with the exception and only stripped when the exception 7581 * is injected as intercepted #PF VM-Exits for AMD's Paged Real Mode do 7582 * report an error code despite the CPU being in Real Mode. 7583 */ 7584 vcpu->arch.exception.has_error_code &= is_protmode(vcpu); 7585 7586 trace_kvm_inj_exception(vcpu->arch.exception.vector, 7587 vcpu->arch.exception.has_error_code, 7588 vcpu->arch.exception.error_code, 7589 vcpu->arch.exception.injected); 7590 7591 kvm_x86_call(inject_exception)(vcpu); 7592 } 7593 7594 /* 7595 * Check for any event (interrupt or exception) that is ready to be injected, 7596 * and if there is at least one event, inject the event with the highest 7597 * priority. This handles both "pending" events, i.e. events that have never 7598 * been injected into the guest, and "injected" events, i.e. events that were 7599 * injected as part of a previous VM-Enter, but weren't successfully delivered 7600 * and need to be re-injected. 7601 * 7602 * Note, this is not guaranteed to be invoked on a guest instruction boundary, 7603 * i.e. doesn't guarantee that there's an event window in the guest. KVM must 7604 * be able to inject exceptions in the "middle" of an instruction, and so must 7605 * also be able to re-inject NMIs and IRQs in the middle of an instruction. 7606 * I.e. for exceptions and re-injected events, NOT invoking this on instruction 7607 * boundaries is necessary and correct. 7608 * 7609 * For simplicity, KVM uses a single path to inject all events (except events 7610 * that are injected directly from L1 to L2) and doesn't explicitly track 7611 * instruction boundaries for asynchronous events. However, because VM-Exits 7612 * that can occur during instruction execution typically result in KVM skipping 7613 * the instruction or injecting an exception, e.g. instruction and exception 7614 * intercepts, and because pending exceptions have higher priority than pending 7615 * interrupts, KVM still honors instruction boundaries in most scenarios. 7616 * 7617 * But, if a VM-Exit occurs during instruction execution, and KVM does NOT skip 7618 * the instruction or inject an exception, then KVM can incorrecty inject a new 7619 * asynchronous event if the event became pending after the CPU fetched the 7620 * instruction (in the guest). E.g. if a page fault (#PF, #NPF, EPT violation) 7621 * occurs and is resolved by KVM, a coincident NMI, SMI, IRQ, etc... can be 7622 * injected on the restarted instruction instead of being deferred until the 7623 * instruction completes. 7624 * 7625 * In practice, this virtualization hole is unlikely to be observed by the 7626 * guest, and even less likely to cause functional problems. To detect the 7627 * hole, the guest would have to trigger an event on a side effect of an early 7628 * phase of instruction execution, e.g. on the instruction fetch from memory. 7629 * And for it to be a functional problem, the guest would need to depend on the 7630 * ordering between that side effect, the instruction completing, _and_ the 7631 * delivery of the asynchronous event. 7632 */ 7633 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu, 7634 bool *req_immediate_exit) 7635 { 7636 bool can_inject; 7637 int r; 7638 7639 /* 7640 * Process nested events first, as nested VM-Exit supersedes event 7641 * re-injection. If there's an event queued for re-injection, it will 7642 * be saved into the appropriate vmc{b,s}12 fields on nested VM-Exit. 7643 */ 7644 if (is_guest_mode(vcpu)) 7645 r = kvm_check_nested_events(vcpu); 7646 else 7647 r = 0; 7648 7649 /* 7650 * Re-inject exceptions and events *especially* if immediate entry+exit 7651 * to/from L2 is needed, as any event that has already been injected 7652 * into L2 needs to complete its lifecycle before injecting a new event. 7653 * 7654 * Don't re-inject an NMI or interrupt if there is a pending exception. 7655 * This collision arises if an exception occurred while vectoring the 7656 * injected event, KVM intercepted said exception, and KVM ultimately 7657 * determined the fault belongs to the guest and queues the exception 7658 * for injection back into the guest. 7659 * 7660 * "Injected" interrupts can also collide with pending exceptions if 7661 * userspace ignores the "ready for injection" flag and blindly queues 7662 * an interrupt. In that case, prioritizing the exception is correct, 7663 * as the exception "occurred" before the exit to userspace. Trap-like 7664 * exceptions, e.g. most #DBs, have higher priority than interrupts. 7665 * And while fault-like exceptions, e.g. #GP and #PF, are the lowest 7666 * priority, they're only generated (pended) during instruction 7667 * execution, and interrupts are recognized at instruction boundaries. 7668 * Thus a pending fault-like exception means the fault occurred on the 7669 * *previous* instruction and must be serviced prior to recognizing any 7670 * new events in order to fully complete the previous instruction. 7671 */ 7672 if (vcpu->arch.exception.injected) 7673 kvm_inject_exception(vcpu); 7674 else if (kvm_is_exception_pending(vcpu)) 7675 ; /* see above */ 7676 else if (vcpu->arch.nmi_injected) 7677 kvm_x86_call(inject_nmi)(vcpu); 7678 else if (vcpu->arch.interrupt.injected) 7679 kvm_x86_call(inject_irq)(vcpu, true); 7680 7681 /* 7682 * Exceptions that morph to VM-Exits are handled above, and pending 7683 * exceptions on top of injected exceptions that do not VM-Exit should 7684 * either morph to #DF or, sadly, override the injected exception. 7685 */ 7686 WARN_ON_ONCE(vcpu->arch.exception.injected && 7687 vcpu->arch.exception.pending); 7688 7689 /* 7690 * Bail if immediate entry+exit to/from the guest is needed to complete 7691 * nested VM-Enter or event re-injection so that a different pending 7692 * event can be serviced (or if KVM needs to exit to userspace). 7693 * 7694 * Otherwise, continue processing events even if VM-Exit occurred. The 7695 * VM-Exit will have cleared exceptions that were meant for L2, but 7696 * there may now be events that can be injected into L1. 7697 */ 7698 if (r < 0) 7699 goto out; 7700 7701 /* 7702 * A pending exception VM-Exit should either result in nested VM-Exit 7703 * or force an immediate re-entry and exit to/from L2, and exception 7704 * VM-Exits cannot be injected (flag should _never_ be set). 7705 */ 7706 WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected || 7707 vcpu->arch.exception_vmexit.pending); 7708 7709 /* 7710 * New events, other than exceptions, cannot be injected if KVM needs 7711 * to re-inject a previous event. See above comments on re-injecting 7712 * for why pending exceptions get priority. 7713 */ 7714 can_inject = !kvm_event_needs_reinjection(vcpu); 7715 7716 if (vcpu->arch.exception.pending) { 7717 /* 7718 * Fault-class exceptions, except #DBs, set RF=1 in the RFLAGS 7719 * value pushed on the stack. Trap-like exception and all #DBs 7720 * leave RF as-is (KVM follows Intel's behavior in this regard; 7721 * AMD states that code breakpoint #DBs excplitly clear RF=0). 7722 * 7723 * Note, most versions of Intel's SDM and AMD's APM incorrectly 7724 * describe the behavior of General Detect #DBs, which are 7725 * fault-like. They do _not_ set RF, a la code breakpoints. 7726 */ 7727 if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT) 7728 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) | 7729 X86_EFLAGS_RF); 7730 7731 if (vcpu->arch.exception.vector == DB_VECTOR && 7732 vcpu->arch.dr7 & DR7_GD) { 7733 vcpu->arch.dr7 &= ~DR7_GD; 7734 kvm_update_dr7(vcpu); 7735 } 7736 7737 kvm_inject_exception(vcpu); 7738 7739 vcpu->arch.exception.pending = false; 7740 vcpu->arch.exception.injected = true; 7741 7742 can_inject = false; 7743 } 7744 7745 /* Don't inject interrupts if the user asked to avoid doing so */ 7746 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) 7747 return 0; 7748 7749 /* 7750 * Finally, inject interrupt events. If an event cannot be injected 7751 * due to architectural conditions (e.g. IF=0) a window-open exit 7752 * will re-request KVM_REQ_EVENT. Sometimes however an event is pending 7753 * and can architecturally be injected, but we cannot do it right now: 7754 * an interrupt could have arrived just now and we have to inject it 7755 * as a vmexit, or there could already an event in the queue, which is 7756 * indicated by can_inject. In that case we request an immediate exit 7757 * in order to make progress and get back here for another iteration. 7758 * The kvm_x86_ops hooks communicate this by returning -EBUSY. 7759 */ 7760 #ifdef CONFIG_KVM_SMM 7761 if (vcpu->arch.smi_pending) { 7762 r = can_inject ? kvm_x86_call(smi_allowed)(vcpu, true) : 7763 -EBUSY; 7764 if (r < 0) 7765 goto out; 7766 if (r) { 7767 vcpu->arch.smi_pending = false; 7768 ++vcpu->arch.smi_count; 7769 enter_smm(vcpu); 7770 can_inject = false; 7771 } else 7772 kvm_x86_call(enable_smi_window)(vcpu); 7773 } 7774 #endif 7775 7776 if (vcpu->arch.nmi_pending) { 7777 r = can_inject ? kvm_x86_call(nmi_allowed)(vcpu, true) : 7778 -EBUSY; 7779 if (r < 0) 7780 goto out; 7781 if (r) { 7782 --vcpu->arch.nmi_pending; 7783 vcpu->arch.nmi_injected = true; 7784 kvm_x86_call(inject_nmi)(vcpu); 7785 can_inject = false; 7786 WARN_ON(kvm_x86_call(nmi_allowed)(vcpu, true) < 0); 7787 } 7788 if (vcpu->arch.nmi_pending) 7789 kvm_x86_call(enable_nmi_window)(vcpu); 7790 } 7791 7792 if (kvm_cpu_has_injectable_intr(vcpu)) { 7793 r = can_inject ? kvm_x86_call(interrupt_allowed)(vcpu, true) : 7794 -EBUSY; 7795 if (r < 0) 7796 goto out; 7797 if (r) { 7798 int irq = kvm_cpu_get_interrupt(vcpu); 7799 7800 if (likely(irq != -1)) { 7801 kvm_queue_interrupt(vcpu, irq, false); 7802 kvm_x86_call(inject_irq)(vcpu, false); 7803 WARN_ON(kvm_x86_call(interrupt_allowed)(vcpu, true) < 0); 7804 } else { 7805 kvm_warn_on_lost_irq(vcpu); 7806 } 7807 } 7808 if (kvm_cpu_has_injectable_intr(vcpu)) 7809 kvm_x86_call(enable_irq_window)(vcpu); 7810 } 7811 7812 if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, true)) 7813 *req_immediate_exit = true; 7814 7815 /* 7816 * KVM must never queue a new exception while injecting an event; KVM 7817 * is done emulating and should only propagate the to-be-injected event 7818 * to the VMCS/VMCB. Queueing a new exception can put the vCPU into an 7819 * infinite loop as KVM will bail from VM-Enter to inject the pending 7820 * exception and start the cycle all over. 7821 * 7822 * Exempt triple faults as they have special handling and won't put the 7823 * vCPU into an infinite loop. Triple fault can be queued when running 7824 * VMX without unrestricted guest, as that requires KVM to emulate Real 7825 * Mode events (see kvm_inject_realmode_interrupt()). 7826 */ 7827 WARN_ON_ONCE(vcpu->arch.exception.pending || 7828 vcpu->arch.exception_vmexit.pending); 7829 return 0; 7830 7831 out: 7832 if (r == -EBUSY) { 7833 *req_immediate_exit = true; 7834 r = 0; 7835 } 7836 return r; 7837 } 7838 7839 static void process_nmi(struct kvm_vcpu *vcpu) 7840 { 7841 unsigned int limit; 7842 7843 /* 7844 * x86 is limited to one NMI pending, but because KVM can't react to 7845 * incoming NMIs as quickly as bare metal, e.g. if the vCPU is 7846 * scheduled out, KVM needs to play nice with two queued NMIs showing 7847 * up at the same time. To handle this scenario, allow two NMIs to be 7848 * (temporarily) pending so long as NMIs are not blocked and KVM is not 7849 * waiting for a previous NMI injection to complete (which effectively 7850 * blocks NMIs). KVM will immediately inject one of the two NMIs, and 7851 * will request an NMI window to handle the second NMI. 7852 */ 7853 if (kvm_x86_call(get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected) 7854 limit = 1; 7855 else 7856 limit = 2; 7857 7858 /* 7859 * Adjust the limit to account for pending virtual NMIs, which aren't 7860 * tracked in vcpu->arch.nmi_pending. 7861 */ 7862 if (kvm_x86_call(is_vnmi_pending)(vcpu)) 7863 limit--; 7864 7865 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0); 7866 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit); 7867 7868 if (vcpu->arch.nmi_pending && 7869 (kvm_x86_call(set_vnmi_pending)(vcpu))) 7870 vcpu->arch.nmi_pending--; 7871 7872 if (vcpu->arch.nmi_pending) 7873 kvm_make_request(KVM_REQ_EVENT, vcpu); 7874 } 7875 7876 /* Return total number of NMIs pending injection to the VM */ 7877 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu) 7878 { 7879 return vcpu->arch.nmi_pending + 7880 kvm_x86_call(is_vnmi_pending)(vcpu); 7881 } 7882 7883 void kvm_make_scan_ioapic_request_mask(struct kvm *kvm, 7884 unsigned long *vcpu_bitmap) 7885 { 7886 kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, vcpu_bitmap); 7887 } 7888 7889 void kvm_make_scan_ioapic_request(struct kvm *kvm) 7890 { 7891 kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC); 7892 } 7893 7894 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) 7895 { 7896 struct kvm_lapic *apic = vcpu->arch.apic; 7897 bool activate; 7898 7899 if (!lapic_in_kernel(vcpu)) 7900 return; 7901 7902 down_read(&vcpu->kvm->arch.apicv_update_lock); 7903 preempt_disable(); 7904 7905 /* Do not activate APICV when APIC is disabled */ 7906 activate = kvm_vcpu_apicv_activated(vcpu) && 7907 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED); 7908 7909 if (apic->apicv_active == activate) 7910 goto out; 7911 7912 apic->apicv_active = activate; 7913 kvm_apic_update_apicv(vcpu); 7914 kvm_x86_call(refresh_apicv_exec_ctrl)(vcpu); 7915 7916 /* 7917 * When APICv gets disabled, we may still have injected interrupts 7918 * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was 7919 * still active when the interrupt got accepted. Make sure 7920 * kvm_check_and_inject_events() is called to check for that. 7921 */ 7922 if (!apic->apicv_active) 7923 kvm_make_request(KVM_REQ_EVENT, vcpu); 7924 7925 out: 7926 preempt_enable(); 7927 up_read(&vcpu->kvm->arch.apicv_update_lock); 7928 } 7929 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_vcpu_update_apicv); 7930 7931 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) 7932 { 7933 if (!lapic_in_kernel(vcpu)) 7934 return; 7935 7936 /* 7937 * Due to sharing page tables across vCPUs, the xAPIC memslot must be 7938 * deleted if any vCPU has xAPIC virtualization and x2APIC enabled, but 7939 * and hardware doesn't support x2APIC virtualization. E.g. some AMD 7940 * CPUs support AVIC but not x2APIC. KVM still allows enabling AVIC in 7941 * this case so that KVM can use the AVIC doorbell to inject interrupts 7942 * to running vCPUs, but KVM must not create SPTEs for the APIC base as 7943 * the vCPU would incorrectly be able to access the vAPIC page via MMIO 7944 * despite being in x2APIC mode. For simplicity, inhibiting the APIC 7945 * access page is sticky. 7946 */ 7947 if (apic_x2apic_mode(vcpu->arch.apic) && 7948 kvm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization) 7949 kvm_inhibit_apic_access_page(vcpu); 7950 7951 __kvm_vcpu_update_apicv(vcpu); 7952 } 7953 7954 void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, 7955 enum kvm_apicv_inhibit reason, bool set) 7956 { 7957 unsigned long old, new; 7958 7959 lockdep_assert_held_write(&kvm->arch.apicv_update_lock); 7960 7961 if (!(kvm_x86_ops.required_apicv_inhibits & BIT(reason))) 7962 return; 7963 7964 old = new = kvm->arch.apicv_inhibit_reasons; 7965 7966 if (reason != APICV_INHIBIT_REASON_IRQWIN) 7967 set_or_clear_apicv_inhibit(&new, reason, set); 7968 7969 set_or_clear_apicv_inhibit(&new, APICV_INHIBIT_REASON_IRQWIN, 7970 atomic_read(&kvm->arch.apicv_nr_irq_window_req)); 7971 7972 if (!!old != !!new) { 7973 /* 7974 * Kick all vCPUs before setting apicv_inhibit_reasons to avoid 7975 * false positives in the sanity check WARN in vcpu_enter_guest(). 7976 * This task will wait for all vCPUs to ack the kick IRQ before 7977 * updating apicv_inhibit_reasons, and all other vCPUs will 7978 * block on acquiring apicv_update_lock so that vCPUs can't 7979 * redo vcpu_enter_guest() without seeing the new inhibit state. 7980 * 7981 * Note, holding apicv_update_lock and taking it in the read 7982 * side (handling the request) also prevents other vCPUs from 7983 * servicing the request with a stale apicv_inhibit_reasons. 7984 */ 7985 kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE); 7986 kvm->arch.apicv_inhibit_reasons = new; 7987 if (new) { 7988 unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE); 7989 7990 kvm_zap_gfn_range(kvm, gfn, gfn+1); 7991 } 7992 } else { 7993 kvm->arch.apicv_inhibit_reasons = new; 7994 } 7995 } 7996 7997 void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, 7998 enum kvm_apicv_inhibit reason, bool set) 7999 { 8000 if (!enable_apicv) 8001 return; 8002 8003 down_write(&kvm->arch.apicv_update_lock); 8004 __kvm_set_or_clear_apicv_inhibit(kvm, reason, set); 8005 up_write(&kvm->arch.apicv_update_lock); 8006 } 8007 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_or_clear_apicv_inhibit); 8008 8009 void kvm_inc_or_dec_irq_window_inhibit(struct kvm *kvm, bool inc) 8010 { 8011 int add = inc ? 1 : -1; 8012 8013 if (!enable_apicv) 8014 return; 8015 8016 /* 8017 * IRQ windows are requested either because of ExtINT injections, or 8018 * because APICv is already disabled/inhibited for another reason. 8019 * While ExtINT injections are rare and should not happen while the 8020 * vCPU is running its actual workload, it's worth avoiding thrashing 8021 * if the IRQ window is being requested because APICv is already 8022 * inhibited. So, toggle the actual inhibit (which requires taking 8023 * the lock for write) if and only if there's no other inhibit. 8024 * kvm_set_or_clear_apicv_inhibit() always evaluates the IRQ window 8025 * count; thus the IRQ window inhibit call _will_ be lazily updated on 8026 * the next call, if it ever happens. 8027 */ 8028 if (READ_ONCE(kvm->arch.apicv_inhibit_reasons) & ~BIT(APICV_INHIBIT_REASON_IRQWIN)) { 8029 guard(rwsem_read)(&kvm->arch.apicv_update_lock); 8030 if (READ_ONCE(kvm->arch.apicv_inhibit_reasons) & ~BIT(APICV_INHIBIT_REASON_IRQWIN)) { 8031 atomic_add(add, &kvm->arch.apicv_nr_irq_window_req); 8032 return; 8033 } 8034 } 8035 8036 /* 8037 * Strictly speaking, the lock is only needed if going 0->1 or 1->0, 8038 * a la atomic_dec_and_mutex_lock. However, ExtINTs are rare and 8039 * only target a single CPU, so that is the common case; do not 8040 * bother eliding the down_write()/up_write() pair. 8041 */ 8042 guard(rwsem_write)(&kvm->arch.apicv_update_lock); 8043 if (atomic_add_return(add, &kvm->arch.apicv_nr_irq_window_req) == inc) 8044 __kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_IRQWIN, inc); 8045 } 8046 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_inc_or_dec_irq_window_inhibit); 8047 8048 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) 8049 { 8050 if (!kvm_apic_present(vcpu)) 8051 return; 8052 8053 bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256); 8054 vcpu->arch.highest_stale_pending_ioapic_eoi = -1; 8055 8056 kvm_x86_call(sync_pir_to_irr)(vcpu); 8057 8058 if (irqchip_split(vcpu->kvm)) 8059 kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors); 8060 #ifdef CONFIG_KVM_IOAPIC 8061 else if (ioapic_in_kernel(vcpu->kvm)) 8062 kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors); 8063 #endif 8064 8065 if (is_guest_mode(vcpu)) 8066 vcpu->arch.load_eoi_exitmap_pending = true; 8067 else 8068 kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu); 8069 } 8070 8071 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) 8072 { 8073 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 8074 return; 8075 8076 #ifdef CONFIG_KVM_HYPERV 8077 if (to_hv_vcpu(vcpu)) { 8078 u64 eoi_exit_bitmap[4]; 8079 8080 bitmap_or((ulong *)eoi_exit_bitmap, 8081 vcpu->arch.ioapic_handled_vectors, 8082 to_hv_synic(vcpu)->vec_bitmap, 256); 8083 kvm_x86_call(load_eoi_exitmap)(vcpu, eoi_exit_bitmap); 8084 return; 8085 } 8086 #endif 8087 kvm_x86_call(load_eoi_exitmap)( 8088 vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors); 8089 } 8090 8091 void kvm_arch_guest_memory_reclaimed(struct kvm *kvm) 8092 { 8093 kvm_x86_call(guest_memory_reclaimed)(kvm); 8094 } 8095 8096 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) 8097 { 8098 if (!lapic_in_kernel(vcpu)) 8099 return; 8100 8101 kvm_x86_call(set_apic_access_page_addr)(vcpu); 8102 } 8103 8104 static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 8105 { 8106 /* 8107 * Note, nr_memslots_dirty_logging can be changed concurrent with this 8108 * code, but in that case another update request will be made and so 8109 * the guest will never run with a stale PML value. 8110 */ 8111 bool enable = atomic_read(&vcpu->kvm->nr_memslots_dirty_logging); 8112 8113 if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) 8114 return; 8115 8116 kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable); 8117 } 8118 8119 /* 8120 * Called within kvm->srcu read side. 8121 * Returns 1 to let vcpu_run() continue the guest execution loop without 8122 * exiting to the userspace. Otherwise, the value will be returned to the 8123 * userspace. 8124 */ 8125 static int vcpu_enter_guest(struct kvm_vcpu *vcpu) 8126 { 8127 int r; 8128 bool req_int_win = 8129 dm_request_for_irq_injection(vcpu) && 8130 kvm_cpu_accept_dm_intr(vcpu); 8131 fastpath_t exit_fastpath; 8132 u64 run_flags, debug_ctl; 8133 8134 bool req_immediate_exit = false; 8135 8136 if (kvm_request_pending(vcpu)) { 8137 if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) { 8138 r = -EIO; 8139 goto out; 8140 } 8141 8142 if (kvm_dirty_ring_check_request(vcpu)) { 8143 r = 0; 8144 goto out; 8145 } 8146 8147 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { 8148 if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) { 8149 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 8150 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 8151 vcpu->run->internal.ndata = 0; 8152 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 8153 r = 0; 8154 goto out; 8155 } 8156 } 8157 if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)) 8158 kvm_mmu_free_obsolete_roots(vcpu); 8159 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu)) 8160 __kvm_migrate_timers(vcpu); 8161 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) 8162 kvm_update_masterclock(vcpu->kvm); 8163 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu)) 8164 kvm_gen_kvmclock_update(vcpu); 8165 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) { 8166 r = kvm_guest_time_update(vcpu); 8167 if (unlikely(r)) 8168 goto out; 8169 } 8170 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu)) 8171 kvm_mmu_sync_roots(vcpu); 8172 if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu)) 8173 kvm_mmu_load_pgd(vcpu); 8174 8175 /* 8176 * Note, the order matters here, as flushing "all" TLB entries 8177 * also flushes the "current" TLB entries, i.e. servicing the 8178 * flush "all" will clear any request to flush "current". 8179 */ 8180 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 8181 kvm_vcpu_flush_tlb_all(vcpu); 8182 8183 kvm_service_local_tlb_flush_requests(vcpu); 8184 8185 /* 8186 * Fall back to a "full" guest flush if Hyper-V's precise 8187 * flushing fails. Note, Hyper-V's flushing is per-vCPU, but 8188 * the flushes are considered "remote" and not "local" because 8189 * the requests can be initiated from other vCPUs. 8190 */ 8191 #ifdef CONFIG_KVM_HYPERV 8192 if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) && 8193 kvm_hv_vcpu_flush_tlb(vcpu)) 8194 kvm_vcpu_flush_tlb_guest(vcpu); 8195 #endif 8196 8197 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { 8198 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; 8199 r = 0; 8200 goto out; 8201 } 8202 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 8203 if (is_guest_mode(vcpu)) 8204 kvm_nested_call(triple_fault)(vcpu); 8205 8206 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 8207 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 8208 vcpu->mmio_needed = 0; 8209 r = 0; 8210 goto out; 8211 } 8212 } 8213 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) { 8214 /* Page is swapped out. Do synthetic halt */ 8215 vcpu->arch.apf.halted = true; 8216 r = 1; 8217 goto out; 8218 } 8219 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 8220 record_steal_time(vcpu); 8221 if (kvm_check_request(KVM_REQ_PMU, vcpu)) 8222 kvm_pmu_handle_event(vcpu); 8223 if (kvm_check_request(KVM_REQ_PMI, vcpu)) 8224 kvm_pmu_deliver_pmi(vcpu); 8225 #ifdef CONFIG_KVM_SMM 8226 if (kvm_check_request(KVM_REQ_SMI, vcpu)) 8227 process_smi(vcpu); 8228 #endif 8229 if (kvm_check_request(KVM_REQ_NMI, vcpu)) 8230 process_nmi(vcpu); 8231 if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { 8232 if (test_bit(vcpu->arch.pending_ioapic_eoi, 8233 vcpu->arch.ioapic_handled_vectors)) { 8234 vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; 8235 vcpu->run->eoi.vector = 8236 vcpu->arch.pending_ioapic_eoi; 8237 r = 0; 8238 goto out; 8239 } 8240 } 8241 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu)) 8242 vcpu_scan_ioapic(vcpu); 8243 if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu)) 8244 vcpu_load_eoi_exitmap(vcpu); 8245 if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu)) 8246 kvm_vcpu_reload_apic_access_page(vcpu); 8247 #ifdef CONFIG_KVM_HYPERV 8248 if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) { 8249 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 8250 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH; 8251 vcpu->run->system_event.ndata = 0; 8252 r = 0; 8253 goto out; 8254 } 8255 if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) { 8256 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 8257 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET; 8258 vcpu->run->system_event.ndata = 0; 8259 r = 0; 8260 goto out; 8261 } 8262 if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) { 8263 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 8264 8265 vcpu->run->exit_reason = KVM_EXIT_HYPERV; 8266 vcpu->run->hyperv = hv_vcpu->exit; 8267 r = 0; 8268 goto out; 8269 } 8270 8271 /* 8272 * KVM_REQ_HV_STIMER has to be processed after 8273 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers 8274 * depend on the guest clock being up-to-date 8275 */ 8276 if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu)) 8277 kvm_hv_process_stimers(vcpu); 8278 #endif 8279 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) 8280 kvm_vcpu_update_apicv(vcpu); 8281 if (kvm_check_request(KVM_REQ_APF_READY, vcpu)) 8282 kvm_check_async_pf_completion(vcpu); 8283 8284 if (kvm_check_request(KVM_REQ_RECALC_INTERCEPTS, vcpu)) 8285 kvm_x86_call(recalc_intercepts)(vcpu); 8286 8287 if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) 8288 kvm_update_cpu_dirty_logging(vcpu); 8289 8290 if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) { 8291 kvm_vcpu_reset(vcpu, true); 8292 if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE) { 8293 r = 1; 8294 goto out; 8295 } 8296 } 8297 if (kvm_check_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu)) 8298 kvm_x86_call(reload_vmsa)(vcpu); 8299 } 8300 8301 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || 8302 kvm_xen_has_interrupt(vcpu)) { 8303 ++vcpu->stat.req_event; 8304 r = kvm_apic_accept_events(vcpu); 8305 if (r < 0) { 8306 r = 0; 8307 goto out; 8308 } 8309 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 8310 r = 1; 8311 goto out; 8312 } 8313 8314 r = kvm_check_and_inject_events(vcpu, &req_immediate_exit); 8315 if (r < 0) { 8316 r = 0; 8317 goto out; 8318 } 8319 if (req_int_win) 8320 kvm_x86_call(enable_irq_window)(vcpu); 8321 8322 if (kvm_lapic_enabled(vcpu)) { 8323 kvm_lapic_update_cr8_intercept(vcpu); 8324 kvm_lapic_sync_to_vapic(vcpu); 8325 } 8326 } 8327 8328 r = kvm_mmu_reload(vcpu); 8329 if (unlikely(r)) { 8330 goto cancel_injection; 8331 } 8332 8333 preempt_disable(); 8334 8335 kvm_x86_call(prepare_switch_to_guest)(vcpu); 8336 8337 /* 8338 * Disable IRQs before setting IN_GUEST_MODE. Posted interrupt 8339 * IPI are then delayed after guest entry, which ensures that they 8340 * result in virtual interrupt delivery. 8341 */ 8342 local_irq_disable(); 8343 8344 /* Store vcpu->apicv_active before vcpu->mode. */ 8345 smp_store_release(&vcpu->mode, IN_GUEST_MODE); 8346 8347 kvm_vcpu_srcu_read_unlock(vcpu); 8348 8349 /* 8350 * 1) We should set ->mode before checking ->requests. Please see 8351 * the comment in kvm_vcpu_exiting_guest_mode(). 8352 * 8353 * 2) For APICv, we should set ->mode before checking PID.ON. This 8354 * pairs with the memory barrier implicit in pi_test_and_set_on 8355 * (see vmx_deliver_posted_interrupt). 8356 * 8357 * 3) This also orders the write to mode from any reads to the page 8358 * tables done while the VCPU is running. Please see the comment 8359 * in kvm_flush_remote_tlbs. 8360 */ 8361 smp_mb__after_srcu_read_unlock(); 8362 8363 /* 8364 * Process pending posted interrupts to handle the case where the 8365 * notification IRQ arrived in the host, or was never sent (because the 8366 * target vCPU wasn't running). Do this regardless of the vCPU's APICv 8367 * status, KVM doesn't update assigned devices when APICv is inhibited, 8368 * i.e. they can post interrupts even if APICv is temporarily disabled. 8369 */ 8370 if (kvm_lapic_enabled(vcpu)) 8371 kvm_x86_call(sync_pir_to_irr)(vcpu); 8372 8373 if (kvm_vcpu_exit_request(vcpu)) { 8374 vcpu->mode = OUTSIDE_GUEST_MODE; 8375 smp_wmb(); 8376 local_irq_enable(); 8377 preempt_enable(); 8378 kvm_vcpu_srcu_read_lock(vcpu); 8379 r = 1; 8380 goto cancel_injection; 8381 } 8382 8383 run_flags = 0; 8384 if (req_immediate_exit) { 8385 run_flags |= KVM_RUN_FORCE_IMMEDIATE_EXIT; 8386 kvm_make_request(KVM_REQ_EVENT, vcpu); 8387 } 8388 8389 fpregs_assert_state_consistent(); 8390 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 8391 switch_fpu_return(); 8392 8393 if (vcpu->arch.guest_fpu.xfd_err) 8394 wrmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); 8395 8396 kvm_load_xfeatures(vcpu, true); 8397 8398 if (unlikely(vcpu->arch.switch_db_regs && 8399 !(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH))) { 8400 set_debugreg(DR7_FIXED_1, 7); 8401 set_debugreg(vcpu->arch.eff_db[0], 0); 8402 set_debugreg(vcpu->arch.eff_db[1], 1); 8403 set_debugreg(vcpu->arch.eff_db[2], 2); 8404 set_debugreg(vcpu->arch.eff_db[3], 3); 8405 /* When KVM_DEBUGREG_WONT_EXIT, dr6 is accessible in guest. */ 8406 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) 8407 run_flags |= KVM_RUN_LOAD_GUEST_DR6; 8408 } else if (unlikely(hw_breakpoint_active())) { 8409 set_debugreg(DR7_FIXED_1, 7); 8410 } 8411 8412 /* 8413 * Refresh the host DEBUGCTL snapshot after disabling IRQs, as DEBUGCTL 8414 * can be modified in IRQ context, e.g. via SMP function calls. Inform 8415 * vendor code if any host-owned bits were changed, e.g. so that the 8416 * value loaded into hardware while running the guest can be updated. 8417 */ 8418 debug_ctl = get_debugctlmsr(); 8419 if ((debug_ctl ^ vcpu->arch.host_debugctl) & kvm_x86_ops.HOST_OWNED_DEBUGCTL && 8420 !vcpu->arch.guest_state_protected) 8421 run_flags |= KVM_RUN_LOAD_DEBUGCTL; 8422 vcpu->arch.host_debugctl = debug_ctl; 8423 8424 kvm_mediated_pmu_load(vcpu); 8425 8426 guest_timing_enter_irqoff(); 8427 8428 /* 8429 * Swap PKRU with hardware breakpoints disabled to minimize the number 8430 * of flows where non-KVM code can run with guest state loaded. 8431 */ 8432 kvm_load_guest_pkru(vcpu); 8433 8434 for (;;) { 8435 /* 8436 * Assert that vCPU vs. VM APICv state is consistent. An APICv 8437 * update must kick and wait for all vCPUs before toggling the 8438 * per-VM state, and responding vCPUs must wait for the update 8439 * to complete before servicing KVM_REQ_APICV_UPDATE. 8440 */ 8441 WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) && 8442 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED)); 8443 8444 exit_fastpath = kvm_x86_call(vcpu_run)(vcpu, run_flags); 8445 if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST)) 8446 break; 8447 8448 if (kvm_lapic_enabled(vcpu)) 8449 kvm_x86_call(sync_pir_to_irr)(vcpu); 8450 8451 if (unlikely(kvm_vcpu_exit_request(vcpu))) { 8452 exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED; 8453 break; 8454 } 8455 8456 run_flags = 0; 8457 8458 /* Note, VM-Exits that go down the "slow" path are accounted below. */ 8459 ++vcpu->stat.exits; 8460 } 8461 8462 kvm_load_host_pkru(vcpu); 8463 8464 kvm_mediated_pmu_put(vcpu); 8465 8466 /* 8467 * Do this here before restoring debug registers on the host. And 8468 * since we do this before handling the vmexit, a DR access vmexit 8469 * can (a) read the correct value of the debug registers, (b) set 8470 * KVM_DEBUGREG_WONT_EXIT again. 8471 */ 8472 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) { 8473 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP); 8474 WARN_ON(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH); 8475 kvm_x86_call(sync_dirty_debug_regs)(vcpu); 8476 kvm_update_dr0123(vcpu); 8477 vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); 8478 kvm_update_dr7(vcpu); 8479 } 8480 8481 /* 8482 * If the guest has used debug registers, at least dr7 8483 * will be disabled while returning to the host. 8484 * If we don't have active breakpoints in the host, we don't 8485 * care about the messed up debug address registers. But if 8486 * we have some of them active, restore the old state. 8487 */ 8488 if (hw_breakpoint_active()) 8489 hw_breakpoint_restore(); 8490 8491 vcpu->arch.last_vmentry_cpu = vcpu->cpu; 8492 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 8493 8494 vcpu->mode = OUTSIDE_GUEST_MODE; 8495 smp_wmb(); 8496 8497 kvm_load_xfeatures(vcpu, false); 8498 8499 /* 8500 * Sync xfd before calling handle_exit_irqoff() which may 8501 * rely on the fact that guest_fpu::xfd is up-to-date (e.g. 8502 * in #NM irqoff handler). 8503 */ 8504 if (vcpu->arch.xfd_no_write_intercept) 8505 fpu_sync_guest_vmexit_xfd_state(); 8506 8507 kvm_x86_call(handle_exit_irqoff)(vcpu); 8508 8509 if (vcpu->arch.guest_fpu.xfd_err) 8510 wrmsrq(MSR_IA32_XFD_ERR, 0); 8511 8512 /* 8513 * Mark this CPU as needing a branch predictor flush before running 8514 * userspace. Must be done before enabling preemption to ensure it gets 8515 * set for the CPU that actually ran the guest, and not the CPU that it 8516 * may migrate to. 8517 */ 8518 if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER)) 8519 this_cpu_write(x86_ibpb_exit_to_user, true); 8520 8521 /* 8522 * Consume any pending interrupts, including the possible source of 8523 * VM-Exit on SVM and any ticks that occur between VM-Exit and now. 8524 * An instruction is required after local_irq_enable() to fully unblock 8525 * interrupts on processors that implement an interrupt shadow, the 8526 * stat.exits increment will do nicely. 8527 */ 8528 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); 8529 local_irq_enable(); 8530 ++vcpu->stat.exits; 8531 local_irq_disable(); 8532 kvm_after_interrupt(vcpu); 8533 8534 /* 8535 * Wait until after servicing IRQs to account guest time so that any 8536 * ticks that occurred while running the guest are properly accounted 8537 * to the guest. Waiting until IRQs are enabled degrades the accuracy 8538 * of accounting via context tracking, but the loss of accuracy is 8539 * acceptable for all known use cases. 8540 */ 8541 guest_timing_exit_irqoff(); 8542 8543 local_irq_enable(); 8544 preempt_enable(); 8545 8546 kvm_vcpu_srcu_read_lock(vcpu); 8547 8548 /* 8549 * Call this to ensure WC buffers in guest are evicted after each VM 8550 * Exit, so that the evicted WC writes can be snooped across all cpus 8551 */ 8552 smp_mb__after_srcu_read_lock(); 8553 8554 /* 8555 * Profile KVM exit RIPs: 8556 */ 8557 if (unlikely(prof_on == KVM_PROFILING && 8558 !vcpu->arch.guest_state_protected)) { 8559 unsigned long rip = kvm_rip_read(vcpu); 8560 profile_hit(KVM_PROFILING, (void *)rip); 8561 } 8562 8563 if (unlikely(vcpu->arch.tsc_always_catchup)) 8564 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 8565 8566 if (vcpu->arch.apic_attention) 8567 kvm_lapic_sync_from_vapic(vcpu); 8568 8569 r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath); 8570 return r; 8571 8572 cancel_injection: 8573 if (req_immediate_exit) 8574 kvm_make_request(KVM_REQ_EVENT, vcpu); 8575 kvm_x86_call(cancel_injection)(vcpu); 8576 if (unlikely(vcpu->arch.apic_attention)) 8577 kvm_lapic_sync_from_vapic(vcpu); 8578 out: 8579 return r; 8580 } 8581 8582 static bool kvm_vcpu_running(struct kvm_vcpu *vcpu) 8583 { 8584 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && 8585 !vcpu->arch.apf.halted); 8586 } 8587 8588 bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) 8589 { 8590 if (!list_empty_careful(&vcpu->async_pf.done)) 8591 return true; 8592 8593 if (kvm_apic_has_pending_init_or_sipi(vcpu) && 8594 kvm_apic_init_sipi_allowed(vcpu)) 8595 return true; 8596 8597 if (kvm_is_exception_pending(vcpu)) 8598 return true; 8599 8600 if (kvm_test_request(KVM_REQ_NMI, vcpu) || 8601 (vcpu->arch.nmi_pending && 8602 kvm_x86_call(nmi_allowed)(vcpu, false))) 8603 return true; 8604 8605 #ifdef CONFIG_KVM_SMM 8606 if (kvm_test_request(KVM_REQ_SMI, vcpu) || 8607 (vcpu->arch.smi_pending && 8608 kvm_x86_call(smi_allowed)(vcpu, false))) 8609 return true; 8610 #endif 8611 8612 if (kvm_test_request(KVM_REQ_PMI, vcpu)) 8613 return true; 8614 8615 if (kvm_test_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) 8616 return true; 8617 8618 if (kvm_is_interrupt_allowed(vcpu) && kvm_cpu_has_interrupt(vcpu)) 8619 return true; 8620 8621 if (kvm_hv_has_stimer_pending(vcpu)) 8622 return true; 8623 8624 if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, false)) 8625 return true; 8626 8627 if (kvm_xen_has_pending_events(vcpu)) 8628 return true; 8629 8630 return false; 8631 } 8632 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_has_events); 8633 8634 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 8635 { 8636 return kvm_vcpu_running(vcpu) || vcpu->arch.pv.pv_unhalted || 8637 kvm_vcpu_has_events(vcpu); 8638 } 8639 8640 /* Called within kvm->srcu read side. */ 8641 static inline int vcpu_block(struct kvm_vcpu *vcpu) 8642 { 8643 bool hv_timer; 8644 8645 if (!kvm_arch_vcpu_runnable(vcpu)) { 8646 /* 8647 * Switch to the software timer before halt-polling/blocking as 8648 * the guest's timer may be a break event for the vCPU, and the 8649 * hypervisor timer runs only when the CPU is in guest mode. 8650 * Switch before halt-polling so that KVM recognizes an expired 8651 * timer before blocking. 8652 */ 8653 hv_timer = kvm_lapic_hv_timer_in_use(vcpu); 8654 if (hv_timer) 8655 kvm_lapic_switch_to_sw_timer(vcpu); 8656 8657 kvm_vcpu_srcu_read_unlock(vcpu); 8658 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 8659 kvm_vcpu_halt(vcpu); 8660 else 8661 kvm_vcpu_block(vcpu); 8662 kvm_vcpu_srcu_read_lock(vcpu); 8663 8664 if (hv_timer) 8665 kvm_lapic_switch_to_hv_timer(vcpu); 8666 8667 /* 8668 * If the vCPU is not runnable, a signal or another host event 8669 * of some kind is pending; service it without changing the 8670 * vCPU's activity state. 8671 */ 8672 if (!kvm_arch_vcpu_runnable(vcpu)) 8673 return 1; 8674 } 8675 8676 /* 8677 * Evaluate nested events before exiting the halted state. This allows 8678 * the halt state to be recorded properly in the VMCS12's activity 8679 * state field (AMD does not have a similar field and a VM-Exit always 8680 * causes a spurious wakeup from HLT). 8681 */ 8682 if (is_guest_mode(vcpu)) { 8683 int r = kvm_check_nested_events(vcpu); 8684 8685 if (r < 0 && r != -EBUSY) 8686 return 0; 8687 } 8688 8689 if (kvm_apic_accept_events(vcpu) < 0) 8690 return 0; 8691 switch(vcpu->arch.mp_state) { 8692 case KVM_MP_STATE_HALTED: 8693 case KVM_MP_STATE_AP_RESET_HOLD: 8694 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 8695 fallthrough; 8696 case KVM_MP_STATE_RUNNABLE: 8697 vcpu->arch.apf.halted = false; 8698 break; 8699 case KVM_MP_STATE_INIT_RECEIVED: 8700 break; 8701 default: 8702 WARN_ON_ONCE(1); 8703 break; 8704 } 8705 return 1; 8706 } 8707 8708 /* Called within kvm->srcu read side. */ 8709 static int vcpu_run(struct kvm_vcpu *vcpu) 8710 { 8711 int r; 8712 8713 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; 8714 8715 for (;;) { 8716 /* 8717 * If another guest vCPU requests a PV TLB flush in the middle 8718 * of instruction emulation, the rest of the emulation could 8719 * use a stale page translation. Assume that any code after 8720 * this point can start executing an instruction. 8721 */ 8722 vcpu->arch.at_instruction_boundary = false; 8723 if (kvm_vcpu_running(vcpu)) { 8724 r = vcpu_enter_guest(vcpu); 8725 } else { 8726 r = vcpu_block(vcpu); 8727 } 8728 8729 if (r <= 0) 8730 break; 8731 8732 kvm_clear_request(KVM_REQ_UNBLOCK, vcpu); 8733 if (kvm_xen_has_pending_events(vcpu)) 8734 kvm_xen_inject_pending_events(vcpu); 8735 8736 if (kvm_cpu_has_pending_timer(vcpu)) 8737 kvm_inject_pending_timer_irqs(vcpu); 8738 8739 if (dm_request_for_irq_injection(vcpu) && 8740 kvm_vcpu_ready_for_interrupt_injection(vcpu)) { 8741 r = 0; 8742 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; 8743 ++vcpu->stat.request_irq_exits; 8744 break; 8745 } 8746 8747 if (__xfer_to_guest_mode_work_pending()) { 8748 kvm_vcpu_srcu_read_unlock(vcpu); 8749 r = kvm_xfer_to_guest_mode_handle_work(vcpu); 8750 kvm_vcpu_srcu_read_lock(vcpu); 8751 if (r) 8752 return r; 8753 } 8754 } 8755 8756 return r; 8757 } 8758 8759 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason) 8760 { 8761 /* 8762 * The vCPU has halted, e.g. executed HLT. Update the run state if the 8763 * local APIC is in-kernel, the run loop will detect the non-runnable 8764 * state and halt the vCPU. Exit to userspace if the local APIC is 8765 * managed by userspace, in which case userspace is responsible for 8766 * handling wake events. 8767 */ 8768 ++vcpu->stat.halt_exits; 8769 if (lapic_in_kernel(vcpu)) { 8770 if (kvm_vcpu_has_events(vcpu) || vcpu->arch.pv.pv_unhalted) 8771 state = KVM_MP_STATE_RUNNABLE; 8772 kvm_set_mp_state(vcpu, state); 8773 return 1; 8774 } else { 8775 vcpu->run->exit_reason = reason; 8776 return 0; 8777 } 8778 } 8779 8780 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu) 8781 { 8782 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT); 8783 } 8784 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_halt_noskip); 8785 8786 int kvm_emulate_halt(struct kvm_vcpu *vcpu) 8787 { 8788 int ret = kvm_skip_emulated_instruction(vcpu); 8789 /* 8790 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered 8791 * KVM_EXIT_DEBUG here. 8792 */ 8793 return kvm_emulate_halt_noskip(vcpu) && ret; 8794 } 8795 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_halt); 8796 8797 fastpath_t handle_fastpath_hlt(struct kvm_vcpu *vcpu) 8798 { 8799 if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu)) 8800 return EXIT_FASTPATH_NONE; 8801 8802 if (!kvm_emulate_halt(vcpu)) 8803 return EXIT_FASTPATH_EXIT_USERSPACE; 8804 8805 if (kvm_vcpu_running(vcpu)) 8806 return EXIT_FASTPATH_REENTER_GUEST; 8807 8808 return EXIT_FASTPATH_EXIT_HANDLED; 8809 } 8810 EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_hlt); 8811 8812 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu) 8813 { 8814 int ret = kvm_skip_emulated_instruction(vcpu); 8815 8816 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD, 8817 KVM_EXIT_AP_RESET_HOLD) && ret; 8818 } 8819 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_ap_reset_hold); 8820 8821 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu) 8822 { 8823 return kvm_vcpu_apicv_active(vcpu) && 8824 kvm_x86_call(dy_apicv_has_pending_interrupt)(vcpu); 8825 } 8826 8827 bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu) 8828 { 8829 return vcpu->arch.preempted_in_kernel; 8830 } 8831 8832 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) 8833 { 8834 if (READ_ONCE(vcpu->arch.pv.pv_unhalted)) 8835 return true; 8836 8837 if (kvm_test_request(KVM_REQ_NMI, vcpu) || 8838 #ifdef CONFIG_KVM_SMM 8839 kvm_test_request(KVM_REQ_SMI, vcpu) || 8840 #endif 8841 kvm_test_request(KVM_REQ_EVENT, vcpu)) 8842 return true; 8843 8844 return kvm_arch_dy_has_pending_interrupt(vcpu); 8845 } 8846 8847 static inline int complete_emulated_io(struct kvm_vcpu *vcpu) 8848 { 8849 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE); 8850 } 8851 8852 static int complete_emulated_pio(struct kvm_vcpu *vcpu) 8853 { 8854 if (KVM_BUG_ON(!vcpu->arch.pio.count, vcpu->kvm)) 8855 return -EIO; 8856 8857 return complete_emulated_io(vcpu); 8858 } 8859 8860 /* 8861 * Implements the following, as a state machine: 8862 * 8863 * read: 8864 * for each fragment 8865 * for each mmio piece in the fragment 8866 * write gpa, len 8867 * exit 8868 * copy data 8869 * execute insn 8870 * 8871 * write: 8872 * for each fragment 8873 * for each mmio piece in the fragment 8874 * write gpa, len 8875 * copy data 8876 * exit 8877 */ 8878 static int complete_emulated_mmio(struct kvm_vcpu *vcpu) 8879 { 8880 struct kvm_run *run = vcpu->run; 8881 struct kvm_mmio_fragment *frag; 8882 unsigned len; 8883 8884 if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm)) 8885 return -EIO; 8886 8887 /* Complete previous fragment */ 8888 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment]; 8889 len = min(8u, frag->len); 8890 if (!vcpu->mmio_is_write) 8891 memcpy(frag->data, run->mmio.data, len); 8892 8893 if (frag->len <= 8) { 8894 /* Switch to the next fragment. */ 8895 frag++; 8896 vcpu->mmio_cur_fragment++; 8897 } else { 8898 if (WARN_ON_ONCE(frag->data == &frag->val)) 8899 return -EIO; 8900 8901 /* Go forward to the next mmio piece. */ 8902 frag->data += len; 8903 frag->gpa += len; 8904 frag->len -= len; 8905 } 8906 8907 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) { 8908 vcpu->mmio_needed = 0; 8909 8910 /* FIXME: return into emulator if single-stepping. */ 8911 if (vcpu->mmio_is_write) 8912 return 1; 8913 vcpu->mmio_read_completed = 1; 8914 return complete_emulated_io(vcpu); 8915 } 8916 8917 kvm_prepare_emulated_mmio_exit(vcpu, frag); 8918 vcpu->arch.complete_userspace_io = complete_emulated_mmio; 8919 return 0; 8920 } 8921 8922 static int kvm_x86_vcpu_pre_run(struct kvm_vcpu *vcpu) 8923 { 8924 /* 8925 * Userspace may have modified vCPU state, mark nested_run_pending as 8926 * "untrusted" to avoid triggering false-positive WARNs. 8927 */ 8928 if (vcpu->arch.nested_run_pending == KVM_NESTED_RUN_PENDING) 8929 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING_UNTRUSTED; 8930 8931 /* 8932 * SIPI_RECEIVED is obsolete; KVM leaves the vCPU in Wait-For-SIPI and 8933 * tracks the pending SIPI separately. SIPI_RECEIVED is still accepted 8934 * by KVM_SET_VCPU_EVENTS for backwards compatibility, but should be 8935 * converted to INIT_RECEIVED. 8936 */ 8937 if (WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) 8938 return -EINVAL; 8939 8940 /* 8941 * Disallow running the vCPU if userspace forced it into an impossible 8942 * MP_STATE, e.g. if the vCPU is in WFS but SIPI is blocked. 8943 */ 8944 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED && 8945 !kvm_apic_init_sipi_allowed(vcpu)) 8946 return -EINVAL; 8947 8948 if (kvm_x86_call(vcpu_needs_initialization)(vcpu)) 8949 return -EINVAL; 8950 8951 if (kvm_x86_call(unhandleable_emulation_required)(vcpu)) { 8952 kvm_prepare_emulation_failure_exit(vcpu); 8953 return 0; 8954 } 8955 8956 return 1; 8957 } 8958 8959 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 8960 { 8961 struct kvm_queued_exception *ex = &vcpu->arch.exception; 8962 struct kvm_run *kvm_run = vcpu->run; 8963 u64 sync_valid_fields; 8964 int r; 8965 8966 r = kvm_mmu_post_init_vm(vcpu->kvm); 8967 if (r) 8968 return r; 8969 8970 vcpu_load(vcpu); 8971 kvm_sigset_activate(vcpu); 8972 kvm_run->flags = 0; 8973 kvm_load_guest_fpu(vcpu); 8974 8975 kvm_vcpu_srcu_read_lock(vcpu); 8976 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { 8977 if (!vcpu->wants_to_run) { 8978 r = -EINTR; 8979 goto out; 8980 } 8981 8982 /* 8983 * Don't bother switching APIC timer emulation from the 8984 * hypervisor timer to the software timer, the only way for the 8985 * APIC timer to be active is if userspace stuffed vCPU state, 8986 * i.e. put the vCPU into a nonsensical state. Only an INIT 8987 * will transition the vCPU out of UNINITIALIZED (without more 8988 * state stuffing from userspace), which will reset the local 8989 * APIC and thus cancel the timer or drop the IRQ (if the timer 8990 * already expired). 8991 */ 8992 kvm_vcpu_srcu_read_unlock(vcpu); 8993 kvm_vcpu_block(vcpu); 8994 kvm_vcpu_srcu_read_lock(vcpu); 8995 8996 if (kvm_apic_accept_events(vcpu) < 0) { 8997 r = 0; 8998 goto out; 8999 } 9000 r = -EAGAIN; 9001 if (signal_pending(current)) { 9002 r = -EINTR; 9003 kvm_run->exit_reason = KVM_EXIT_INTR; 9004 ++vcpu->stat.signal_exits; 9005 } 9006 goto out; 9007 } 9008 9009 sync_valid_fields = kvm_sync_valid_fields(vcpu->kvm); 9010 if ((kvm_run->kvm_valid_regs & ~sync_valid_fields) || 9011 (kvm_run->kvm_dirty_regs & ~sync_valid_fields)) { 9012 r = -EINVAL; 9013 goto out; 9014 } 9015 9016 if (kvm_run->kvm_dirty_regs) { 9017 r = sync_regs(vcpu); 9018 if (r != 0) 9019 goto out; 9020 } 9021 9022 /* re-sync apic's tpr */ 9023 if (!lapic_in_kernel(vcpu)) { 9024 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) { 9025 r = -EINVAL; 9026 goto out; 9027 } 9028 } 9029 9030 /* 9031 * If userspace set a pending exception and L2 is active, convert it to 9032 * a pending VM-Exit if L1 wants to intercept the exception. 9033 */ 9034 if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) && 9035 kvm_nested_call(is_exception_vmexit)(vcpu, ex->vector, ex->error_code)) { 9036 kvm_queue_exception_vmexit(vcpu, ex->vector, 9037 ex->has_error_code, ex->error_code, 9038 ex->has_payload, ex->payload); 9039 ex->injected = false; 9040 ex->pending = false; 9041 } 9042 vcpu->arch.exception_from_userspace = false; 9043 9044 if (unlikely(vcpu->arch.complete_userspace_io)) { 9045 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io; 9046 vcpu->arch.complete_userspace_io = NULL; 9047 r = cui(vcpu); 9048 if (r <= 0) 9049 goto out; 9050 } else { 9051 WARN_ON_ONCE(vcpu->arch.pio.count); 9052 WARN_ON_ONCE(vcpu->mmio_needed); 9053 } 9054 9055 if (!vcpu->wants_to_run) { 9056 r = -EINTR; 9057 goto out; 9058 } 9059 9060 r = kvm_x86_vcpu_pre_run(vcpu); 9061 if (r <= 0) 9062 goto out; 9063 9064 r = vcpu_run(vcpu); 9065 9066 out: 9067 kvm_put_guest_fpu(vcpu); 9068 if (kvm_run->kvm_valid_regs && likely(!vcpu->arch.guest_state_protected)) 9069 store_regs(vcpu); 9070 post_kvm_run_save(vcpu); 9071 kvm_vcpu_srcu_read_unlock(vcpu); 9072 9073 kvm_sigset_deactivate(vcpu); 9074 vcpu_put(vcpu); 9075 return r; 9076 } 9077 9078 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 9079 struct kvm_mp_state *mp_state) 9080 { 9081 int r; 9082 9083 vcpu_load(vcpu); 9084 kvm_vcpu_srcu_read_lock(vcpu); 9085 9086 r = kvm_apic_accept_events(vcpu); 9087 if (r < 0) 9088 goto out; 9089 r = 0; 9090 9091 if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED || 9092 vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) && 9093 vcpu->arch.pv.pv_unhalted) 9094 mp_state->mp_state = KVM_MP_STATE_RUNNABLE; 9095 else 9096 mp_state->mp_state = vcpu->arch.mp_state; 9097 9098 out: 9099 kvm_vcpu_srcu_read_unlock(vcpu); 9100 vcpu_put(vcpu); 9101 return r; 9102 } 9103 9104 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 9105 struct kvm_mp_state *mp_state) 9106 { 9107 int ret = -EINVAL; 9108 9109 vcpu_load(vcpu); 9110 9111 switch (mp_state->mp_state) { 9112 case KVM_MP_STATE_UNINITIALIZED: 9113 case KVM_MP_STATE_HALTED: 9114 case KVM_MP_STATE_AP_RESET_HOLD: 9115 case KVM_MP_STATE_INIT_RECEIVED: 9116 case KVM_MP_STATE_SIPI_RECEIVED: 9117 if (!lapic_in_kernel(vcpu)) 9118 goto out; 9119 break; 9120 9121 case KVM_MP_STATE_RUNNABLE: 9122 break; 9123 9124 default: 9125 goto out; 9126 } 9127 9128 /* 9129 * SIPI_RECEIVED is obsolete and no longer used internally; KVM instead 9130 * leaves the vCPU in INIT_RECIEVED (Wait-For-SIPI) and pends the SIPI. 9131 * Translate SIPI_RECEIVED as appropriate for backwards compatibility. 9132 */ 9133 if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) { 9134 mp_state->mp_state = KVM_MP_STATE_INIT_RECEIVED; 9135 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events); 9136 } 9137 9138 kvm_set_mp_state(vcpu, mp_state->mp_state); 9139 9140 /* 9141 * Force the vCPU out of any hardware-tracked halted state, e.g. VMX's 9142 * GUEST_ACTIVITY_STATE=HLT, when userspace puts the vCPU into a state 9143 * other than HALTED. The hardware state is sticky across VM-Exit and 9144 * VM-Enter and is not touched by any other ioctl, so a vCPU that halted 9145 * with HLT-exiting disabled stays wedged even after userspace rewrites 9146 * its registers, e.g. when a VMM emulates a machine reset. 9147 */ 9148 if (kvm_hlt_in_guest(vcpu->kvm) && 9149 mp_state->mp_state != KVM_MP_STATE_HALTED) 9150 kvm_x86_call(clear_hlt)(vcpu); 9151 9152 kvm_make_request(KVM_REQ_EVENT, vcpu); 9153 9154 ret = 0; 9155 out: 9156 vcpu_put(vcpu); 9157 return ret; 9158 } 9159 9160 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, 9161 int reason, bool has_error_code, u32 error_code) 9162 { 9163 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 9164 int ret; 9165 9166 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_CET)) { 9167 u64 u_cet, s_cet; 9168 9169 /* 9170 * Check both User and Supervisor on task switches as inter- 9171 * privilege level task switches are impacted by CET at both 9172 * the current privilege level and the new privilege level, and 9173 * that information is not known at this time. The expectation 9174 * is that the guest won't require emulation of task switches 9175 * while using IBT or Shadow Stacks. 9176 */ 9177 if (__kvm_emulate_msr_read(vcpu, MSR_IA32_U_CET, &u_cet) || 9178 __kvm_emulate_msr_read(vcpu, MSR_IA32_S_CET, &s_cet)) 9179 goto unhandled_task_switch; 9180 9181 if ((u_cet | s_cet) & (CET_ENDBR_EN | CET_SHSTK_EN)) 9182 goto unhandled_task_switch; 9183 } 9184 9185 init_emulate_ctxt(vcpu); 9186 9187 ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason, 9188 has_error_code, error_code); 9189 9190 /* 9191 * Report an error userspace if MMIO is needed, as KVM doesn't support 9192 * MMIO during a task switch (or any other complex operation). 9193 */ 9194 if (ret || vcpu->mmio_needed) 9195 goto unhandled_task_switch; 9196 9197 kvm_rip_write(vcpu, ctxt->eip); 9198 kvm_set_rflags(vcpu, ctxt->eflags); 9199 return 1; 9200 9201 unhandled_task_switch: 9202 vcpu->mmio_needed = false; 9203 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 9204 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 9205 vcpu->run->internal.ndata = 0; 9206 return 0; 9207 } 9208 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_task_switch); 9209 9210 static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm) 9211 { 9212 bool set = false; 9213 struct kvm_vcpu *vcpu; 9214 unsigned long i; 9215 9216 if (!enable_apicv) 9217 return; 9218 9219 down_write(&kvm->arch.apicv_update_lock); 9220 9221 kvm_for_each_vcpu(i, vcpu, kvm) { 9222 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) { 9223 set = true; 9224 break; 9225 } 9226 } 9227 __kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_BLOCKIRQ, set); 9228 up_write(&kvm->arch.apicv_update_lock); 9229 } 9230 9231 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 9232 struct kvm_guest_debug *dbg) 9233 { 9234 unsigned long rflags; 9235 int i, r; 9236 9237 if (vcpu->arch.guest_state_protected) 9238 return -EINVAL; 9239 9240 vcpu_load(vcpu); 9241 9242 if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) { 9243 r = -EBUSY; 9244 if (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected) 9245 goto out; 9246 if (dbg->control & KVM_GUESTDBG_INJECT_DB) 9247 kvm_queue_exception(vcpu, DB_VECTOR); 9248 else 9249 kvm_queue_exception(vcpu, BP_VECTOR); 9250 } 9251 9252 /* 9253 * Read rflags as long as potentially injected trace flags are still 9254 * filtered out. 9255 */ 9256 rflags = kvm_get_rflags(vcpu); 9257 9258 vcpu->guest_debug = dbg->control; 9259 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE)) 9260 vcpu->guest_debug = 0; 9261 9262 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 9263 for (i = 0; i < KVM_NR_DB_REGS; ++i) 9264 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; 9265 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7]; 9266 } else { 9267 for (i = 0; i < KVM_NR_DB_REGS; i++) 9268 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 9269 } 9270 kvm_update_dr7(vcpu); 9271 9272 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 9273 vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu); 9274 9275 /* 9276 * Trigger an rflags update that will inject or remove the trace 9277 * flags. 9278 */ 9279 kvm_set_rflags(vcpu, rflags); 9280 9281 kvm_x86_call(update_exception_bitmap)(vcpu); 9282 9283 kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm); 9284 9285 r = 0; 9286 9287 out: 9288 vcpu_put(vcpu); 9289 return r; 9290 } 9291 9292 /* 9293 * Translate a guest virtual address to a guest physical address. 9294 */ 9295 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 9296 struct kvm_translation *tr) 9297 { 9298 unsigned long vaddr = tr->linear_address; 9299 gpa_t gpa; 9300 int idx; 9301 9302 vcpu_load(vcpu); 9303 9304 idx = srcu_read_lock(&vcpu->kvm->srcu); 9305 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL); 9306 srcu_read_unlock(&vcpu->kvm->srcu, idx); 9307 tr->physical_address = gpa; 9308 tr->valid = gpa != INVALID_GPA; 9309 tr->writeable = 1; 9310 tr->usermode = 0; 9311 9312 vcpu_put(vcpu); 9313 return 0; 9314 } 9315 9316 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 9317 { 9318 struct fxregs_state *fxsave; 9319 9320 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 9321 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 9322 9323 vcpu_load(vcpu); 9324 9325 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; 9326 memcpy(fpu->fpr, fxsave->st_space, 128); 9327 fpu->fcw = fxsave->cwd; 9328 fpu->fsw = fxsave->swd; 9329 fpu->ftwx = fxsave->twd; 9330 fpu->last_opcode = fxsave->fop; 9331 fpu->last_ip = fxsave->rip; 9332 fpu->last_dp = fxsave->rdp; 9333 memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space)); 9334 9335 vcpu_put(vcpu); 9336 return 0; 9337 } 9338 9339 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 9340 { 9341 struct fxregs_state *fxsave; 9342 9343 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 9344 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 9345 9346 vcpu_load(vcpu); 9347 9348 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; 9349 9350 memcpy(fxsave->st_space, fpu->fpr, 128); 9351 fxsave->cwd = fpu->fcw; 9352 fxsave->swd = fpu->fsw; 9353 fxsave->twd = fpu->ftwx; 9354 fxsave->fop = fpu->last_opcode; 9355 fxsave->rip = fpu->last_ip; 9356 fxsave->rdp = fpu->last_dp; 9357 memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space)); 9358 9359 vcpu_put(vcpu); 9360 return 0; 9361 } 9362 9363 static void store_regs(struct kvm_vcpu *vcpu) 9364 { 9365 kvm_run_sync_regs_to_user(vcpu); 9366 9367 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS) 9368 kvm_vcpu_ioctl_x86_get_vcpu_events( 9369 vcpu, &vcpu->run->s.regs.events); 9370 } 9371 9372 static int sync_regs(struct kvm_vcpu *vcpu) 9373 { 9374 if (kvm_run_sync_regs_from_user(vcpu)) 9375 return -EINVAL; 9376 9377 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) { 9378 struct kvm_vcpu_events events = vcpu->run->s.regs.events; 9379 9380 if (kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events)) 9381 return -EINVAL; 9382 9383 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS; 9384 } 9385 9386 return 0; 9387 } 9388 9389 #define PERF_MEDIATED_PMU_MSG \ 9390 "Failed to enable mediated vPMU, try disabling system wide perf events and nmi_watchdog.\n" 9391 9392 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 9393 { 9394 int r; 9395 9396 if (kvm_check_tsc_unstable() && kvm->created_vcpus) 9397 pr_warn_once("SMP vm created on host with unstable TSC; " 9398 "guest TSC will not be reliable\n"); 9399 9400 if (!kvm->arch.max_vcpu_ids) 9401 kvm->arch.max_vcpu_ids = KVM_MAX_VCPU_IDS; 9402 9403 if (id >= kvm->arch.max_vcpu_ids) 9404 return -EINVAL; 9405 9406 /* 9407 * Note, any actions done by .vcpu_create() must be idempotent with 9408 * respect to creating multiple vCPUs, and therefore are not undone if 9409 * creating a vCPU fails (including failure during pre-create). 9410 */ 9411 r = kvm_x86_call(vcpu_precreate)(kvm); 9412 if (r) 9413 return r; 9414 9415 if (enable_mediated_pmu && kvm->arch.enable_pmu && 9416 !kvm->arch.created_mediated_pmu) { 9417 if (irqchip_in_kernel(kvm)) { 9418 r = perf_create_mediated_pmu(); 9419 if (r) { 9420 pr_warn_ratelimited(PERF_MEDIATED_PMU_MSG); 9421 return r; 9422 } 9423 kvm->arch.created_mediated_pmu = true; 9424 } else { 9425 kvm->arch.enable_pmu = false; 9426 } 9427 } 9428 return 0; 9429 } 9430 9431 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 9432 { 9433 struct page *page; 9434 int r; 9435 9436 vcpu->arch.last_vmentry_cpu = -1; 9437 bitmap_fill(vcpu->arch.regs_avail, NR_VCPU_TOTAL_REGS); 9438 bitmap_fill(vcpu->arch.regs_dirty, NR_VCPU_TOTAL_REGS); 9439 9440 kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm); 9441 9442 if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) 9443 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 9444 else 9445 kvm_set_mp_state(vcpu, KVM_MP_STATE_UNINITIALIZED); 9446 9447 r = kvm_mmu_create(vcpu); 9448 if (r < 0) 9449 return r; 9450 9451 r = kvm_create_lapic(vcpu); 9452 if (r < 0) 9453 goto fail_mmu_destroy; 9454 9455 r = -ENOMEM; 9456 9457 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 9458 if (!page) 9459 goto fail_free_lapic; 9460 vcpu->arch.pio_data = page_address(page); 9461 9462 vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64), 9463 GFP_KERNEL_ACCOUNT); 9464 vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64), 9465 GFP_KERNEL_ACCOUNT); 9466 if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks) 9467 goto fail_free_mce_banks; 9468 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS; 9469 9470 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, 9471 GFP_KERNEL_ACCOUNT)) 9472 goto fail_free_mce_banks; 9473 9474 if (!alloc_emulate_ctxt(vcpu)) 9475 goto free_wbinvd_dirty_mask; 9476 9477 if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) { 9478 pr_err("failed to allocate vcpu's fpu\n"); 9479 goto free_emulate_ctxt; 9480 } 9481 9482 kvm_async_pf_hash_reset(vcpu); 9483 9484 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) { 9485 vcpu->arch.arch_capabilities = kvm_get_arch_capabilities(); 9486 vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT; 9487 vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap; 9488 } 9489 kvm_pmu_init(vcpu); 9490 9491 vcpu->arch.pending_external_vector = -1; 9492 vcpu->arch.preempted_in_kernel = false; 9493 9494 #if IS_ENABLED(CONFIG_HYPERV) 9495 vcpu->arch.hv_root_tdp = INVALID_PAGE; 9496 #endif 9497 9498 r = kvm_x86_call(vcpu_create)(vcpu); 9499 if (r) 9500 goto free_guest_fpu; 9501 9502 kvm_xen_init_vcpu(vcpu); 9503 vcpu_load(vcpu); 9504 kvm_vcpu_after_set_cpuid(vcpu); 9505 kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz); 9506 kvm_vcpu_reset(vcpu, false); 9507 kvm_init_mmu(vcpu); 9508 vcpu_put(vcpu); 9509 return 0; 9510 9511 free_guest_fpu: 9512 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); 9513 free_emulate_ctxt: 9514 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); 9515 free_wbinvd_dirty_mask: 9516 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); 9517 fail_free_mce_banks: 9518 kfree(vcpu->arch.mce_banks); 9519 kfree(vcpu->arch.mci_ctl2_banks); 9520 free_page((unsigned long)vcpu->arch.pio_data); 9521 fail_free_lapic: 9522 kvm_free_lapic(vcpu); 9523 fail_mmu_destroy: 9524 kvm_mmu_destroy(vcpu); 9525 return r; 9526 } 9527 9528 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 9529 { 9530 if (mutex_lock_killable(&vcpu->mutex)) 9531 return; 9532 vcpu_load(vcpu); 9533 kvm_synchronize_tsc(vcpu, NULL); 9534 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) 9535 kvm_update_masterclock(vcpu->kvm); 9536 vcpu_put(vcpu); 9537 9538 /* poll control enabled by default */ 9539 vcpu->arch.msr_kvm_poll_control = 1; 9540 9541 mutex_unlock(&vcpu->mutex); 9542 } 9543 9544 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 9545 { 9546 int idx, cpu; 9547 9548 kvm_clear_async_pf_completion_queue(vcpu); 9549 kvm_mmu_unload(vcpu); 9550 9551 kvmclock_reset(vcpu); 9552 9553 for_each_possible_cpu(cpu) 9554 cmpxchg(per_cpu_ptr(&last_vcpu, cpu), vcpu, NULL); 9555 9556 kvm_x86_call(vcpu_free)(vcpu); 9557 9558 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); 9559 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); 9560 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); 9561 9562 kvm_xen_destroy_vcpu(vcpu); 9563 kvm_hv_vcpu_uninit(vcpu); 9564 kvm_pmu_destroy(vcpu); 9565 kfree(vcpu->arch.mce_banks); 9566 kfree(vcpu->arch.mci_ctl2_banks); 9567 kvm_free_lapic(vcpu); 9568 idx = srcu_read_lock(&vcpu->kvm->srcu); 9569 kvm_mmu_destroy(vcpu); 9570 srcu_read_unlock(&vcpu->kvm->srcu, idx); 9571 free_page((unsigned long)vcpu->arch.pio_data); 9572 kvfree(vcpu->arch.cpuid_entries); 9573 } 9574 9575 static void kvm_xstate_reset(struct kvm_vcpu *vcpu, bool init_event) 9576 { 9577 struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate; 9578 u64 xfeatures_mask; 9579 bool fpu_in_use; 9580 int i; 9581 9582 /* 9583 * Guest FPU state is zero allocated and so doesn't need to be manually 9584 * cleared on RESET, i.e. during vCPU creation. 9585 */ 9586 if (!init_event || !fpstate) 9587 return; 9588 9589 /* 9590 * On INIT, only select XSTATE components are zeroed, most components 9591 * are unchanged. Currently, the only components that are zeroed and 9592 * supported by KVM are MPX and CET related. 9593 */ 9594 xfeatures_mask = (kvm_caps.supported_xcr0 | kvm_caps.supported_xss) & 9595 (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR | 9596 XFEATURE_MASK_CET_ALL); 9597 if (!xfeatures_mask) 9598 return; 9599 9600 BUILD_BUG_ON(sizeof(xfeatures_mask) * BITS_PER_BYTE <= XFEATURE_MAX); 9601 9602 /* 9603 * Unload guest FPU state (if necessary) before zeroing XSTATE fields 9604 * as the kernel can only modify the state when its resident in memory, 9605 * i.e. when it's not loaded into hardware. 9606 * 9607 * WARN if the vCPU's desire to run, i.e. whether or not its in KVM_RUN, 9608 * doesn't match the loaded/in-use state of the FPU, as KVM_RUN is the 9609 * only path that can trigger INIT emulation _and_ loads FPU state, and 9610 * KVM_RUN should _always_ load FPU state. 9611 */ 9612 WARN_ON_ONCE(vcpu->wants_to_run != fpstate->in_use); 9613 fpu_in_use = fpstate->in_use; 9614 if (fpu_in_use) 9615 kvm_put_guest_fpu(vcpu); 9616 for_each_set_bit(i, (unsigned long *)&xfeatures_mask, XFEATURE_MAX) 9617 fpstate_clear_xstate_component(fpstate, i); 9618 if (fpu_in_use) 9619 kvm_load_guest_fpu(vcpu); 9620 } 9621 9622 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 9623 { 9624 struct kvm_cpuid_entry2 *cpuid_0x1; 9625 unsigned long old_cr0 = kvm_read_cr0(vcpu); 9626 unsigned long new_cr0; 9627 9628 /* 9629 * Several of the "set" flows, e.g. ->set_cr0(), read other registers 9630 * to handle side effects. RESET emulation hits those flows and relies 9631 * on emulated/virtualized registers, including those that are loaded 9632 * into hardware, to be zeroed at vCPU creation. Use CRs as a sentinel 9633 * to detect improper or missing initialization. 9634 */ 9635 WARN_ON_ONCE(!init_event && 9636 (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu))); 9637 9638 /* 9639 * SVM doesn't unconditionally VM-Exit on INIT and SHUTDOWN, thus it's 9640 * possible to INIT the vCPU while L2 is active. Force the vCPU back 9641 * into L1 as EFER.SVME is cleared on INIT (along with all other EFER 9642 * bits), i.e. virtualization is disabled. 9643 */ 9644 if (is_guest_mode(vcpu)) 9645 kvm_leave_nested(vcpu); 9646 9647 kvm_lapic_reset(vcpu, init_event); 9648 9649 WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu)); 9650 vcpu->arch.hflags = 0; 9651 9652 vcpu->arch.smi_pending = 0; 9653 vcpu->arch.smi_count = 0; 9654 atomic_set(&vcpu->arch.nmi_queued, 0); 9655 vcpu->arch.nmi_pending = 0; 9656 vcpu->arch.nmi_injected = false; 9657 kvm_clear_interrupt_queue(vcpu); 9658 kvm_clear_exception_queue(vcpu); 9659 9660 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); 9661 kvm_update_dr0123(vcpu); 9662 vcpu->arch.dr6 = DR6_ACTIVE_LOW; 9663 vcpu->arch.dr7 = DR7_FIXED_1; 9664 kvm_update_dr7(vcpu); 9665 9666 vcpu->arch.cr2 = 0; 9667 9668 kvm_make_request(KVM_REQ_EVENT, vcpu); 9669 vcpu->arch.apf.msr_en_val = 0; 9670 vcpu->arch.apf.msr_int_val = 0; 9671 vcpu->arch.st.msr_val = 0; 9672 9673 kvmclock_reset(vcpu); 9674 9675 kvm_clear_async_pf_completion_queue(vcpu); 9676 kvm_async_pf_hash_reset(vcpu); 9677 vcpu->arch.apf.halted = false; 9678 9679 kvm_xstate_reset(vcpu, init_event); 9680 9681 if (!init_event) { 9682 vcpu->arch.smbase = 0x30000; 9683 9684 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT; 9685 9686 vcpu->arch.msr_misc_features_enables = 0; 9687 vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL | 9688 MSR_IA32_MISC_ENABLE_BTS_UNAVAIL; 9689 9690 __kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP); 9691 kvm_msr_write(vcpu, MSR_IA32_XSS, 0); 9692 } 9693 9694 /* All GPRs except RDX (handled below) are zeroed on RESET/INIT. */ 9695 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs)); 9696 kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP); 9697 9698 /* 9699 * Fall back to KVM's default Family/Model/Stepping of 0x600 (P6/Athlon) 9700 * if no CPUID match is found. Note, it's impossible to get a match at 9701 * RESET since KVM emulates RESET before exposing the vCPU to userspace, 9702 * i.e. it's impossible for kvm_find_cpuid_entry() to find a valid entry 9703 * on RESET. But, go through the motions in case that's ever remedied. 9704 */ 9705 cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1); 9706 kvm_edx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600); 9707 9708 kvm_x86_call(vcpu_reset)(vcpu, init_event); 9709 9710 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED); 9711 kvm_rip_write(vcpu, 0xfff0); 9712 9713 vcpu->arch.cr3 = 0; 9714 kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); 9715 9716 /* 9717 * CR0.CD/NW are set on RESET, preserved on INIT. Note, some versions 9718 * of Intel's SDM list CD/NW as being set on INIT, but they contradict 9719 * (or qualify) that with a footnote stating that CD/NW are preserved. 9720 */ 9721 new_cr0 = X86_CR0_ET; 9722 if (init_event) 9723 new_cr0 |= (old_cr0 & (X86_CR0_NW | X86_CR0_CD)); 9724 else 9725 new_cr0 |= X86_CR0_NW | X86_CR0_CD; 9726 9727 kvm_x86_call(set_cr0)(vcpu, new_cr0); 9728 kvm_x86_call(set_cr4)(vcpu, 0); 9729 kvm_x86_call(set_efer)(vcpu, 0); 9730 kvm_x86_call(update_exception_bitmap)(vcpu); 9731 9732 /* 9733 * On the standard CR0/CR4/EFER modification paths, there are several 9734 * complex conditions determining whether the MMU has to be reset and/or 9735 * which PCIDs have to be flushed. However, CR0.WP and the paging-related 9736 * bits in CR4 and EFER are irrelevant if CR0.PG was '0'; and a reset+flush 9737 * is needed anyway if CR0.PG was '1' (which can only happen for INIT, as 9738 * CR0 will be '0' prior to RESET). So we only need to check CR0.PG here. 9739 */ 9740 if (old_cr0 & X86_CR0_PG) { 9741 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 9742 kvm_mmu_reset_context(vcpu); 9743 } 9744 9745 /* 9746 * Intel's SDM states that all TLB entries are flushed on INIT. AMD's 9747 * APM states the TLBs are untouched by INIT, but it also states that 9748 * the TLBs are flushed on "External initialization of the processor." 9749 * Flush the guest TLB regardless of vendor, there is no meaningful 9750 * benefit in relying on the guest to flush the TLB immediately after 9751 * INIT. A spurious TLB flush is benign and likely negligible from a 9752 * performance perspective. 9753 */ 9754 if (init_event) 9755 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 9756 } 9757 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_reset); 9758 9759 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 9760 { 9761 struct kvm_segment cs; 9762 9763 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS); 9764 cs.selector = vector << 8; 9765 cs.base = vector << 12; 9766 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS); 9767 kvm_rip_write(vcpu, 0); 9768 } 9769 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_deliver_sipi_vector); 9770 9771 void kvm_arch_enable_virtualization(void) 9772 { 9773 x86_virt_register_emergency_callback(kvm_x86_ops.emergency_disable_virtualization_cpu); 9774 } 9775 9776 void kvm_arch_disable_virtualization(void) 9777 { 9778 x86_virt_unregister_emergency_callback(kvm_x86_ops.emergency_disable_virtualization_cpu); 9779 } 9780 9781 int kvm_arch_enable_virtualization_cpu(void) 9782 { 9783 struct kvm *kvm; 9784 struct kvm_vcpu *vcpu; 9785 unsigned long i; 9786 int ret; 9787 u64 local_tsc; 9788 u64 max_tsc = 0; 9789 bool stable, backwards_tsc = false; 9790 9791 kvm_user_return_msr_cpu_online(); 9792 9793 ret = kvm_x86_check_processor_compatibility(); 9794 if (ret) 9795 return ret; 9796 9797 ret = kvm_x86_call(enable_virtualization_cpu)(); 9798 if (ret != 0) 9799 return ret; 9800 9801 local_tsc = rdtsc(); 9802 stable = !kvm_check_tsc_unstable(); 9803 list_for_each_entry(kvm, &vm_list, vm_list) { 9804 kvm_for_each_vcpu(i, vcpu, kvm) { 9805 if (!stable && vcpu->cpu == smp_processor_id()) 9806 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 9807 if (stable && vcpu->arch.last_host_tsc > local_tsc) { 9808 backwards_tsc = true; 9809 if (vcpu->arch.last_host_tsc > max_tsc) 9810 max_tsc = vcpu->arch.last_host_tsc; 9811 } 9812 } 9813 } 9814 9815 /* 9816 * Sometimes, even reliable TSCs go backwards. This happens on 9817 * platforms that reset TSC during suspend or hibernate actions, but 9818 * maintain synchronization. We must compensate. Fortunately, we can 9819 * detect that condition here, which happens early in CPU bringup, 9820 * before any KVM threads can be running. Unfortunately, we can't 9821 * bring the TSCs fully up to date with real time, as we aren't yet far 9822 * enough into CPU bringup that we know how much real time has actually 9823 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot 9824 * variables that haven't been updated yet. 9825 * 9826 * So we simply find the maximum observed TSC above, then record the 9827 * adjustment to TSC in each VCPU. When the VCPU later gets loaded, 9828 * the adjustment will be applied. Note that we accumulate 9829 * adjustments, in case multiple suspend cycles happen before some VCPU 9830 * gets a chance to run again. In the event that no KVM threads get a 9831 * chance to run, we will miss the entire elapsed period, as we'll have 9832 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may 9833 * loose cycle time. This isn't too big a deal, since the loss will be 9834 * uniform across all VCPUs (not to mention the scenario is extremely 9835 * unlikely). It is possible that a second hibernate recovery happens 9836 * much faster than a first, causing the observed TSC here to be 9837 * smaller; this would require additional padding adjustment, which is 9838 * why we set last_host_tsc to the local tsc observed here. 9839 * 9840 * N.B. - this code below runs only on platforms with reliable TSC, 9841 * as that is the only way backwards_tsc is set above. Also note 9842 * that this runs for ALL vcpus, which is not a bug; all VCPUs should 9843 * have the same delta_cyc adjustment applied if backwards_tsc 9844 * is detected. Note further, this adjustment is only done once, 9845 * as we reset last_host_tsc on all VCPUs to stop this from being 9846 * called multiple times (one for each physical CPU bringup). 9847 * 9848 * Platforms with unreliable TSCs don't have to deal with this, they 9849 * will be compensated by the logic in vcpu_load, which sets the TSC to 9850 * catchup mode. This will catchup all VCPUs to real time, but cannot 9851 * guarantee that they stay in perfect synchronization. 9852 */ 9853 if (backwards_tsc) { 9854 u64 delta_cyc = max_tsc - local_tsc; 9855 list_for_each_entry(kvm, &vm_list, vm_list) { 9856 kvm->arch.backwards_tsc_observed = true; 9857 kvm_for_each_vcpu(i, vcpu, kvm) { 9858 vcpu->arch.tsc_offset_adjustment += delta_cyc; 9859 vcpu->arch.last_host_tsc = local_tsc; 9860 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 9861 } 9862 9863 /* 9864 * We have to disable TSC offset matching.. if you were 9865 * booting a VM while issuing an S4 host suspend.... 9866 * you may have some problem. Solving this issue is 9867 * left as an exercise to the reader. 9868 */ 9869 kvm->arch.last_tsc_nsec = 0; 9870 kvm->arch.last_tsc_write = 0; 9871 } 9872 9873 } 9874 return 0; 9875 } 9876 9877 void kvm_arch_shutdown(void) 9878 { 9879 /* 9880 * Set virt_rebooting to indicate that KVM has asynchronously disabled 9881 * hardware virtualization, i.e. that errors and/or exceptions on SVM 9882 * and VMX instructions are expected and should be ignored. 9883 */ 9884 virt_rebooting = true; 9885 9886 /* 9887 * Ensure virt_rebooting is visible before IPIs are sent to other CPUs 9888 * to disable virtualization. Effectively pairs with the reception of 9889 * the IPI (virt_rebooting is read in task/exception context, but only 9890 * _needs_ to be read as %true after the IPI function callback disables 9891 * virtualization). 9892 */ 9893 smp_wmb(); 9894 } 9895 9896 void kvm_arch_disable_virtualization_cpu(void) 9897 { 9898 kvm_x86_call(disable_virtualization_cpu)(); 9899 9900 /* 9901 * Leave the user-return notifiers as-is when disabling virtualization 9902 * for reboot, i.e. when disabling via IPI function call, and instead 9903 * pin kvm.ko (if it's a module) to defend against use-after-free (in 9904 * the *very* unlikely scenario module unload is racing with reboot). 9905 * On a forced reboot, tasks aren't frozen before shutdown, and so KVM 9906 * could be actively modifying user-return MSR state when the IPI to 9907 * disable virtualization arrives. Handle the extreme edge case here 9908 * instead of trying to account for it in the normal flows. 9909 */ 9910 if (in_task() || WARN_ON_ONCE(!virt_rebooting)) 9911 drop_user_return_notifiers(); 9912 else 9913 __module_get(THIS_MODULE); 9914 } 9915 9916 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu) 9917 { 9918 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id; 9919 } 9920 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_is_reset_bsp); 9921 9922 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) 9923 { 9924 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0; 9925 } 9926 9927 void kvm_arch_free_vm(struct kvm *kvm) 9928 { 9929 #if IS_ENABLED(CONFIG_HYPERV) 9930 kfree(kvm->arch.hv_pa_pg); 9931 #endif 9932 __kvm_arch_free_vm(kvm); 9933 } 9934 9935 9936 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 9937 { 9938 int ret; 9939 unsigned long flags; 9940 9941 if (!kvm_is_vm_type_supported(type)) 9942 return -EINVAL; 9943 9944 kvm->arch.vm_type = type; 9945 kvm->arch.has_private_mem = 9946 (type == KVM_X86_SW_PROTECTED_VM); 9947 /* Decided by the vendor code for other VM types. */ 9948 kvm->arch.pre_fault_allowed = 9949 type == KVM_X86_DEFAULT_VM || type == KVM_X86_SW_PROTECTED_VM; 9950 kvm->arch.disabled_quirks = kvm_caps.inapplicable_quirks & kvm_caps.supported_quirks; 9951 9952 ret = kvm_page_track_init(kvm); 9953 if (ret) 9954 goto out; 9955 9956 ret = kvm_mmu_init_vm(kvm); 9957 if (ret) 9958 goto out_cleanup_page_track; 9959 9960 ret = kvm_x86_call(vm_init)(kvm); 9961 if (ret) 9962 goto out_uninit_mmu; 9963 9964 atomic_set(&kvm->arch.noncoherent_dma_count, 0); 9965 9966 raw_spin_lock_init(&kvm->arch.tsc_write_lock); 9967 mutex_init(&kvm->arch.apic_map_lock); 9968 seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock); 9969 ratelimit_state_init(&kvm->arch.kvmclock_update_rs, HZ, 10); 9970 ratelimit_set_flags(&kvm->arch.kvmclock_update_rs, RATELIMIT_MSG_ON_RELEASE); 9971 kvm->arch.kvmclock_offset = -get_kvmclock_base_ns(); 9972 9973 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 9974 pvclock_update_vm_gtod_copy(kvm); 9975 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 9976 9977 kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz; 9978 kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT; 9979 kvm->arch.guest_can_read_msr_platform_info = true; 9980 kvm->arch.enable_pmu = enable_pmu && !kvm->arch.has_protected_pmu; 9981 9982 #if IS_ENABLED(CONFIG_HYPERV) 9983 spin_lock_init(&kvm->arch.hv_root_tdp_lock); 9984 kvm->arch.hv_root_tdp = INVALID_PAGE; 9985 #endif 9986 9987 kvm_apicv_init(kvm); 9988 kvm_hv_init_vm(kvm); 9989 kvm_xen_init_vm(kvm); 9990 9991 if (ignore_msrs && !report_ignored_msrs) { 9992 pr_warn_once("Running KVM with ignore_msrs=1 and report_ignored_msrs=0 is not a\n" 9993 "a supported configuration. Lying to the guest about the existence of MSRs\n" 9994 "may cause the guest operating system to hang or produce errors. If a guest\n" 9995 "does not run without ignore_msrs=1, please report it to kvm@vger.kernel.org.\n"); 9996 } 9997 9998 once_init(&kvm->arch.nx_once); 9999 return 0; 10000 10001 out_uninit_mmu: 10002 kvm_mmu_uninit_vm(kvm); 10003 out_cleanup_page_track: 10004 kvm_page_track_cleanup(kvm); 10005 out: 10006 return ret; 10007 } 10008 10009 /** 10010 * __x86_set_memory_region: Setup KVM internal memory slot 10011 * 10012 * @kvm: the kvm pointer to the VM. 10013 * @id: the slot ID to setup. 10014 * @gpa: the GPA to install the slot (unused when @size == 0). 10015 * @size: the size of the slot. Set to zero to uninstall a slot. 10016 * 10017 * This function helps to setup a KVM internal memory slot. Specify 10018 * @size > 0 to install a new slot, while @size == 0 to uninstall a 10019 * slot. The return code can be one of the following: 10020 * 10021 * HVA: on success (uninstall will return a bogus HVA) 10022 * -errno: on error 10023 * 10024 * The caller should always use IS_ERR() to check the return value 10025 * before use. Note, the KVM internal memory slots are guaranteed to 10026 * remain valid and unchanged until the VM is destroyed, i.e., the 10027 * GPA->HVA translation will not change. However, the HVA is a user 10028 * address, i.e. its accessibility is not guaranteed, and must be 10029 * accessed via __copy_{to,from}_user(). 10030 */ 10031 void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, 10032 u32 size) 10033 { 10034 int i, r; 10035 unsigned long hva, old_npages; 10036 struct kvm_memslots *slots = kvm_memslots(kvm); 10037 struct kvm_memory_slot *slot; 10038 10039 lockdep_assert_held(&kvm->slots_lock); 10040 10041 if (WARN_ON(id >= KVM_MEM_SLOTS_NUM)) 10042 return ERR_PTR_USR(-EINVAL); 10043 10044 slot = id_to_memslot(slots, id); 10045 if (size) { 10046 if (slot && slot->npages) 10047 return ERR_PTR_USR(-EEXIST); 10048 10049 /* 10050 * MAP_SHARED to prevent internal slot pages from being moved 10051 * by fork()/COW. 10052 */ 10053 hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE, 10054 MAP_SHARED | MAP_ANONYMOUS, 0); 10055 if (IS_ERR_VALUE(hva)) 10056 return (void __user *)hva; 10057 } else { 10058 if (!slot || !slot->npages) 10059 return NULL; 10060 10061 old_npages = slot->npages; 10062 hva = slot->userspace_addr; 10063 } 10064 10065 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { 10066 struct kvm_userspace_memory_region2 m; 10067 10068 m.slot = id | (i << 16); 10069 m.flags = 0; 10070 m.guest_phys_addr = gpa; 10071 m.userspace_addr = hva; 10072 m.memory_size = size; 10073 r = kvm_set_internal_memslot(kvm, &m); 10074 if (r < 0) 10075 return ERR_PTR_USR(r); 10076 } 10077 10078 if (!size) 10079 vm_munmap(hva, old_npages * PAGE_SIZE); 10080 10081 return (void __user *)hva; 10082 } 10083 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__x86_set_memory_region); 10084 10085 void kvm_arch_pre_destroy_vm(struct kvm *kvm) 10086 { 10087 /* 10088 * Stop all background workers and kthreads before destroying vCPUs, as 10089 * iterating over vCPUs in a different task while vCPUs are being freed 10090 * is unsafe, i.e. will lead to use-after-free. The PIT also needs to 10091 * be stopped before IRQ routing is freed. 10092 * 10093 * Do NOT free the in-kernel PIC or I/O APIC here (but as above, make 10094 * sure to flush any background work), as KVM expects interrupt routing 10095 * structures to be valid until vCPUs are destroyed. 10096 */ 10097 #ifdef CONFIG_KVM_IOAPIC 10098 kvm_free_pit(kvm); 10099 if (kvm->arch.vioapic) 10100 cancel_delayed_work_sync(&kvm->arch.vioapic->eoi_inject); 10101 #endif 10102 10103 kvm_mmu_pre_destroy_vm(kvm); 10104 kvm_x86_call(vm_pre_destroy)(kvm); 10105 } 10106 10107 void kvm_arch_destroy_vm(struct kvm *kvm) 10108 { 10109 if (current->mm == kvm->mm) { 10110 /* 10111 * Free memory regions allocated on behalf of userspace, 10112 * unless the memory map has changed due to process exit 10113 * or fd copying. 10114 */ 10115 mutex_lock(&kvm->slots_lock); 10116 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 10117 0, 0); 10118 __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 10119 0, 0); 10120 __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0); 10121 mutex_unlock(&kvm->slots_lock); 10122 } 10123 if (kvm->arch.created_mediated_pmu) 10124 perf_release_mediated_pmu(); 10125 kvm_destroy_vcpus(kvm); 10126 kvm_free_msr_filter((void * __force)kvm->arch.msr_filter); 10127 #ifdef CONFIG_KVM_IOAPIC 10128 kvm_pic_destroy(kvm); 10129 kvm_ioapic_destroy(kvm); 10130 #endif 10131 kvfree(rcu_dereference_check(kvm->arch.apic_map, 1)); 10132 kfree((void * __force)kvm->arch.pmu_event_filter); 10133 kvm_mmu_uninit_vm(kvm); 10134 kvm_page_track_cleanup(kvm); 10135 kvm_xen_destroy_vm(kvm); 10136 kvm_hv_destroy_vm(kvm); 10137 kvm_x86_call(vm_destroy)(kvm); 10138 } 10139 10140 static void memslot_rmap_free(struct kvm_memory_slot *slot) 10141 { 10142 int i; 10143 10144 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 10145 vfree(slot->arch.rmap[i]); 10146 slot->arch.rmap[i] = NULL; 10147 } 10148 } 10149 10150 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) 10151 { 10152 int i; 10153 10154 memslot_rmap_free(slot); 10155 10156 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { 10157 vfree(slot->arch.lpage_info[i - 1]); 10158 slot->arch.lpage_info[i - 1] = NULL; 10159 } 10160 10161 kvm_page_track_free_memslot(slot); 10162 } 10163 10164 int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages) 10165 { 10166 const int sz = sizeof(*slot->arch.rmap[0]); 10167 int i; 10168 10169 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 10170 int level = i + 1; 10171 int lpages = __kvm_mmu_slot_lpages(slot, npages, level); 10172 10173 if (slot->arch.rmap[i]) 10174 continue; 10175 10176 slot->arch.rmap[i] = __vcalloc(lpages, sz, GFP_KERNEL_ACCOUNT); 10177 if (!slot->arch.rmap[i]) { 10178 memslot_rmap_free(slot); 10179 return -ENOMEM; 10180 } 10181 } 10182 10183 return 0; 10184 } 10185 10186 static int kvm_alloc_memslot_metadata(struct kvm *kvm, 10187 struct kvm_memory_slot *slot) 10188 { 10189 unsigned long npages = slot->npages; 10190 int i, r; 10191 10192 /* 10193 * Clear out the previous array pointers for the KVM_MR_MOVE case. The 10194 * old arrays will be freed by kvm_set_memory_region() if installing 10195 * the new memslot is successful. 10196 */ 10197 memset(&slot->arch, 0, sizeof(slot->arch)); 10198 10199 if (kvm_memslots_have_rmaps(kvm)) { 10200 r = memslot_rmap_alloc(slot, npages); 10201 if (r) 10202 return r; 10203 } 10204 10205 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { 10206 struct kvm_lpage_info *linfo; 10207 unsigned long ugfn; 10208 int lpages; 10209 int level = i + 1; 10210 10211 lpages = __kvm_mmu_slot_lpages(slot, npages, level); 10212 10213 linfo = __vcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT); 10214 if (!linfo) 10215 goto out_free; 10216 10217 slot->arch.lpage_info[i - 1] = linfo; 10218 10219 if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1)) 10220 linfo[0].disallow_lpage = 1; 10221 if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1)) 10222 linfo[lpages - 1].disallow_lpage = 1; 10223 ugfn = slot->userspace_addr >> PAGE_SHIFT; 10224 /* 10225 * If the gfn and userspace address are not aligned wrt each 10226 * other, disable large page support for this slot. 10227 */ 10228 if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) { 10229 unsigned long j; 10230 10231 for (j = 0; j < lpages; ++j) 10232 linfo[j].disallow_lpage = 1; 10233 } 10234 } 10235 10236 #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES 10237 kvm_mmu_init_memslot_memory_attributes(kvm, slot); 10238 #endif 10239 10240 if (kvm_page_track_create_memslot(kvm, slot, npages)) 10241 goto out_free; 10242 10243 return 0; 10244 10245 out_free: 10246 memslot_rmap_free(slot); 10247 10248 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { 10249 vfree(slot->arch.lpage_info[i - 1]); 10250 slot->arch.lpage_info[i - 1] = NULL; 10251 } 10252 return -ENOMEM; 10253 } 10254 10255 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) 10256 { 10257 struct kvm_vcpu *vcpu; 10258 unsigned long i; 10259 10260 /* 10261 * memslots->generation has been incremented. 10262 * mmio generation may have reached its maximum value. 10263 */ 10264 kvm_mmu_invalidate_mmio_sptes(kvm, gen); 10265 10266 /* Force re-initialization of steal_time cache */ 10267 kvm_for_each_vcpu(i, vcpu, kvm) 10268 kvm_vcpu_kick(vcpu); 10269 } 10270 10271 int kvm_arch_prepare_memory_region(struct kvm *kvm, 10272 const struct kvm_memory_slot *old, 10273 struct kvm_memory_slot *new, 10274 enum kvm_mr_change change) 10275 { 10276 /* 10277 * KVM doesn't support moving memslots when there are external page 10278 * trackers attached to the VM, i.e. if KVMGT is in use. 10279 */ 10280 if (change == KVM_MR_MOVE && kvm_page_track_has_external_user(kvm)) 10281 return -EINVAL; 10282 10283 if (change == KVM_MR_CREATE || change == KVM_MR_MOVE) { 10284 if ((new->base_gfn + new->npages - 1) > kvm_mmu_max_gfn()) 10285 return -EINVAL; 10286 10287 if (kvm_is_gfn_alias(kvm, new->base_gfn + new->npages - 1)) 10288 return -EINVAL; 10289 10290 return kvm_alloc_memslot_metadata(kvm, new); 10291 } 10292 10293 if (change == KVM_MR_FLAGS_ONLY) 10294 memcpy(&new->arch, &old->arch, sizeof(old->arch)); 10295 else if (WARN_ON_ONCE(change != KVM_MR_DELETE)) 10296 return -EIO; 10297 10298 return 0; 10299 } 10300 10301 10302 static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable) 10303 { 10304 int nr_slots; 10305 10306 if (!kvm->arch.cpu_dirty_log_size) 10307 return; 10308 10309 nr_slots = atomic_read(&kvm->nr_memslots_dirty_logging); 10310 if ((enable && nr_slots == 1) || !nr_slots) 10311 kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING); 10312 } 10313 10314 static void kvm_mmu_slot_apply_flags(struct kvm *kvm, 10315 struct kvm_memory_slot *old, 10316 const struct kvm_memory_slot *new, 10317 enum kvm_mr_change change) 10318 { 10319 u32 old_flags = old ? old->flags : 0; 10320 u32 new_flags = new ? new->flags : 0; 10321 bool log_dirty_pages = new_flags & KVM_MEM_LOG_DIRTY_PAGES; 10322 10323 /* 10324 * Update CPU dirty logging if dirty logging is being toggled. This 10325 * applies to all operations. 10326 */ 10327 if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES) 10328 kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages); 10329 10330 /* 10331 * Nothing more to do for RO slots (which can't be dirtied and can't be 10332 * made writable) or CREATE/MOVE/DELETE of a slot. 10333 * 10334 * For a memslot with dirty logging disabled: 10335 * CREATE: No dirty mappings will already exist. 10336 * MOVE/DELETE: The old mappings will already have been cleaned up by 10337 * kvm_arch_flush_shadow_memslot() 10338 * 10339 * For a memslot with dirty logging enabled: 10340 * CREATE: No shadow pages exist, thus nothing to write-protect 10341 * and no dirty bits to clear. 10342 * MOVE/DELETE: The old mappings will already have been cleaned up by 10343 * kvm_arch_flush_shadow_memslot(). 10344 */ 10345 if ((change != KVM_MR_FLAGS_ONLY) || (new_flags & KVM_MEM_READONLY)) 10346 return; 10347 10348 /* 10349 * READONLY and non-flags changes were filtered out above, and the only 10350 * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty 10351 * logging isn't being toggled on or off. 10352 */ 10353 if (WARN_ON_ONCE(!((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES))) 10354 return; 10355 10356 if (!log_dirty_pages) { 10357 /* 10358 * Recover huge page mappings in the slot now that dirty logging 10359 * is disabled, i.e. now that KVM does not have to track guest 10360 * writes at 4KiB granularity. 10361 * 10362 * Dirty logging might be disabled by userspace if an ongoing VM 10363 * live migration is cancelled and the VM must continue running 10364 * on the source. 10365 */ 10366 kvm_mmu_recover_huge_pages(kvm, new); 10367 } else { 10368 /* 10369 * Initially-all-set does not require write protecting any page, 10370 * because they're all assumed to be dirty. 10371 */ 10372 if (kvm_dirty_log_manual_protect_and_init_set(kvm)) 10373 return; 10374 10375 if (READ_ONCE(eager_page_split)) 10376 kvm_mmu_slot_try_split_huge_pages(kvm, new, PG_LEVEL_4K); 10377 10378 if (kvm->arch.cpu_dirty_log_size) { 10379 kvm_mmu_slot_leaf_clear_dirty(kvm, new); 10380 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M); 10381 } else { 10382 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K); 10383 } 10384 10385 /* 10386 * Unconditionally flush the TLBs after enabling dirty logging. 10387 * A flush is almost always going to be necessary (see below), 10388 * and unconditionally flushing allows the helpers to omit 10389 * the subtly complex checks when removing write access. 10390 * 10391 * Do the flush outside of mmu_lock to reduce the amount of 10392 * time mmu_lock is held. Flushing after dropping mmu_lock is 10393 * safe as KVM only needs to guarantee the slot is fully 10394 * write-protected before returning to userspace, i.e. before 10395 * userspace can consume the dirty status. 10396 * 10397 * Flushing outside of mmu_lock requires KVM to be careful when 10398 * making decisions based on writable status of an SPTE, e.g. a 10399 * !writable SPTE doesn't guarantee a CPU can't perform writes. 10400 * 10401 * Specifically, KVM also write-protects guest page tables to 10402 * monitor changes when using shadow paging, and must guarantee 10403 * no CPUs can write to those page before mmu_lock is dropped. 10404 * Because CPUs may have stale TLB entries at this point, a 10405 * !writable SPTE doesn't guarantee CPUs can't perform writes. 10406 * 10407 * KVM also allows making SPTES writable outside of mmu_lock, 10408 * e.g. to allow dirty logging without taking mmu_lock. 10409 * 10410 * To handle these scenarios, KVM uses a separate software-only 10411 * bit (MMU-writable) to track if a SPTE is !writable due to 10412 * a guest page table being write-protected (KVM clears the 10413 * MMU-writable flag when write-protecting for shadow paging). 10414 * 10415 * The use of MMU-writable is also the primary motivation for 10416 * the unconditional flush. Because KVM must guarantee that a 10417 * CPU doesn't contain stale, writable TLB entries for a 10418 * !MMU-writable SPTE, KVM must flush if it encounters any 10419 * MMU-writable SPTE regardless of whether the actual hardware 10420 * writable bit was set. I.e. KVM is almost guaranteed to need 10421 * to flush, while unconditionally flushing allows the "remove 10422 * write access" helpers to ignore MMU-writable entirely. 10423 * 10424 * See is_writable_pte() for more details (the case involving 10425 * access-tracked SPTEs is particularly relevant). 10426 */ 10427 kvm_flush_remote_tlbs_memslot(kvm, new); 10428 } 10429 } 10430 10431 void kvm_arch_commit_memory_region(struct kvm *kvm, 10432 struct kvm_memory_slot *old, 10433 const struct kvm_memory_slot *new, 10434 enum kvm_mr_change change) 10435 { 10436 if (change == KVM_MR_DELETE) 10437 kvm_page_track_delete_slot(kvm, old); 10438 10439 if (!kvm->arch.n_requested_mmu_pages && 10440 (change == KVM_MR_CREATE || change == KVM_MR_DELETE)) { 10441 unsigned long nr_mmu_pages; 10442 10443 nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO; 10444 nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES); 10445 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); 10446 } 10447 10448 kvm_mmu_slot_apply_flags(kvm, old, new, change); 10449 10450 /* Free the arrays associated with the old memslot. */ 10451 if (change == KVM_MR_MOVE) 10452 kvm_arch_free_memslot(kvm, old); 10453 } 10454 10455 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 10456 { 10457 WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu)); 10458 10459 if (vcpu->arch.guest_state_protected) 10460 return true; 10461 10462 return kvm_x86_call(get_cpl)(vcpu) == 0; 10463 } 10464 10465 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) 10466 { 10467 WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu)); 10468 10469 if (vcpu->arch.guest_state_protected) 10470 return 0; 10471 10472 return kvm_rip_read(vcpu); 10473 } 10474 10475 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 10476 { 10477 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 10478 } 10479 10480 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) 10481 { 10482 BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU)); 10483 10484 return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU)); 10485 } 10486 10487 static inline u32 kvm_async_pf_next_probe(u32 key) 10488 { 10489 return (key + 1) & (ASYNC_PF_PER_VCPU - 1); 10490 } 10491 10492 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 10493 { 10494 u32 key = kvm_async_pf_hash_fn(gfn); 10495 10496 while (vcpu->arch.apf.gfns[key] != ~0) 10497 key = kvm_async_pf_next_probe(key); 10498 10499 vcpu->arch.apf.gfns[key] = gfn; 10500 } 10501 10502 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn) 10503 { 10504 int i; 10505 u32 key = kvm_async_pf_hash_fn(gfn); 10506 10507 for (i = 0; i < ASYNC_PF_PER_VCPU && 10508 (vcpu->arch.apf.gfns[key] != gfn && 10509 vcpu->arch.apf.gfns[key] != ~0); i++) 10510 key = kvm_async_pf_next_probe(key); 10511 10512 return key; 10513 } 10514 10515 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 10516 { 10517 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn; 10518 } 10519 10520 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 10521 { 10522 u32 i, j, k; 10523 10524 i = j = kvm_async_pf_gfn_slot(vcpu, gfn); 10525 10526 if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn)) 10527 return; 10528 10529 while (true) { 10530 vcpu->arch.apf.gfns[i] = ~0; 10531 do { 10532 j = kvm_async_pf_next_probe(j); 10533 if (vcpu->arch.apf.gfns[j] == ~0) 10534 return; 10535 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]); 10536 /* 10537 * k lies cyclically in ]i,j] 10538 * | i.k.j | 10539 * |....j i.k.| or |.k..j i...| 10540 */ 10541 } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j)); 10542 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j]; 10543 i = j; 10544 } 10545 } 10546 10547 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu) 10548 { 10549 u32 reason = KVM_PV_REASON_PAGE_NOT_PRESENT; 10550 10551 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason, 10552 sizeof(reason)); 10553 } 10554 10555 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token) 10556 { 10557 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token); 10558 10559 return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, 10560 &token, offset, sizeof(token)); 10561 } 10562 10563 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu) 10564 { 10565 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token); 10566 u32 val; 10567 10568 if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, 10569 &val, offset, sizeof(val))) 10570 return false; 10571 10572 return !val; 10573 } 10574 10575 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) 10576 { 10577 10578 if (!kvm_pv_async_pf_enabled(vcpu)) 10579 return false; 10580 10581 if (!(vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_SEND_ALWAYS) && 10582 (vcpu->arch.guest_state_protected || !kvm_x86_call(get_cpl)(vcpu))) 10583 return false; 10584 10585 if (is_guest_mode(vcpu)) { 10586 /* 10587 * L1 needs to opt into the special #PF vmexits that are 10588 * used to deliver async page faults. 10589 */ 10590 return vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; 10591 } else { 10592 /* 10593 * Play it safe in case the guest temporarily disables paging. 10594 * The real mode IDT in particular is unlikely to have a #PF 10595 * exception setup. 10596 */ 10597 return is_paging(vcpu); 10598 } 10599 } 10600 10601 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu) 10602 { 10603 if (unlikely(!lapic_in_kernel(vcpu) || 10604 kvm_event_needs_reinjection(vcpu) || 10605 kvm_is_exception_pending(vcpu))) 10606 return false; 10607 10608 if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu)) 10609 return false; 10610 10611 /* 10612 * If interrupts are off we cannot even use an artificial 10613 * halt state. 10614 */ 10615 return kvm_is_interrupt_allowed(vcpu); 10616 } 10617 10618 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 10619 struct kvm_async_pf *work) 10620 { 10621 struct x86_exception fault; 10622 10623 trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa); 10624 kvm_add_async_pf_gfn(vcpu, work->arch.gfn); 10625 10626 if (kvm_can_deliver_async_pf(vcpu) && 10627 !apf_put_user_notpresent(vcpu)) { 10628 fault.vector = PF_VECTOR; 10629 fault.error_code_valid = true; 10630 fault.error_code = 0; 10631 fault.nested_page_fault = false; 10632 fault.address = work->arch.token; 10633 fault.async_page_fault = true; 10634 kvm_inject_page_fault(vcpu, &fault, false); 10635 return true; 10636 } else { 10637 /* 10638 * It is not possible to deliver a paravirtualized asynchronous 10639 * page fault, but putting the guest in an artificial halt state 10640 * can be beneficial nevertheless: if an interrupt arrives, we 10641 * can deliver it timely and perhaps the guest will schedule 10642 * another process. When the instruction that triggered a page 10643 * fault is retried, hopefully the page will be ready in the host. 10644 */ 10645 kvm_make_request(KVM_REQ_APF_HALT, vcpu); 10646 return false; 10647 } 10648 } 10649 10650 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 10651 struct kvm_async_pf *work) 10652 { 10653 struct kvm_lapic_irq irq = { 10654 .delivery_mode = APIC_DM_FIXED, 10655 .vector = vcpu->arch.apf.vec 10656 }; 10657 10658 if (work->wakeup_all) 10659 work->arch.token = ~0; /* broadcast wakeup */ 10660 else 10661 kvm_del_async_pf_gfn(vcpu, work->arch.gfn); 10662 trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa); 10663 10664 if ((work->wakeup_all || work->notpresent_injected) && 10665 kvm_pv_async_pf_enabled(vcpu) && 10666 !apf_put_user_ready(vcpu, work->arch.token)) { 10667 WRITE_ONCE(vcpu->arch.apf.pageready_pending, true); 10668 kvm_apic_set_irq(vcpu, &irq, NULL); 10669 } 10670 10671 vcpu->arch.apf.halted = false; 10672 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 10673 } 10674 10675 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) 10676 { 10677 kvm_make_request(KVM_REQ_APF_READY, vcpu); 10678 10679 /* Pairs with smp_store_mb() in kvm_set_msr_common(). */ 10680 smp_mb__after_atomic(); 10681 10682 if (!READ_ONCE(vcpu->arch.apf.pageready_pending)) 10683 kvm_vcpu_kick(vcpu); 10684 } 10685 10686 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) 10687 { 10688 if (!kvm_pv_async_pf_enabled(vcpu)) 10689 return true; 10690 else 10691 return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu); 10692 } 10693 10694 static void kvm_noncoherent_dma_assignment_start_or_stop(struct kvm *kvm) 10695 { 10696 /* 10697 * Non-coherent DMA assignment and de-assignment may affect whether or 10698 * not KVM honors guest PAT, and thus may cause changes in EPT SPTEs 10699 * due to toggling the "ignore PAT" bit. Zap all SPTEs when the first 10700 * (or last) non-coherent device is (un)registered to so that new SPTEs 10701 * with the correct "ignore guest PAT" setting are created. 10702 * 10703 * If KVM always honors guest PAT, however, there is nothing to do. 10704 */ 10705 if (kvm_check_has_quirk(kvm, KVM_X86_QUIRK_IGNORE_GUEST_PAT)) 10706 kvm_zap_gfn_range(kvm, gpa_to_gfn(0), gpa_to_gfn(~0ULL)); 10707 } 10708 10709 void kvm_arch_register_noncoherent_dma(struct kvm *kvm) 10710 { 10711 if (atomic_inc_return(&kvm->arch.noncoherent_dma_count) == 1) 10712 kvm_noncoherent_dma_assignment_start_or_stop(kvm); 10713 } 10714 10715 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm) 10716 { 10717 if (!atomic_dec_return(&kvm->arch.noncoherent_dma_count)) 10718 kvm_noncoherent_dma_assignment_start_or_stop(kvm); 10719 } 10720 10721 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm) 10722 { 10723 return atomic_read(&kvm->arch.noncoherent_dma_count); 10724 } 10725 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_arch_has_noncoherent_dma); 10726 10727 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) 10728 { 10729 return (vcpu->arch.msr_kvm_poll_control & 1) == 0; 10730 } 10731 10732 #ifdef CONFIG_KVM_GUEST_MEMFD 10733 bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) 10734 { 10735 /* 10736 * INIT_SHARED is supported if in-place conversion is enabled, or if 10737 * the VM doesn't support private memory. If the VM has private memory 10738 * and in-place conversion is disabled, then guest_memfd can _only_ be 10739 * used for private memory. 10740 */ 10741 return gmem_in_place_conversion || !kvm_arch_has_private_mem(kvm); 10742 } 10743 10744 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT 10745 int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, 10746 kvm_pfn_t nr_pages) 10747 { 10748 return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages); 10749 } 10750 10751 void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages) 10752 { 10753 kvm_x86_call(gmem_make_shared)(pfn, nr_pages); 10754 } 10755 #endif 10756 10757 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM 10758 void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages) 10759 { 10760 kvm_x86_call(gmem_make_shared)(pfn, nr_pages); 10761 } 10762 #endif 10763 10764 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE 10765 void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) 10766 { 10767 kvm_x86_call(gmem_invalidate_range)(kvm, range); 10768 } 10769 #endif 10770 #endif 10771 10772 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code) 10773 { 10774 struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk; 10775 struct x86_exception fault; 10776 u64 access = error_code & 10777 (PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK); 10778 10779 if (!(error_code & PFERR_PRESENT_MASK) || 10780 gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, &fault) != INVALID_GPA) { 10781 /* 10782 * If gva_walk->gva_to_gpa succeeded, the page 10783 * tables probably do not match the TLB. Just proceed 10784 * with the error code that the processor gave. 10785 */ 10786 fault.vector = PF_VECTOR; 10787 fault.error_code_valid = true; 10788 fault.error_code = error_code; 10789 fault.nested_page_fault = false; 10790 fault.address = gva; 10791 fault.async_page_fault = false; 10792 } 10793 gva_walk->inject_page_fault(vcpu, &fault, true); 10794 } 10795 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fixup_and_inject_pf_error); 10796 10797 /* 10798 * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns 10799 * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value 10800 * indicates whether exit to userspace is needed. 10801 */ 10802 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r, 10803 struct x86_exception *e) 10804 { 10805 if (r == X86EMUL_PROPAGATE_FAULT) { 10806 if (KVM_BUG_ON(!e, vcpu->kvm)) 10807 return -EIO; 10808 10809 kvm_inject_emulated_page_fault(vcpu, e); 10810 return 1; 10811 } 10812 10813 /* 10814 * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED 10815 * while handling a VMX instruction KVM could've handled the request 10816 * correctly by exiting to userspace and performing I/O but there 10817 * doesn't seem to be a real use-case behind such requests, just return 10818 * KVM_EXIT_INTERNAL_ERROR for now. 10819 */ 10820 kvm_prepare_emulation_failure_exit(vcpu); 10821 10822 return 0; 10823 } 10824 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_memory_failure); 10825 10826 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) 10827 { 10828 bool pcid_enabled; 10829 struct x86_exception e; 10830 struct { 10831 u64 pcid; 10832 u64 gla; 10833 } operand; 10834 int r; 10835 10836 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 10837 if (r != X86EMUL_CONTINUE) 10838 return kvm_handle_memory_failure(vcpu, r, &e); 10839 10840 if (operand.pcid >> 12 != 0) { 10841 kvm_inject_gp(vcpu, 0); 10842 return 1; 10843 } 10844 10845 if (WARN_ON_ONCE(tdp_enabled)) 10846 return 0; 10847 10848 pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE); 10849 10850 switch (type) { 10851 case INVPCID_TYPE_INDIV_ADDR: 10852 /* 10853 * LAM doesn't apply to addresses that are inputs to TLB 10854 * invalidation. 10855 */ 10856 if ((!pcid_enabled && (operand.pcid != 0)) || 10857 is_noncanonical_invlpg_address(operand.gla, vcpu)) { 10858 kvm_inject_gp(vcpu, 0); 10859 return 1; 10860 } 10861 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid); 10862 return kvm_skip_emulated_instruction(vcpu); 10863 10864 case INVPCID_TYPE_SINGLE_CTXT: 10865 if (!pcid_enabled && (operand.pcid != 0)) { 10866 kvm_inject_gp(vcpu, 0); 10867 return 1; 10868 } 10869 10870 /* 10871 * When ERAPS is supported, invalidating a specific PCID clears 10872 * the RAP (Return Address Predicator). 10873 */ 10874 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS)) 10875 kvm_register_mark_dirty(vcpu, VCPU_REG_ERAPS); 10876 10877 kvm_invalidate_pcid(vcpu, operand.pcid); 10878 return kvm_skip_emulated_instruction(vcpu); 10879 10880 case INVPCID_TYPE_ALL_NON_GLOBAL: 10881 /* 10882 * Currently, KVM doesn't mark global entries in the shadow 10883 * page tables, so a non-global flush just degenerates to a 10884 * global flush. If needed, we could optimize this later by 10885 * keeping track of global entries in shadow page tables. 10886 */ 10887 10888 fallthrough; 10889 case INVPCID_TYPE_ALL_INCL_GLOBAL: 10890 /* 10891 * Don't bother marking VCPU_REG_ERAPS dirty, SVM will take 10892 * care of doing so when emulating the full guest TLB flush 10893 * (the RAP is cleared on all implicit TLB flushes). 10894 */ 10895 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 10896 return kvm_skip_emulated_instruction(vcpu); 10897 10898 default: 10899 kvm_inject_gp(vcpu, 0); 10900 return 1; 10901 } 10902 } 10903 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_invpcid); 10904 10905 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu) 10906 { 10907 struct kvm_run *run = vcpu->run; 10908 struct kvm_mmio_fragment *frag; 10909 unsigned int len; 10910 10911 if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm)) 10912 return -EIO; 10913 10914 /* Complete previous fragment */ 10915 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment]; 10916 len = min(8u, frag->len); 10917 if (!vcpu->mmio_is_write) 10918 memcpy(frag->data, run->mmio.data, len); 10919 10920 if (frag->len <= 8) { 10921 /* Switch to the next fragment. */ 10922 frag++; 10923 vcpu->mmio_cur_fragment++; 10924 } else { 10925 /* Go forward to the next mmio piece. */ 10926 frag->data += len; 10927 frag->gpa += len; 10928 frag->len -= len; 10929 } 10930 10931 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) { 10932 vcpu->mmio_needed = 0; 10933 10934 /* 10935 * All done, as frag->data always points at the GHCB scratch 10936 * area and VMGEXIT is trap-like (RIP is advanced by hardware). 10937 */ 10938 return 1; 10939 } 10940 10941 // More MMIO is needed 10942 kvm_prepare_emulated_mmio_exit(vcpu, frag); 10943 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; 10944 return 0; 10945 } 10946 10947 int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa, 10948 unsigned int bytes, void *data) 10949 { 10950 struct kvm_mmio_fragment *frag; 10951 int handled; 10952 10953 if (!data || WARN_ON_ONCE(object_is_on_stack(data))) 10954 return -EINVAL; 10955 10956 if (is_write) 10957 handled = vcpu_mmio_write(vcpu, gpa, bytes, data); 10958 else 10959 handled = vcpu_mmio_read(vcpu, gpa, bytes, data); 10960 if (handled == bytes) 10961 return 1; 10962 10963 bytes -= handled; 10964 gpa += handled; 10965 data += handled; 10966 10967 /* 10968 * TODO: Determine whether or not userspace plays nice with MMIO 10969 * requests that split a page boundary. 10970 */ 10971 frag = vcpu->mmio_fragments; 10972 frag->len = bytes; 10973 frag->gpa = gpa; 10974 frag->data = data; 10975 10976 vcpu->mmio_needed = 1; 10977 vcpu->mmio_cur_fragment = 0; 10978 vcpu->mmio_nr_fragments = 1; 10979 vcpu->mmio_is_write = is_write; 10980 10981 kvm_prepare_emulated_mmio_exit(vcpu, frag); 10982 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; 10983 return 0; 10984 } 10985 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio); 10986 10987 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size) 10988 { 10989 vcpu->arch.sev_pio_count -= count; 10990 vcpu->arch.sev_pio_data += count * size; 10991 } 10992 10993 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size, 10994 unsigned int port); 10995 10996 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu) 10997 { 10998 int size = vcpu->arch.pio.size; 10999 int port = vcpu->arch.pio.port; 11000 11001 vcpu->arch.pio.count = 0; 11002 if (vcpu->arch.sev_pio_count) 11003 return kvm_sev_es_outs(vcpu, size, port); 11004 return 1; 11005 } 11006 11007 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size, 11008 unsigned int port) 11009 { 11010 for (;;) { 11011 unsigned int count = 11012 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count); 11013 int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count); 11014 11015 /* memcpy done already by emulator_pio_out. */ 11016 advance_sev_es_emulated_pio(vcpu, count, size); 11017 if (!ret) 11018 break; 11019 11020 /* Emulation done by the kernel. */ 11021 if (!vcpu->arch.sev_pio_count) 11022 return 1; 11023 } 11024 11025 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs; 11026 return 0; 11027 } 11028 11029 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size, 11030 unsigned int port); 11031 11032 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu) 11033 { 11034 unsigned count = vcpu->arch.pio.count; 11035 int size = vcpu->arch.pio.size; 11036 int port = vcpu->arch.pio.port; 11037 11038 complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data); 11039 advance_sev_es_emulated_pio(vcpu, count, size); 11040 if (vcpu->arch.sev_pio_count) 11041 return kvm_sev_es_ins(vcpu, size, port); 11042 return 1; 11043 } 11044 11045 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size, 11046 unsigned int port) 11047 { 11048 for (;;) { 11049 unsigned int count = 11050 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count); 11051 if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count)) 11052 break; 11053 11054 /* Emulation done by the kernel. */ 11055 advance_sev_es_emulated_pio(vcpu, count, size); 11056 if (!vcpu->arch.sev_pio_count) 11057 return 1; 11058 } 11059 11060 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins; 11061 return 0; 11062 } 11063 11064 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size, 11065 unsigned int port, void *data, unsigned int count, 11066 int in) 11067 { 11068 vcpu->arch.sev_pio_data = data; 11069 vcpu->arch.sev_pio_count = count; 11070 return in ? kvm_sev_es_ins(vcpu, size, port) 11071 : kvm_sev_es_outs(vcpu, size, port); 11072 } 11073 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_string_io); 11074 11075 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_entry); 11076 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit); 11077 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_mmio); 11078 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio); 11079 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); 11080 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); 11081 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr); 11082 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr); 11083 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter); 11084 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit); 11085 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject); 11086 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit); 11087 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed); 11088 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga); 11089 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit); 11090 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts); 11091 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset); 11092 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update); 11093 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full); 11094 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access); 11095 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi); 11096 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log); 11097 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_kick_vcpu_slowpath); 11098 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_doorbell); 11099 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq); 11100 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter); 11101 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit); 11102 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_enter); 11103 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_exit); 11104 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_rmp_fault); 11105 11106 static int __init kvm_x86_init(void) 11107 { 11108 kvm_init_xstate_sizes(); 11109 11110 kvm_mmu_x86_module_init(); 11111 mitigate_smt_rsb &= boot_cpu_has_bug(X86_BUG_SMT_RSB) && cpu_smt_possible(); 11112 return 0; 11113 } 11114 module_init(kvm_x86_init); 11115 11116 static void __exit kvm_x86_exit(void) 11117 { 11118 WARN_ON_ONCE(static_branch_unlikely(&kvm_has_noapic_vcpu)); 11119 } 11120 module_exit(kvm_x86_exit); 11121