1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * derived from drivers/kvm/kvm_main.c 5 * 6 * Copyright (C) 2006 Qumranet, Inc. 7 * Copyright (C) 2008 Qumranet, Inc. 8 * Copyright IBM Corporation, 2008 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 * Amit Shah <amit.shah@qumranet.com> 15 * Ben-Ami Yassour <benami@il.ibm.com> 16 * 17 * This work is licensed under the terms of the GNU GPL, version 2. See 18 * the COPYING file in the top-level directory. 19 * 20 */ 21 22 #include <linux/kvm_host.h> 23 #include "irq.h" 24 #include "mmu.h" 25 #include "i8254.h" 26 #include "tss.h" 27 #include "kvm_cache_regs.h" 28 #include "x86.h" 29 30 #include <linux/clocksource.h> 31 #include <linux/interrupt.h> 32 #include <linux/kvm.h> 33 #include <linux/fs.h> 34 #include <linux/vmalloc.h> 35 #include <linux/module.h> 36 #include <linux/mman.h> 37 #include <linux/highmem.h> 38 #include <linux/iommu.h> 39 #include <linux/intel-iommu.h> 40 #include <linux/cpufreq.h> 41 #include <linux/user-return-notifier.h> 42 #include <linux/srcu.h> 43 #include <linux/slab.h> 44 #include <linux/perf_event.h> 45 #include <linux/uaccess.h> 46 #include <linux/hash.h> 47 #include <trace/events/kvm.h> 48 49 #define CREATE_TRACE_POINTS 50 #include "trace.h" 51 52 #include <asm/debugreg.h> 53 #include <asm/msr.h> 54 #include <asm/desc.h> 55 #include <asm/mtrr.h> 56 #include <asm/mce.h> 57 #include <asm/i387.h> 58 #include <asm/xcr.h> 59 #include <asm/pvclock.h> 60 #include <asm/div64.h> 61 62 #define MAX_IO_MSRS 256 63 #define KVM_MAX_MCE_BANKS 32 64 #define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P) 65 66 #define emul_to_vcpu(ctxt) \ 67 container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt) 68 69 /* EFER defaults: 70 * - enable syscall per default because its emulated by KVM 71 * - enable LME and LMA per default on 64 bit KVM 72 */ 73 #ifdef CONFIG_X86_64 74 static 75 u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA)); 76 #else 77 static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); 78 #endif 79 80 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM 81 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 82 83 static void update_cr8_intercept(struct kvm_vcpu *vcpu); 84 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, 85 struct kvm_cpuid_entry2 __user *entries); 86 87 struct kvm_x86_ops *kvm_x86_ops; 88 EXPORT_SYMBOL_GPL(kvm_x86_ops); 89 90 int ignore_msrs = 0; 91 module_param_named(ignore_msrs, ignore_msrs, bool, S_IRUGO | S_IWUSR); 92 93 bool kvm_has_tsc_control; 94 EXPORT_SYMBOL_GPL(kvm_has_tsc_control); 95 u32 kvm_max_guest_tsc_khz; 96 EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz); 97 98 #define KVM_NR_SHARED_MSRS 16 99 100 struct kvm_shared_msrs_global { 101 int nr; 102 u32 msrs[KVM_NR_SHARED_MSRS]; 103 }; 104 105 struct kvm_shared_msrs { 106 struct user_return_notifier urn; 107 bool registered; 108 struct kvm_shared_msr_values { 109 u64 host; 110 u64 curr; 111 } values[KVM_NR_SHARED_MSRS]; 112 }; 113 114 static struct kvm_shared_msrs_global __read_mostly shared_msrs_global; 115 static DEFINE_PER_CPU(struct kvm_shared_msrs, shared_msrs); 116 117 struct kvm_stats_debugfs_item debugfs_entries[] = { 118 { "pf_fixed", VCPU_STAT(pf_fixed) }, 119 { "pf_guest", VCPU_STAT(pf_guest) }, 120 { "tlb_flush", VCPU_STAT(tlb_flush) }, 121 { "invlpg", VCPU_STAT(invlpg) }, 122 { "exits", VCPU_STAT(exits) }, 123 { "io_exits", VCPU_STAT(io_exits) }, 124 { "mmio_exits", VCPU_STAT(mmio_exits) }, 125 { "signal_exits", VCPU_STAT(signal_exits) }, 126 { "irq_window", VCPU_STAT(irq_window_exits) }, 127 { "nmi_window", VCPU_STAT(nmi_window_exits) }, 128 { "halt_exits", VCPU_STAT(halt_exits) }, 129 { "halt_wakeup", VCPU_STAT(halt_wakeup) }, 130 { "hypercalls", VCPU_STAT(hypercalls) }, 131 { "request_irq", VCPU_STAT(request_irq_exits) }, 132 { "irq_exits", VCPU_STAT(irq_exits) }, 133 { "host_state_reload", VCPU_STAT(host_state_reload) }, 134 { "efer_reload", VCPU_STAT(efer_reload) }, 135 { "fpu_reload", VCPU_STAT(fpu_reload) }, 136 { "insn_emulation", VCPU_STAT(insn_emulation) }, 137 { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, 138 { "irq_injections", VCPU_STAT(irq_injections) }, 139 { "nmi_injections", VCPU_STAT(nmi_injections) }, 140 { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, 141 { "mmu_pte_write", VM_STAT(mmu_pte_write) }, 142 { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, 143 { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, 144 { "mmu_flooded", VM_STAT(mmu_flooded) }, 145 { "mmu_recycled", VM_STAT(mmu_recycled) }, 146 { "mmu_cache_miss", VM_STAT(mmu_cache_miss) }, 147 { "mmu_unsync", VM_STAT(mmu_unsync) }, 148 { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, 149 { "largepages", VM_STAT(lpages) }, 150 { NULL } 151 }; 152 153 u64 __read_mostly host_xcr0; 154 155 int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt); 156 157 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) 158 { 159 int i; 160 for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++) 161 vcpu->arch.apf.gfns[i] = ~0; 162 } 163 164 static void kvm_on_user_return(struct user_return_notifier *urn) 165 { 166 unsigned slot; 167 struct kvm_shared_msrs *locals 168 = container_of(urn, struct kvm_shared_msrs, urn); 169 struct kvm_shared_msr_values *values; 170 171 for (slot = 0; slot < shared_msrs_global.nr; ++slot) { 172 values = &locals->values[slot]; 173 if (values->host != values->curr) { 174 wrmsrl(shared_msrs_global.msrs[slot], values->host); 175 values->curr = values->host; 176 } 177 } 178 locals->registered = false; 179 user_return_notifier_unregister(urn); 180 } 181 182 static void shared_msr_update(unsigned slot, u32 msr) 183 { 184 struct kvm_shared_msrs *smsr; 185 u64 value; 186 187 smsr = &__get_cpu_var(shared_msrs); 188 /* only read, and nobody should modify it at this time, 189 * so don't need lock */ 190 if (slot >= shared_msrs_global.nr) { 191 printk(KERN_ERR "kvm: invalid MSR slot!"); 192 return; 193 } 194 rdmsrl_safe(msr, &value); 195 smsr->values[slot].host = value; 196 smsr->values[slot].curr = value; 197 } 198 199 void kvm_define_shared_msr(unsigned slot, u32 msr) 200 { 201 if (slot >= shared_msrs_global.nr) 202 shared_msrs_global.nr = slot + 1; 203 shared_msrs_global.msrs[slot] = msr; 204 /* we need ensured the shared_msr_global have been updated */ 205 smp_wmb(); 206 } 207 EXPORT_SYMBOL_GPL(kvm_define_shared_msr); 208 209 static void kvm_shared_msr_cpu_online(void) 210 { 211 unsigned i; 212 213 for (i = 0; i < shared_msrs_global.nr; ++i) 214 shared_msr_update(i, shared_msrs_global.msrs[i]); 215 } 216 217 void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask) 218 { 219 struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs); 220 221 if (((value ^ smsr->values[slot].curr) & mask) == 0) 222 return; 223 smsr->values[slot].curr = value; 224 wrmsrl(shared_msrs_global.msrs[slot], value); 225 if (!smsr->registered) { 226 smsr->urn.on_user_return = kvm_on_user_return; 227 user_return_notifier_register(&smsr->urn); 228 smsr->registered = true; 229 } 230 } 231 EXPORT_SYMBOL_GPL(kvm_set_shared_msr); 232 233 static void drop_user_return_notifiers(void *ignore) 234 { 235 struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs); 236 237 if (smsr->registered) 238 kvm_on_user_return(&smsr->urn); 239 } 240 241 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) 242 { 243 if (irqchip_in_kernel(vcpu->kvm)) 244 return vcpu->arch.apic_base; 245 else 246 return vcpu->arch.apic_base; 247 } 248 EXPORT_SYMBOL_GPL(kvm_get_apic_base); 249 250 void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) 251 { 252 /* TODO: reserve bits check */ 253 if (irqchip_in_kernel(vcpu->kvm)) 254 kvm_lapic_set_base(vcpu, data); 255 else 256 vcpu->arch.apic_base = data; 257 } 258 EXPORT_SYMBOL_GPL(kvm_set_apic_base); 259 260 #define EXCPT_BENIGN 0 261 #define EXCPT_CONTRIBUTORY 1 262 #define EXCPT_PF 2 263 264 static int exception_class(int vector) 265 { 266 switch (vector) { 267 case PF_VECTOR: 268 return EXCPT_PF; 269 case DE_VECTOR: 270 case TS_VECTOR: 271 case NP_VECTOR: 272 case SS_VECTOR: 273 case GP_VECTOR: 274 return EXCPT_CONTRIBUTORY; 275 default: 276 break; 277 } 278 return EXCPT_BENIGN; 279 } 280 281 static void kvm_multiple_exception(struct kvm_vcpu *vcpu, 282 unsigned nr, bool has_error, u32 error_code, 283 bool reinject) 284 { 285 u32 prev_nr; 286 int class1, class2; 287 288 kvm_make_request(KVM_REQ_EVENT, vcpu); 289 290 if (!vcpu->arch.exception.pending) { 291 queue: 292 vcpu->arch.exception.pending = true; 293 vcpu->arch.exception.has_error_code = has_error; 294 vcpu->arch.exception.nr = nr; 295 vcpu->arch.exception.error_code = error_code; 296 vcpu->arch.exception.reinject = reinject; 297 return; 298 } 299 300 /* to check exception */ 301 prev_nr = vcpu->arch.exception.nr; 302 if (prev_nr == DF_VECTOR) { 303 /* triple fault -> shutdown */ 304 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 305 return; 306 } 307 class1 = exception_class(prev_nr); 308 class2 = exception_class(nr); 309 if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) 310 || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) { 311 /* generate double fault per SDM Table 5-5 */ 312 vcpu->arch.exception.pending = true; 313 vcpu->arch.exception.has_error_code = true; 314 vcpu->arch.exception.nr = DF_VECTOR; 315 vcpu->arch.exception.error_code = 0; 316 } else 317 /* replace previous exception with a new one in a hope 318 that instruction re-execution will regenerate lost 319 exception */ 320 goto queue; 321 } 322 323 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) 324 { 325 kvm_multiple_exception(vcpu, nr, false, 0, false); 326 } 327 EXPORT_SYMBOL_GPL(kvm_queue_exception); 328 329 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr) 330 { 331 kvm_multiple_exception(vcpu, nr, false, 0, true); 332 } 333 EXPORT_SYMBOL_GPL(kvm_requeue_exception); 334 335 void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) 336 { 337 if (err) 338 kvm_inject_gp(vcpu, 0); 339 else 340 kvm_x86_ops->skip_emulated_instruction(vcpu); 341 } 342 EXPORT_SYMBOL_GPL(kvm_complete_insn_gp); 343 344 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) 345 { 346 ++vcpu->stat.pf_guest; 347 vcpu->arch.cr2 = fault->address; 348 kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code); 349 } 350 EXPORT_SYMBOL_GPL(kvm_inject_page_fault); 351 352 void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) 353 { 354 if (mmu_is_nested(vcpu) && !fault->nested_page_fault) 355 vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault); 356 else 357 vcpu->arch.mmu.inject_page_fault(vcpu, fault); 358 } 359 360 void kvm_inject_nmi(struct kvm_vcpu *vcpu) 361 { 362 kvm_make_request(KVM_REQ_EVENT, vcpu); 363 vcpu->arch.nmi_pending = 1; 364 } 365 EXPORT_SYMBOL_GPL(kvm_inject_nmi); 366 367 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) 368 { 369 kvm_multiple_exception(vcpu, nr, true, error_code, false); 370 } 371 EXPORT_SYMBOL_GPL(kvm_queue_exception_e); 372 373 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) 374 { 375 kvm_multiple_exception(vcpu, nr, true, error_code, true); 376 } 377 EXPORT_SYMBOL_GPL(kvm_requeue_exception_e); 378 379 /* 380 * Checks if cpl <= required_cpl; if true, return true. Otherwise queue 381 * a #GP and return false. 382 */ 383 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl) 384 { 385 if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl) 386 return true; 387 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 388 return false; 389 } 390 EXPORT_SYMBOL_GPL(kvm_require_cpl); 391 392 /* 393 * This function will be used to read from the physical memory of the currently 394 * running guest. The difference to kvm_read_guest_page is that this function 395 * can read from guest physical or from the guest's guest physical memory. 396 */ 397 int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 398 gfn_t ngfn, void *data, int offset, int len, 399 u32 access) 400 { 401 gfn_t real_gfn; 402 gpa_t ngpa; 403 404 ngpa = gfn_to_gpa(ngfn); 405 real_gfn = mmu->translate_gpa(vcpu, ngpa, access); 406 if (real_gfn == UNMAPPED_GVA) 407 return -EFAULT; 408 409 real_gfn = gpa_to_gfn(real_gfn); 410 411 return kvm_read_guest_page(vcpu->kvm, real_gfn, data, offset, len); 412 } 413 EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu); 414 415 int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 416 void *data, int offset, int len, u32 access) 417 { 418 return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn, 419 data, offset, len, access); 420 } 421 422 /* 423 * Load the pae pdptrs. Return true is they are all valid. 424 */ 425 int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3) 426 { 427 gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; 428 unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; 429 int i; 430 int ret; 431 u64 pdpte[ARRAY_SIZE(mmu->pdptrs)]; 432 433 ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte, 434 offset * sizeof(u64), sizeof(pdpte), 435 PFERR_USER_MASK|PFERR_WRITE_MASK); 436 if (ret < 0) { 437 ret = 0; 438 goto out; 439 } 440 for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { 441 if (is_present_gpte(pdpte[i]) && 442 (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) { 443 ret = 0; 444 goto out; 445 } 446 } 447 ret = 1; 448 449 memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)); 450 __set_bit(VCPU_EXREG_PDPTR, 451 (unsigned long *)&vcpu->arch.regs_avail); 452 __set_bit(VCPU_EXREG_PDPTR, 453 (unsigned long *)&vcpu->arch.regs_dirty); 454 out: 455 456 return ret; 457 } 458 EXPORT_SYMBOL_GPL(load_pdptrs); 459 460 static bool pdptrs_changed(struct kvm_vcpu *vcpu) 461 { 462 u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)]; 463 bool changed = true; 464 int offset; 465 gfn_t gfn; 466 int r; 467 468 if (is_long_mode(vcpu) || !is_pae(vcpu)) 469 return false; 470 471 if (!test_bit(VCPU_EXREG_PDPTR, 472 (unsigned long *)&vcpu->arch.regs_avail)) 473 return true; 474 475 gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT; 476 offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1); 477 r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte), 478 PFERR_USER_MASK | PFERR_WRITE_MASK); 479 if (r < 0) 480 goto out; 481 changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0; 482 out: 483 484 return changed; 485 } 486 487 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 488 { 489 unsigned long old_cr0 = kvm_read_cr0(vcpu); 490 unsigned long update_bits = X86_CR0_PG | X86_CR0_WP | 491 X86_CR0_CD | X86_CR0_NW; 492 493 cr0 |= X86_CR0_ET; 494 495 #ifdef CONFIG_X86_64 496 if (cr0 & 0xffffffff00000000UL) 497 return 1; 498 #endif 499 500 cr0 &= ~CR0_RESERVED_BITS; 501 502 if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) 503 return 1; 504 505 if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) 506 return 1; 507 508 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { 509 #ifdef CONFIG_X86_64 510 if ((vcpu->arch.efer & EFER_LME)) { 511 int cs_db, cs_l; 512 513 if (!is_pae(vcpu)) 514 return 1; 515 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); 516 if (cs_l) 517 return 1; 518 } else 519 #endif 520 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, 521 kvm_read_cr3(vcpu))) 522 return 1; 523 } 524 525 kvm_x86_ops->set_cr0(vcpu, cr0); 526 527 if ((cr0 ^ old_cr0) & X86_CR0_PG) { 528 kvm_clear_async_pf_completion_queue(vcpu); 529 kvm_async_pf_hash_reset(vcpu); 530 } 531 532 if ((cr0 ^ old_cr0) & update_bits) 533 kvm_mmu_reset_context(vcpu); 534 return 0; 535 } 536 EXPORT_SYMBOL_GPL(kvm_set_cr0); 537 538 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) 539 { 540 (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); 541 } 542 EXPORT_SYMBOL_GPL(kvm_lmsw); 543 544 int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) 545 { 546 u64 xcr0; 547 548 /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now */ 549 if (index != XCR_XFEATURE_ENABLED_MASK) 550 return 1; 551 xcr0 = xcr; 552 if (kvm_x86_ops->get_cpl(vcpu) != 0) 553 return 1; 554 if (!(xcr0 & XSTATE_FP)) 555 return 1; 556 if ((xcr0 & XSTATE_YMM) && !(xcr0 & XSTATE_SSE)) 557 return 1; 558 if (xcr0 & ~host_xcr0) 559 return 1; 560 vcpu->arch.xcr0 = xcr0; 561 vcpu->guest_xcr0_loaded = 0; 562 return 0; 563 } 564 565 int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) 566 { 567 if (__kvm_set_xcr(vcpu, index, xcr)) { 568 kvm_inject_gp(vcpu, 0); 569 return 1; 570 } 571 return 0; 572 } 573 EXPORT_SYMBOL_GPL(kvm_set_xcr); 574 575 static bool guest_cpuid_has_xsave(struct kvm_vcpu *vcpu) 576 { 577 struct kvm_cpuid_entry2 *best; 578 579 best = kvm_find_cpuid_entry(vcpu, 1, 0); 580 return best && (best->ecx & bit(X86_FEATURE_XSAVE)); 581 } 582 583 static bool guest_cpuid_has_smep(struct kvm_vcpu *vcpu) 584 { 585 struct kvm_cpuid_entry2 *best; 586 587 best = kvm_find_cpuid_entry(vcpu, 7, 0); 588 return best && (best->ebx & bit(X86_FEATURE_SMEP)); 589 } 590 591 static bool guest_cpuid_has_fsgsbase(struct kvm_vcpu *vcpu) 592 { 593 struct kvm_cpuid_entry2 *best; 594 595 best = kvm_find_cpuid_entry(vcpu, 7, 0); 596 return best && (best->ebx & bit(X86_FEATURE_FSGSBASE)); 597 } 598 599 static void update_cpuid(struct kvm_vcpu *vcpu) 600 { 601 struct kvm_cpuid_entry2 *best; 602 603 best = kvm_find_cpuid_entry(vcpu, 1, 0); 604 if (!best) 605 return; 606 607 /* Update OSXSAVE bit */ 608 if (cpu_has_xsave && best->function == 0x1) { 609 best->ecx &= ~(bit(X86_FEATURE_OSXSAVE)); 610 if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) 611 best->ecx |= bit(X86_FEATURE_OSXSAVE); 612 } 613 } 614 615 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 616 { 617 unsigned long old_cr4 = kvm_read_cr4(vcpu); 618 unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | 619 X86_CR4_PAE | X86_CR4_SMEP; 620 if (cr4 & CR4_RESERVED_BITS) 621 return 1; 622 623 if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE)) 624 return 1; 625 626 if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP)) 627 return 1; 628 629 if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_RDWRGSFS)) 630 return 1; 631 632 if (is_long_mode(vcpu)) { 633 if (!(cr4 & X86_CR4_PAE)) 634 return 1; 635 } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) 636 && ((cr4 ^ old_cr4) & pdptr_bits) 637 && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, 638 kvm_read_cr3(vcpu))) 639 return 1; 640 641 if (kvm_x86_ops->set_cr4(vcpu, cr4)) 642 return 1; 643 644 if ((cr4 ^ old_cr4) & pdptr_bits) 645 kvm_mmu_reset_context(vcpu); 646 647 if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE) 648 update_cpuid(vcpu); 649 650 return 0; 651 } 652 EXPORT_SYMBOL_GPL(kvm_set_cr4); 653 654 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) 655 { 656 if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) { 657 kvm_mmu_sync_roots(vcpu); 658 kvm_mmu_flush_tlb(vcpu); 659 return 0; 660 } 661 662 if (is_long_mode(vcpu)) { 663 if (cr3 & CR3_L_MODE_RESERVED_BITS) 664 return 1; 665 } else { 666 if (is_pae(vcpu)) { 667 if (cr3 & CR3_PAE_RESERVED_BITS) 668 return 1; 669 if (is_paging(vcpu) && 670 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) 671 return 1; 672 } 673 /* 674 * We don't check reserved bits in nonpae mode, because 675 * this isn't enforced, and VMware depends on this. 676 */ 677 } 678 679 /* 680 * Does the new cr3 value map to physical memory? (Note, we 681 * catch an invalid cr3 even in real-mode, because it would 682 * cause trouble later on when we turn on paging anyway.) 683 * 684 * A real CPU would silently accept an invalid cr3 and would 685 * attempt to use it - with largely undefined (and often hard 686 * to debug) behavior on the guest side. 687 */ 688 if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) 689 return 1; 690 vcpu->arch.cr3 = cr3; 691 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 692 vcpu->arch.mmu.new_cr3(vcpu); 693 return 0; 694 } 695 EXPORT_SYMBOL_GPL(kvm_set_cr3); 696 697 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) 698 { 699 if (cr8 & CR8_RESERVED_BITS) 700 return 1; 701 if (irqchip_in_kernel(vcpu->kvm)) 702 kvm_lapic_set_tpr(vcpu, cr8); 703 else 704 vcpu->arch.cr8 = cr8; 705 return 0; 706 } 707 EXPORT_SYMBOL_GPL(kvm_set_cr8); 708 709 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) 710 { 711 if (irqchip_in_kernel(vcpu->kvm)) 712 return kvm_lapic_get_cr8(vcpu); 713 else 714 return vcpu->arch.cr8; 715 } 716 EXPORT_SYMBOL_GPL(kvm_get_cr8); 717 718 static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) 719 { 720 switch (dr) { 721 case 0 ... 3: 722 vcpu->arch.db[dr] = val; 723 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) 724 vcpu->arch.eff_db[dr] = val; 725 break; 726 case 4: 727 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) 728 return 1; /* #UD */ 729 /* fall through */ 730 case 6: 731 if (val & 0xffffffff00000000ULL) 732 return -1; /* #GP */ 733 vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1; 734 break; 735 case 5: 736 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) 737 return 1; /* #UD */ 738 /* fall through */ 739 default: /* 7 */ 740 if (val & 0xffffffff00000000ULL) 741 return -1; /* #GP */ 742 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; 743 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { 744 kvm_x86_ops->set_dr7(vcpu, vcpu->arch.dr7); 745 vcpu->arch.switch_db_regs = (val & DR7_BP_EN_MASK); 746 } 747 break; 748 } 749 750 return 0; 751 } 752 753 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) 754 { 755 int res; 756 757 res = __kvm_set_dr(vcpu, dr, val); 758 if (res > 0) 759 kvm_queue_exception(vcpu, UD_VECTOR); 760 else if (res < 0) 761 kvm_inject_gp(vcpu, 0); 762 763 return res; 764 } 765 EXPORT_SYMBOL_GPL(kvm_set_dr); 766 767 static int _kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val) 768 { 769 switch (dr) { 770 case 0 ... 3: 771 *val = vcpu->arch.db[dr]; 772 break; 773 case 4: 774 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) 775 return 1; 776 /* fall through */ 777 case 6: 778 *val = vcpu->arch.dr6; 779 break; 780 case 5: 781 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) 782 return 1; 783 /* fall through */ 784 default: /* 7 */ 785 *val = vcpu->arch.dr7; 786 break; 787 } 788 789 return 0; 790 } 791 792 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val) 793 { 794 if (_kvm_get_dr(vcpu, dr, val)) { 795 kvm_queue_exception(vcpu, UD_VECTOR); 796 return 1; 797 } 798 return 0; 799 } 800 EXPORT_SYMBOL_GPL(kvm_get_dr); 801 802 /* 803 * List of msr numbers which we expose to userspace through KVM_GET_MSRS 804 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. 805 * 806 * This list is modified at module load time to reflect the 807 * capabilities of the host cpu. This capabilities test skips MSRs that are 808 * kvm-specific. Those are put in the beginning of the list. 809 */ 810 811 #define KVM_SAVE_MSRS_BEGIN 9 812 static u32 msrs_to_save[] = { 813 MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, 814 MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW, 815 HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL, 816 HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, 817 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, 818 MSR_STAR, 819 #ifdef CONFIG_X86_64 820 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, 821 #endif 822 MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA 823 }; 824 825 static unsigned num_msrs_to_save; 826 827 static u32 emulated_msrs[] = { 828 MSR_IA32_MISC_ENABLE, 829 MSR_IA32_MCG_STATUS, 830 MSR_IA32_MCG_CTL, 831 }; 832 833 static int set_efer(struct kvm_vcpu *vcpu, u64 efer) 834 { 835 u64 old_efer = vcpu->arch.efer; 836 837 if (efer & efer_reserved_bits) 838 return 1; 839 840 if (is_paging(vcpu) 841 && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME)) 842 return 1; 843 844 if (efer & EFER_FFXSR) { 845 struct kvm_cpuid_entry2 *feat; 846 847 feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); 848 if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT))) 849 return 1; 850 } 851 852 if (efer & EFER_SVME) { 853 struct kvm_cpuid_entry2 *feat; 854 855 feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); 856 if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM))) 857 return 1; 858 } 859 860 efer &= ~EFER_LMA; 861 efer |= vcpu->arch.efer & EFER_LMA; 862 863 kvm_x86_ops->set_efer(vcpu, efer); 864 865 vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled; 866 867 /* Update reserved bits */ 868 if ((efer ^ old_efer) & EFER_NX) 869 kvm_mmu_reset_context(vcpu); 870 871 return 0; 872 } 873 874 void kvm_enable_efer_bits(u64 mask) 875 { 876 efer_reserved_bits &= ~mask; 877 } 878 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits); 879 880 881 /* 882 * Writes msr value into into the appropriate "register". 883 * Returns 0 on success, non-0 otherwise. 884 * Assumes vcpu_load() was already called. 885 */ 886 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 887 { 888 return kvm_x86_ops->set_msr(vcpu, msr_index, data); 889 } 890 891 /* 892 * Adapt set_msr() to msr_io()'s calling convention 893 */ 894 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) 895 { 896 return kvm_set_msr(vcpu, index, *data); 897 } 898 899 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock) 900 { 901 int version; 902 int r; 903 struct pvclock_wall_clock wc; 904 struct timespec boot; 905 906 if (!wall_clock) 907 return; 908 909 r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version)); 910 if (r) 911 return; 912 913 if (version & 1) 914 ++version; /* first time write, random junk */ 915 916 ++version; 917 918 kvm_write_guest(kvm, wall_clock, &version, sizeof(version)); 919 920 /* 921 * The guest calculates current wall clock time by adding 922 * system time (updated by kvm_guest_time_update below) to the 923 * wall clock specified here. guest system time equals host 924 * system time for us, thus we must fill in host boot time here. 925 */ 926 getboottime(&boot); 927 928 wc.sec = boot.tv_sec; 929 wc.nsec = boot.tv_nsec; 930 wc.version = version; 931 932 kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc)); 933 934 version++; 935 kvm_write_guest(kvm, wall_clock, &version, sizeof(version)); 936 } 937 938 static uint32_t div_frac(uint32_t dividend, uint32_t divisor) 939 { 940 uint32_t quotient, remainder; 941 942 /* Don't try to replace with do_div(), this one calculates 943 * "(dividend << 32) / divisor" */ 944 __asm__ ( "divl %4" 945 : "=a" (quotient), "=d" (remainder) 946 : "0" (0), "1" (dividend), "r" (divisor) ); 947 return quotient; 948 } 949 950 static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz, 951 s8 *pshift, u32 *pmultiplier) 952 { 953 uint64_t scaled64; 954 int32_t shift = 0; 955 uint64_t tps64; 956 uint32_t tps32; 957 958 tps64 = base_khz * 1000LL; 959 scaled64 = scaled_khz * 1000LL; 960 while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) { 961 tps64 >>= 1; 962 shift--; 963 } 964 965 tps32 = (uint32_t)tps64; 966 while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) { 967 if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000) 968 scaled64 >>= 1; 969 else 970 tps32 <<= 1; 971 shift++; 972 } 973 974 *pshift = shift; 975 *pmultiplier = div_frac(scaled64, tps32); 976 977 pr_debug("%s: base_khz %u => %u, shift %d, mul %u\n", 978 __func__, base_khz, scaled_khz, shift, *pmultiplier); 979 } 980 981 static inline u64 get_kernel_ns(void) 982 { 983 struct timespec ts; 984 985 WARN_ON(preemptible()); 986 ktime_get_ts(&ts); 987 monotonic_to_bootbased(&ts); 988 return timespec_to_ns(&ts); 989 } 990 991 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz); 992 unsigned long max_tsc_khz; 993 994 static inline int kvm_tsc_changes_freq(void) 995 { 996 int cpu = get_cpu(); 997 int ret = !boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && 998 cpufreq_quick_get(cpu) != 0; 999 put_cpu(); 1000 return ret; 1001 } 1002 1003 static u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu) 1004 { 1005 if (vcpu->arch.virtual_tsc_khz) 1006 return vcpu->arch.virtual_tsc_khz; 1007 else 1008 return __this_cpu_read(cpu_tsc_khz); 1009 } 1010 1011 static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec) 1012 { 1013 u64 ret; 1014 1015 WARN_ON(preemptible()); 1016 if (kvm_tsc_changes_freq()) 1017 printk_once(KERN_WARNING 1018 "kvm: unreliable cycle conversion on adjustable rate TSC\n"); 1019 ret = nsec * vcpu_tsc_khz(vcpu); 1020 do_div(ret, USEC_PER_SEC); 1021 return ret; 1022 } 1023 1024 static void kvm_init_tsc_catchup(struct kvm_vcpu *vcpu, u32 this_tsc_khz) 1025 { 1026 /* Compute a scale to convert nanoseconds in TSC cycles */ 1027 kvm_get_time_scale(this_tsc_khz, NSEC_PER_SEC / 1000, 1028 &vcpu->arch.tsc_catchup_shift, 1029 &vcpu->arch.tsc_catchup_mult); 1030 } 1031 1032 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) 1033 { 1034 u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.last_tsc_nsec, 1035 vcpu->arch.tsc_catchup_mult, 1036 vcpu->arch.tsc_catchup_shift); 1037 tsc += vcpu->arch.last_tsc_write; 1038 return tsc; 1039 } 1040 1041 void kvm_write_tsc(struct kvm_vcpu *vcpu, u64 data) 1042 { 1043 struct kvm *kvm = vcpu->kvm; 1044 u64 offset, ns, elapsed; 1045 unsigned long flags; 1046 s64 sdiff; 1047 1048 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 1049 offset = kvm_x86_ops->compute_tsc_offset(vcpu, data); 1050 ns = get_kernel_ns(); 1051 elapsed = ns - kvm->arch.last_tsc_nsec; 1052 sdiff = data - kvm->arch.last_tsc_write; 1053 if (sdiff < 0) 1054 sdiff = -sdiff; 1055 1056 /* 1057 * Special case: close write to TSC within 5 seconds of 1058 * another CPU is interpreted as an attempt to synchronize 1059 * The 5 seconds is to accommodate host load / swapping as 1060 * well as any reset of TSC during the boot process. 1061 * 1062 * In that case, for a reliable TSC, we can match TSC offsets, 1063 * or make a best guest using elapsed value. 1064 */ 1065 if (sdiff < nsec_to_cycles(vcpu, 5ULL * NSEC_PER_SEC) && 1066 elapsed < 5ULL * NSEC_PER_SEC) { 1067 if (!check_tsc_unstable()) { 1068 offset = kvm->arch.last_tsc_offset; 1069 pr_debug("kvm: matched tsc offset for %llu\n", data); 1070 } else { 1071 u64 delta = nsec_to_cycles(vcpu, elapsed); 1072 offset += delta; 1073 pr_debug("kvm: adjusted tsc offset by %llu\n", delta); 1074 } 1075 ns = kvm->arch.last_tsc_nsec; 1076 } 1077 kvm->arch.last_tsc_nsec = ns; 1078 kvm->arch.last_tsc_write = data; 1079 kvm->arch.last_tsc_offset = offset; 1080 kvm_x86_ops->write_tsc_offset(vcpu, offset); 1081 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 1082 1083 /* Reset of TSC must disable overshoot protection below */ 1084 vcpu->arch.hv_clock.tsc_timestamp = 0; 1085 vcpu->arch.last_tsc_write = data; 1086 vcpu->arch.last_tsc_nsec = ns; 1087 } 1088 EXPORT_SYMBOL_GPL(kvm_write_tsc); 1089 1090 static int kvm_guest_time_update(struct kvm_vcpu *v) 1091 { 1092 unsigned long flags; 1093 struct kvm_vcpu_arch *vcpu = &v->arch; 1094 void *shared_kaddr; 1095 unsigned long this_tsc_khz; 1096 s64 kernel_ns, max_kernel_ns; 1097 u64 tsc_timestamp; 1098 1099 /* Keep irq disabled to prevent changes to the clock */ 1100 local_irq_save(flags); 1101 kvm_get_msr(v, MSR_IA32_TSC, &tsc_timestamp); 1102 kernel_ns = get_kernel_ns(); 1103 this_tsc_khz = vcpu_tsc_khz(v); 1104 if (unlikely(this_tsc_khz == 0)) { 1105 local_irq_restore(flags); 1106 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); 1107 return 1; 1108 } 1109 1110 /* 1111 * We may have to catch up the TSC to match elapsed wall clock 1112 * time for two reasons, even if kvmclock is used. 1113 * 1) CPU could have been running below the maximum TSC rate 1114 * 2) Broken TSC compensation resets the base at each VCPU 1115 * entry to avoid unknown leaps of TSC even when running 1116 * again on the same CPU. This may cause apparent elapsed 1117 * time to disappear, and the guest to stand still or run 1118 * very slowly. 1119 */ 1120 if (vcpu->tsc_catchup) { 1121 u64 tsc = compute_guest_tsc(v, kernel_ns); 1122 if (tsc > tsc_timestamp) { 1123 kvm_x86_ops->adjust_tsc_offset(v, tsc - tsc_timestamp); 1124 tsc_timestamp = tsc; 1125 } 1126 } 1127 1128 local_irq_restore(flags); 1129 1130 if (!vcpu->time_page) 1131 return 0; 1132 1133 /* 1134 * Time as measured by the TSC may go backwards when resetting the base 1135 * tsc_timestamp. The reason for this is that the TSC resolution is 1136 * higher than the resolution of the other clock scales. Thus, many 1137 * possible measurments of the TSC correspond to one measurement of any 1138 * other clock, and so a spread of values is possible. This is not a 1139 * problem for the computation of the nanosecond clock; with TSC rates 1140 * around 1GHZ, there can only be a few cycles which correspond to one 1141 * nanosecond value, and any path through this code will inevitably 1142 * take longer than that. However, with the kernel_ns value itself, 1143 * the precision may be much lower, down to HZ granularity. If the 1144 * first sampling of TSC against kernel_ns ends in the low part of the 1145 * range, and the second in the high end of the range, we can get: 1146 * 1147 * (TSC - offset_low) * S + kns_old > (TSC - offset_high) * S + kns_new 1148 * 1149 * As the sampling errors potentially range in the thousands of cycles, 1150 * it is possible such a time value has already been observed by the 1151 * guest. To protect against this, we must compute the system time as 1152 * observed by the guest and ensure the new system time is greater. 1153 */ 1154 max_kernel_ns = 0; 1155 if (vcpu->hv_clock.tsc_timestamp && vcpu->last_guest_tsc) { 1156 max_kernel_ns = vcpu->last_guest_tsc - 1157 vcpu->hv_clock.tsc_timestamp; 1158 max_kernel_ns = pvclock_scale_delta(max_kernel_ns, 1159 vcpu->hv_clock.tsc_to_system_mul, 1160 vcpu->hv_clock.tsc_shift); 1161 max_kernel_ns += vcpu->last_kernel_ns; 1162 } 1163 1164 if (unlikely(vcpu->hw_tsc_khz != this_tsc_khz)) { 1165 kvm_get_time_scale(NSEC_PER_SEC / 1000, this_tsc_khz, 1166 &vcpu->hv_clock.tsc_shift, 1167 &vcpu->hv_clock.tsc_to_system_mul); 1168 vcpu->hw_tsc_khz = this_tsc_khz; 1169 } 1170 1171 if (max_kernel_ns > kernel_ns) 1172 kernel_ns = max_kernel_ns; 1173 1174 /* With all the info we got, fill in the values */ 1175 vcpu->hv_clock.tsc_timestamp = tsc_timestamp; 1176 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; 1177 vcpu->last_kernel_ns = kernel_ns; 1178 vcpu->last_guest_tsc = tsc_timestamp; 1179 vcpu->hv_clock.flags = 0; 1180 1181 /* 1182 * The interface expects us to write an even number signaling that the 1183 * update is finished. Since the guest won't see the intermediate 1184 * state, we just increase by 2 at the end. 1185 */ 1186 vcpu->hv_clock.version += 2; 1187 1188 shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0); 1189 1190 memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock, 1191 sizeof(vcpu->hv_clock)); 1192 1193 kunmap_atomic(shared_kaddr, KM_USER0); 1194 1195 mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT); 1196 return 0; 1197 } 1198 1199 static bool msr_mtrr_valid(unsigned msr) 1200 { 1201 switch (msr) { 1202 case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1: 1203 case MSR_MTRRfix64K_00000: 1204 case MSR_MTRRfix16K_80000: 1205 case MSR_MTRRfix16K_A0000: 1206 case MSR_MTRRfix4K_C0000: 1207 case MSR_MTRRfix4K_C8000: 1208 case MSR_MTRRfix4K_D0000: 1209 case MSR_MTRRfix4K_D8000: 1210 case MSR_MTRRfix4K_E0000: 1211 case MSR_MTRRfix4K_E8000: 1212 case MSR_MTRRfix4K_F0000: 1213 case MSR_MTRRfix4K_F8000: 1214 case MSR_MTRRdefType: 1215 case MSR_IA32_CR_PAT: 1216 return true; 1217 case 0x2f8: 1218 return true; 1219 } 1220 return false; 1221 } 1222 1223 static bool valid_pat_type(unsigned t) 1224 { 1225 return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */ 1226 } 1227 1228 static bool valid_mtrr_type(unsigned t) 1229 { 1230 return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */ 1231 } 1232 1233 static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1234 { 1235 int i; 1236 1237 if (!msr_mtrr_valid(msr)) 1238 return false; 1239 1240 if (msr == MSR_IA32_CR_PAT) { 1241 for (i = 0; i < 8; i++) 1242 if (!valid_pat_type((data >> (i * 8)) & 0xff)) 1243 return false; 1244 return true; 1245 } else if (msr == MSR_MTRRdefType) { 1246 if (data & ~0xcff) 1247 return false; 1248 return valid_mtrr_type(data & 0xff); 1249 } else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) { 1250 for (i = 0; i < 8 ; i++) 1251 if (!valid_mtrr_type((data >> (i * 8)) & 0xff)) 1252 return false; 1253 return true; 1254 } 1255 1256 /* variable MTRRs */ 1257 return valid_mtrr_type(data & 0xff); 1258 } 1259 1260 static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1261 { 1262 u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges; 1263 1264 if (!mtrr_valid(vcpu, msr, data)) 1265 return 1; 1266 1267 if (msr == MSR_MTRRdefType) { 1268 vcpu->arch.mtrr_state.def_type = data; 1269 vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10; 1270 } else if (msr == MSR_MTRRfix64K_00000) 1271 p[0] = data; 1272 else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000) 1273 p[1 + msr - MSR_MTRRfix16K_80000] = data; 1274 else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000) 1275 p[3 + msr - MSR_MTRRfix4K_C0000] = data; 1276 else if (msr == MSR_IA32_CR_PAT) 1277 vcpu->arch.pat = data; 1278 else { /* Variable MTRRs */ 1279 int idx, is_mtrr_mask; 1280 u64 *pt; 1281 1282 idx = (msr - 0x200) / 2; 1283 is_mtrr_mask = msr - 0x200 - 2 * idx; 1284 if (!is_mtrr_mask) 1285 pt = 1286 (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo; 1287 else 1288 pt = 1289 (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo; 1290 *pt = data; 1291 } 1292 1293 kvm_mmu_reset_context(vcpu); 1294 return 0; 1295 } 1296 1297 static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1298 { 1299 u64 mcg_cap = vcpu->arch.mcg_cap; 1300 unsigned bank_num = mcg_cap & 0xff; 1301 1302 switch (msr) { 1303 case MSR_IA32_MCG_STATUS: 1304 vcpu->arch.mcg_status = data; 1305 break; 1306 case MSR_IA32_MCG_CTL: 1307 if (!(mcg_cap & MCG_CTL_P)) 1308 return 1; 1309 if (data != 0 && data != ~(u64)0) 1310 return -1; 1311 vcpu->arch.mcg_ctl = data; 1312 break; 1313 default: 1314 if (msr >= MSR_IA32_MC0_CTL && 1315 msr < MSR_IA32_MC0_CTL + 4 * bank_num) { 1316 u32 offset = msr - MSR_IA32_MC0_CTL; 1317 /* only 0 or all 1s can be written to IA32_MCi_CTL 1318 * some Linux kernels though clear bit 10 in bank 4 to 1319 * workaround a BIOS/GART TBL issue on AMD K8s, ignore 1320 * this to avoid an uncatched #GP in the guest 1321 */ 1322 if ((offset & 0x3) == 0 && 1323 data != 0 && (data | (1 << 10)) != ~(u64)0) 1324 return -1; 1325 vcpu->arch.mce_banks[offset] = data; 1326 break; 1327 } 1328 return 1; 1329 } 1330 return 0; 1331 } 1332 1333 static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data) 1334 { 1335 struct kvm *kvm = vcpu->kvm; 1336 int lm = is_long_mode(vcpu); 1337 u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64 1338 : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32; 1339 u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64 1340 : kvm->arch.xen_hvm_config.blob_size_32; 1341 u32 page_num = data & ~PAGE_MASK; 1342 u64 page_addr = data & PAGE_MASK; 1343 u8 *page; 1344 int r; 1345 1346 r = -E2BIG; 1347 if (page_num >= blob_size) 1348 goto out; 1349 r = -ENOMEM; 1350 page = kzalloc(PAGE_SIZE, GFP_KERNEL); 1351 if (!page) 1352 goto out; 1353 r = -EFAULT; 1354 if (copy_from_user(page, blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE)) 1355 goto out_free; 1356 if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE)) 1357 goto out_free; 1358 r = 0; 1359 out_free: 1360 kfree(page); 1361 out: 1362 return r; 1363 } 1364 1365 static bool kvm_hv_hypercall_enabled(struct kvm *kvm) 1366 { 1367 return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE; 1368 } 1369 1370 static bool kvm_hv_msr_partition_wide(u32 msr) 1371 { 1372 bool r = false; 1373 switch (msr) { 1374 case HV_X64_MSR_GUEST_OS_ID: 1375 case HV_X64_MSR_HYPERCALL: 1376 r = true; 1377 break; 1378 } 1379 1380 return r; 1381 } 1382 1383 static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1384 { 1385 struct kvm *kvm = vcpu->kvm; 1386 1387 switch (msr) { 1388 case HV_X64_MSR_GUEST_OS_ID: 1389 kvm->arch.hv_guest_os_id = data; 1390 /* setting guest os id to zero disables hypercall page */ 1391 if (!kvm->arch.hv_guest_os_id) 1392 kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE; 1393 break; 1394 case HV_X64_MSR_HYPERCALL: { 1395 u64 gfn; 1396 unsigned long addr; 1397 u8 instructions[4]; 1398 1399 /* if guest os id is not set hypercall should remain disabled */ 1400 if (!kvm->arch.hv_guest_os_id) 1401 break; 1402 if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) { 1403 kvm->arch.hv_hypercall = data; 1404 break; 1405 } 1406 gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT; 1407 addr = gfn_to_hva(kvm, gfn); 1408 if (kvm_is_error_hva(addr)) 1409 return 1; 1410 kvm_x86_ops->patch_hypercall(vcpu, instructions); 1411 ((unsigned char *)instructions)[3] = 0xc3; /* ret */ 1412 if (__copy_to_user((void __user *)addr, instructions, 4)) 1413 return 1; 1414 kvm->arch.hv_hypercall = data; 1415 break; 1416 } 1417 default: 1418 pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x " 1419 "data 0x%llx\n", msr, data); 1420 return 1; 1421 } 1422 return 0; 1423 } 1424 1425 static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1426 { 1427 switch (msr) { 1428 case HV_X64_MSR_APIC_ASSIST_PAGE: { 1429 unsigned long addr; 1430 1431 if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) { 1432 vcpu->arch.hv_vapic = data; 1433 break; 1434 } 1435 addr = gfn_to_hva(vcpu->kvm, data >> 1436 HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT); 1437 if (kvm_is_error_hva(addr)) 1438 return 1; 1439 if (__clear_user((void __user *)addr, PAGE_SIZE)) 1440 return 1; 1441 vcpu->arch.hv_vapic = data; 1442 break; 1443 } 1444 case HV_X64_MSR_EOI: 1445 return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data); 1446 case HV_X64_MSR_ICR: 1447 return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data); 1448 case HV_X64_MSR_TPR: 1449 return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data); 1450 default: 1451 pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x " 1452 "data 0x%llx\n", msr, data); 1453 return 1; 1454 } 1455 1456 return 0; 1457 } 1458 1459 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) 1460 { 1461 gpa_t gpa = data & ~0x3f; 1462 1463 /* Bits 2:5 are resrved, Should be zero */ 1464 if (data & 0x3c) 1465 return 1; 1466 1467 vcpu->arch.apf.msr_val = data; 1468 1469 if (!(data & KVM_ASYNC_PF_ENABLED)) { 1470 kvm_clear_async_pf_completion_queue(vcpu); 1471 kvm_async_pf_hash_reset(vcpu); 1472 return 0; 1473 } 1474 1475 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa)) 1476 return 1; 1477 1478 vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS); 1479 kvm_async_pf_wakeup_all(vcpu); 1480 return 0; 1481 } 1482 1483 static void kvmclock_reset(struct kvm_vcpu *vcpu) 1484 { 1485 if (vcpu->arch.time_page) { 1486 kvm_release_page_dirty(vcpu->arch.time_page); 1487 vcpu->arch.time_page = NULL; 1488 } 1489 } 1490 1491 static void accumulate_steal_time(struct kvm_vcpu *vcpu) 1492 { 1493 u64 delta; 1494 1495 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 1496 return; 1497 1498 delta = current->sched_info.run_delay - vcpu->arch.st.last_steal; 1499 vcpu->arch.st.last_steal = current->sched_info.run_delay; 1500 vcpu->arch.st.accum_steal = delta; 1501 } 1502 1503 static void record_steal_time(struct kvm_vcpu *vcpu) 1504 { 1505 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 1506 return; 1507 1508 if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime, 1509 &vcpu->arch.st.steal, sizeof(struct kvm_steal_time)))) 1510 return; 1511 1512 vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal; 1513 vcpu->arch.st.steal.version += 2; 1514 vcpu->arch.st.accum_steal = 0; 1515 1516 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime, 1517 &vcpu->arch.st.steal, sizeof(struct kvm_steal_time)); 1518 } 1519 1520 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1521 { 1522 switch (msr) { 1523 case MSR_EFER: 1524 return set_efer(vcpu, data); 1525 case MSR_K7_HWCR: 1526 data &= ~(u64)0x40; /* ignore flush filter disable */ 1527 data &= ~(u64)0x100; /* ignore ignne emulation enable */ 1528 if (data != 0) { 1529 pr_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n", 1530 data); 1531 return 1; 1532 } 1533 break; 1534 case MSR_FAM10H_MMIO_CONF_BASE: 1535 if (data != 0) { 1536 pr_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: " 1537 "0x%llx\n", data); 1538 return 1; 1539 } 1540 break; 1541 case MSR_AMD64_NB_CFG: 1542 break; 1543 case MSR_IA32_DEBUGCTLMSR: 1544 if (!data) { 1545 /* We support the non-activated case already */ 1546 break; 1547 } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) { 1548 /* Values other than LBR and BTF are vendor-specific, 1549 thus reserved and should throw a #GP */ 1550 return 1; 1551 } 1552 pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n", 1553 __func__, data); 1554 break; 1555 case MSR_IA32_UCODE_REV: 1556 case MSR_IA32_UCODE_WRITE: 1557 case MSR_VM_HSAVE_PA: 1558 case MSR_AMD64_PATCH_LOADER: 1559 break; 1560 case 0x200 ... 0x2ff: 1561 return set_msr_mtrr(vcpu, msr, data); 1562 case MSR_IA32_APICBASE: 1563 kvm_set_apic_base(vcpu, data); 1564 break; 1565 case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff: 1566 return kvm_x2apic_msr_write(vcpu, msr, data); 1567 case MSR_IA32_MISC_ENABLE: 1568 vcpu->arch.ia32_misc_enable_msr = data; 1569 break; 1570 case MSR_KVM_WALL_CLOCK_NEW: 1571 case MSR_KVM_WALL_CLOCK: 1572 vcpu->kvm->arch.wall_clock = data; 1573 kvm_write_wall_clock(vcpu->kvm, data); 1574 break; 1575 case MSR_KVM_SYSTEM_TIME_NEW: 1576 case MSR_KVM_SYSTEM_TIME: { 1577 kvmclock_reset(vcpu); 1578 1579 vcpu->arch.time = data; 1580 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 1581 1582 /* we verify if the enable bit is set... */ 1583 if (!(data & 1)) 1584 break; 1585 1586 /* ...but clean it before doing the actual write */ 1587 vcpu->arch.time_offset = data & ~(PAGE_MASK | 1); 1588 1589 vcpu->arch.time_page = 1590 gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT); 1591 1592 if (is_error_page(vcpu->arch.time_page)) { 1593 kvm_release_page_clean(vcpu->arch.time_page); 1594 vcpu->arch.time_page = NULL; 1595 } 1596 break; 1597 } 1598 case MSR_KVM_ASYNC_PF_EN: 1599 if (kvm_pv_enable_async_pf(vcpu, data)) 1600 return 1; 1601 break; 1602 case MSR_KVM_STEAL_TIME: 1603 1604 if (unlikely(!sched_info_on())) 1605 return 1; 1606 1607 if (data & KVM_STEAL_RESERVED_MASK) 1608 return 1; 1609 1610 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime, 1611 data & KVM_STEAL_VALID_BITS)) 1612 return 1; 1613 1614 vcpu->arch.st.msr_val = data; 1615 1616 if (!(data & KVM_MSR_ENABLED)) 1617 break; 1618 1619 vcpu->arch.st.last_steal = current->sched_info.run_delay; 1620 1621 preempt_disable(); 1622 accumulate_steal_time(vcpu); 1623 preempt_enable(); 1624 1625 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 1626 1627 break; 1628 1629 case MSR_IA32_MCG_CTL: 1630 case MSR_IA32_MCG_STATUS: 1631 case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1: 1632 return set_msr_mce(vcpu, msr, data); 1633 1634 /* Performance counters are not protected by a CPUID bit, 1635 * so we should check all of them in the generic path for the sake of 1636 * cross vendor migration. 1637 * Writing a zero into the event select MSRs disables them, 1638 * which we perfectly emulate ;-). Any other value should be at least 1639 * reported, some guests depend on them. 1640 */ 1641 case MSR_P6_EVNTSEL0: 1642 case MSR_P6_EVNTSEL1: 1643 case MSR_K7_EVNTSEL0: 1644 case MSR_K7_EVNTSEL1: 1645 case MSR_K7_EVNTSEL2: 1646 case MSR_K7_EVNTSEL3: 1647 if (data != 0) 1648 pr_unimpl(vcpu, "unimplemented perfctr wrmsr: " 1649 "0x%x data 0x%llx\n", msr, data); 1650 break; 1651 /* at least RHEL 4 unconditionally writes to the perfctr registers, 1652 * so we ignore writes to make it happy. 1653 */ 1654 case MSR_P6_PERFCTR0: 1655 case MSR_P6_PERFCTR1: 1656 case MSR_K7_PERFCTR0: 1657 case MSR_K7_PERFCTR1: 1658 case MSR_K7_PERFCTR2: 1659 case MSR_K7_PERFCTR3: 1660 pr_unimpl(vcpu, "unimplemented perfctr wrmsr: " 1661 "0x%x data 0x%llx\n", msr, data); 1662 break; 1663 case MSR_K7_CLK_CTL: 1664 /* 1665 * Ignore all writes to this no longer documented MSR. 1666 * Writes are only relevant for old K7 processors, 1667 * all pre-dating SVM, but a recommended workaround from 1668 * AMD for these chips. It is possible to speicify the 1669 * affected processor models on the command line, hence 1670 * the need to ignore the workaround. 1671 */ 1672 break; 1673 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: 1674 if (kvm_hv_msr_partition_wide(msr)) { 1675 int r; 1676 mutex_lock(&vcpu->kvm->lock); 1677 r = set_msr_hyperv_pw(vcpu, msr, data); 1678 mutex_unlock(&vcpu->kvm->lock); 1679 return r; 1680 } else 1681 return set_msr_hyperv(vcpu, msr, data); 1682 break; 1683 case MSR_IA32_BBL_CR_CTL3: 1684 /* Drop writes to this legacy MSR -- see rdmsr 1685 * counterpart for further detail. 1686 */ 1687 pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data); 1688 break; 1689 default: 1690 if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr)) 1691 return xen_hvm_config(vcpu, data); 1692 if (!ignore_msrs) { 1693 pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", 1694 msr, data); 1695 return 1; 1696 } else { 1697 pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", 1698 msr, data); 1699 break; 1700 } 1701 } 1702 return 0; 1703 } 1704 EXPORT_SYMBOL_GPL(kvm_set_msr_common); 1705 1706 1707 /* 1708 * Reads an msr value (of 'msr_index') into 'pdata'. 1709 * Returns 0 on success, non-0 otherwise. 1710 * Assumes vcpu_load() was already called. 1711 */ 1712 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) 1713 { 1714 return kvm_x86_ops->get_msr(vcpu, msr_index, pdata); 1715 } 1716 1717 static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) 1718 { 1719 u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges; 1720 1721 if (!msr_mtrr_valid(msr)) 1722 return 1; 1723 1724 if (msr == MSR_MTRRdefType) 1725 *pdata = vcpu->arch.mtrr_state.def_type + 1726 (vcpu->arch.mtrr_state.enabled << 10); 1727 else if (msr == MSR_MTRRfix64K_00000) 1728 *pdata = p[0]; 1729 else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000) 1730 *pdata = p[1 + msr - MSR_MTRRfix16K_80000]; 1731 else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000) 1732 *pdata = p[3 + msr - MSR_MTRRfix4K_C0000]; 1733 else if (msr == MSR_IA32_CR_PAT) 1734 *pdata = vcpu->arch.pat; 1735 else { /* Variable MTRRs */ 1736 int idx, is_mtrr_mask; 1737 u64 *pt; 1738 1739 idx = (msr - 0x200) / 2; 1740 is_mtrr_mask = msr - 0x200 - 2 * idx; 1741 if (!is_mtrr_mask) 1742 pt = 1743 (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo; 1744 else 1745 pt = 1746 (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo; 1747 *pdata = *pt; 1748 } 1749 1750 return 0; 1751 } 1752 1753 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) 1754 { 1755 u64 data; 1756 u64 mcg_cap = vcpu->arch.mcg_cap; 1757 unsigned bank_num = mcg_cap & 0xff; 1758 1759 switch (msr) { 1760 case MSR_IA32_P5_MC_ADDR: 1761 case MSR_IA32_P5_MC_TYPE: 1762 data = 0; 1763 break; 1764 case MSR_IA32_MCG_CAP: 1765 data = vcpu->arch.mcg_cap; 1766 break; 1767 case MSR_IA32_MCG_CTL: 1768 if (!(mcg_cap & MCG_CTL_P)) 1769 return 1; 1770 data = vcpu->arch.mcg_ctl; 1771 break; 1772 case MSR_IA32_MCG_STATUS: 1773 data = vcpu->arch.mcg_status; 1774 break; 1775 default: 1776 if (msr >= MSR_IA32_MC0_CTL && 1777 msr < MSR_IA32_MC0_CTL + 4 * bank_num) { 1778 u32 offset = msr - MSR_IA32_MC0_CTL; 1779 data = vcpu->arch.mce_banks[offset]; 1780 break; 1781 } 1782 return 1; 1783 } 1784 *pdata = data; 1785 return 0; 1786 } 1787 1788 static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) 1789 { 1790 u64 data = 0; 1791 struct kvm *kvm = vcpu->kvm; 1792 1793 switch (msr) { 1794 case HV_X64_MSR_GUEST_OS_ID: 1795 data = kvm->arch.hv_guest_os_id; 1796 break; 1797 case HV_X64_MSR_HYPERCALL: 1798 data = kvm->arch.hv_hypercall; 1799 break; 1800 default: 1801 pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr); 1802 return 1; 1803 } 1804 1805 *pdata = data; 1806 return 0; 1807 } 1808 1809 static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) 1810 { 1811 u64 data = 0; 1812 1813 switch (msr) { 1814 case HV_X64_MSR_VP_INDEX: { 1815 int r; 1816 struct kvm_vcpu *v; 1817 kvm_for_each_vcpu(r, v, vcpu->kvm) 1818 if (v == vcpu) 1819 data = r; 1820 break; 1821 } 1822 case HV_X64_MSR_EOI: 1823 return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata); 1824 case HV_X64_MSR_ICR: 1825 return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata); 1826 case HV_X64_MSR_TPR: 1827 return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata); 1828 default: 1829 pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr); 1830 return 1; 1831 } 1832 *pdata = data; 1833 return 0; 1834 } 1835 1836 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) 1837 { 1838 u64 data; 1839 1840 switch (msr) { 1841 case MSR_IA32_PLATFORM_ID: 1842 case MSR_IA32_UCODE_REV: 1843 case MSR_IA32_EBL_CR_POWERON: 1844 case MSR_IA32_DEBUGCTLMSR: 1845 case MSR_IA32_LASTBRANCHFROMIP: 1846 case MSR_IA32_LASTBRANCHTOIP: 1847 case MSR_IA32_LASTINTFROMIP: 1848 case MSR_IA32_LASTINTTOIP: 1849 case MSR_K8_SYSCFG: 1850 case MSR_K7_HWCR: 1851 case MSR_VM_HSAVE_PA: 1852 case MSR_P6_PERFCTR0: 1853 case MSR_P6_PERFCTR1: 1854 case MSR_P6_EVNTSEL0: 1855 case MSR_P6_EVNTSEL1: 1856 case MSR_K7_EVNTSEL0: 1857 case MSR_K7_PERFCTR0: 1858 case MSR_K8_INT_PENDING_MSG: 1859 case MSR_AMD64_NB_CFG: 1860 case MSR_FAM10H_MMIO_CONF_BASE: 1861 data = 0; 1862 break; 1863 case MSR_MTRRcap: 1864 data = 0x500 | KVM_NR_VAR_MTRR; 1865 break; 1866 case 0x200 ... 0x2ff: 1867 return get_msr_mtrr(vcpu, msr, pdata); 1868 case 0xcd: /* fsb frequency */ 1869 data = 3; 1870 break; 1871 /* 1872 * MSR_EBC_FREQUENCY_ID 1873 * Conservative value valid for even the basic CPU models. 1874 * Models 0,1: 000 in bits 23:21 indicating a bus speed of 1875 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz, 1876 * and 266MHz for model 3, or 4. Set Core Clock 1877 * Frequency to System Bus Frequency Ratio to 1 (bits 1878 * 31:24) even though these are only valid for CPU 1879 * models > 2, however guests may end up dividing or 1880 * multiplying by zero otherwise. 1881 */ 1882 case MSR_EBC_FREQUENCY_ID: 1883 data = 1 << 24; 1884 break; 1885 case MSR_IA32_APICBASE: 1886 data = kvm_get_apic_base(vcpu); 1887 break; 1888 case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff: 1889 return kvm_x2apic_msr_read(vcpu, msr, pdata); 1890 break; 1891 case MSR_IA32_MISC_ENABLE: 1892 data = vcpu->arch.ia32_misc_enable_msr; 1893 break; 1894 case MSR_IA32_PERF_STATUS: 1895 /* TSC increment by tick */ 1896 data = 1000ULL; 1897 /* CPU multiplier */ 1898 data |= (((uint64_t)4ULL) << 40); 1899 break; 1900 case MSR_EFER: 1901 data = vcpu->arch.efer; 1902 break; 1903 case MSR_KVM_WALL_CLOCK: 1904 case MSR_KVM_WALL_CLOCK_NEW: 1905 data = vcpu->kvm->arch.wall_clock; 1906 break; 1907 case MSR_KVM_SYSTEM_TIME: 1908 case MSR_KVM_SYSTEM_TIME_NEW: 1909 data = vcpu->arch.time; 1910 break; 1911 case MSR_KVM_ASYNC_PF_EN: 1912 data = vcpu->arch.apf.msr_val; 1913 break; 1914 case MSR_KVM_STEAL_TIME: 1915 data = vcpu->arch.st.msr_val; 1916 break; 1917 case MSR_IA32_P5_MC_ADDR: 1918 case MSR_IA32_P5_MC_TYPE: 1919 case MSR_IA32_MCG_CAP: 1920 case MSR_IA32_MCG_CTL: 1921 case MSR_IA32_MCG_STATUS: 1922 case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1: 1923 return get_msr_mce(vcpu, msr, pdata); 1924 case MSR_K7_CLK_CTL: 1925 /* 1926 * Provide expected ramp-up count for K7. All other 1927 * are set to zero, indicating minimum divisors for 1928 * every field. 1929 * 1930 * This prevents guest kernels on AMD host with CPU 1931 * type 6, model 8 and higher from exploding due to 1932 * the rdmsr failing. 1933 */ 1934 data = 0x20000000; 1935 break; 1936 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: 1937 if (kvm_hv_msr_partition_wide(msr)) { 1938 int r; 1939 mutex_lock(&vcpu->kvm->lock); 1940 r = get_msr_hyperv_pw(vcpu, msr, pdata); 1941 mutex_unlock(&vcpu->kvm->lock); 1942 return r; 1943 } else 1944 return get_msr_hyperv(vcpu, msr, pdata); 1945 break; 1946 case MSR_IA32_BBL_CR_CTL3: 1947 /* This legacy MSR exists but isn't fully documented in current 1948 * silicon. It is however accessed by winxp in very narrow 1949 * scenarios where it sets bit #19, itself documented as 1950 * a "reserved" bit. Best effort attempt to source coherent 1951 * read data here should the balance of the register be 1952 * interpreted by the guest: 1953 * 1954 * L2 cache control register 3: 64GB range, 256KB size, 1955 * enabled, latency 0x1, configured 1956 */ 1957 data = 0xbe702111; 1958 break; 1959 default: 1960 if (!ignore_msrs) { 1961 pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr); 1962 return 1; 1963 } else { 1964 pr_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr); 1965 data = 0; 1966 } 1967 break; 1968 } 1969 *pdata = data; 1970 return 0; 1971 } 1972 EXPORT_SYMBOL_GPL(kvm_get_msr_common); 1973 1974 /* 1975 * Read or write a bunch of msrs. All parameters are kernel addresses. 1976 * 1977 * @return number of msrs set successfully. 1978 */ 1979 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, 1980 struct kvm_msr_entry *entries, 1981 int (*do_msr)(struct kvm_vcpu *vcpu, 1982 unsigned index, u64 *data)) 1983 { 1984 int i, idx; 1985 1986 idx = srcu_read_lock(&vcpu->kvm->srcu); 1987 for (i = 0; i < msrs->nmsrs; ++i) 1988 if (do_msr(vcpu, entries[i].index, &entries[i].data)) 1989 break; 1990 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1991 1992 return i; 1993 } 1994 1995 /* 1996 * Read or write a bunch of msrs. Parameters are user addresses. 1997 * 1998 * @return number of msrs set successfully. 1999 */ 2000 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, 2001 int (*do_msr)(struct kvm_vcpu *vcpu, 2002 unsigned index, u64 *data), 2003 int writeback) 2004 { 2005 struct kvm_msrs msrs; 2006 struct kvm_msr_entry *entries; 2007 int r, n; 2008 unsigned size; 2009 2010 r = -EFAULT; 2011 if (copy_from_user(&msrs, user_msrs, sizeof msrs)) 2012 goto out; 2013 2014 r = -E2BIG; 2015 if (msrs.nmsrs >= MAX_IO_MSRS) 2016 goto out; 2017 2018 r = -ENOMEM; 2019 size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; 2020 entries = kmalloc(size, GFP_KERNEL); 2021 if (!entries) 2022 goto out; 2023 2024 r = -EFAULT; 2025 if (copy_from_user(entries, user_msrs->entries, size)) 2026 goto out_free; 2027 2028 r = n = __msr_io(vcpu, &msrs, entries, do_msr); 2029 if (r < 0) 2030 goto out_free; 2031 2032 r = -EFAULT; 2033 if (writeback && copy_to_user(user_msrs->entries, entries, size)) 2034 goto out_free; 2035 2036 r = n; 2037 2038 out_free: 2039 kfree(entries); 2040 out: 2041 return r; 2042 } 2043 2044 int kvm_dev_ioctl_check_extension(long ext) 2045 { 2046 int r; 2047 2048 switch (ext) { 2049 case KVM_CAP_IRQCHIP: 2050 case KVM_CAP_HLT: 2051 case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: 2052 case KVM_CAP_SET_TSS_ADDR: 2053 case KVM_CAP_EXT_CPUID: 2054 case KVM_CAP_CLOCKSOURCE: 2055 case KVM_CAP_PIT: 2056 case KVM_CAP_NOP_IO_DELAY: 2057 case KVM_CAP_MP_STATE: 2058 case KVM_CAP_SYNC_MMU: 2059 case KVM_CAP_USER_NMI: 2060 case KVM_CAP_REINJECT_CONTROL: 2061 case KVM_CAP_IRQ_INJECT_STATUS: 2062 case KVM_CAP_ASSIGN_DEV_IRQ: 2063 case KVM_CAP_IRQFD: 2064 case KVM_CAP_IOEVENTFD: 2065 case KVM_CAP_PIT2: 2066 case KVM_CAP_PIT_STATE2: 2067 case KVM_CAP_SET_IDENTITY_MAP_ADDR: 2068 case KVM_CAP_XEN_HVM: 2069 case KVM_CAP_ADJUST_CLOCK: 2070 case KVM_CAP_VCPU_EVENTS: 2071 case KVM_CAP_HYPERV: 2072 case KVM_CAP_HYPERV_VAPIC: 2073 case KVM_CAP_HYPERV_SPIN: 2074 case KVM_CAP_PCI_SEGMENT: 2075 case KVM_CAP_DEBUGREGS: 2076 case KVM_CAP_X86_ROBUST_SINGLESTEP: 2077 case KVM_CAP_XSAVE: 2078 case KVM_CAP_ASYNC_PF: 2079 case KVM_CAP_GET_TSC_KHZ: 2080 r = 1; 2081 break; 2082 case KVM_CAP_COALESCED_MMIO: 2083 r = KVM_COALESCED_MMIO_PAGE_OFFSET; 2084 break; 2085 case KVM_CAP_VAPIC: 2086 r = !kvm_x86_ops->cpu_has_accelerated_tpr(); 2087 break; 2088 case KVM_CAP_NR_VCPUS: 2089 r = KVM_MAX_VCPUS; 2090 break; 2091 case KVM_CAP_NR_MEMSLOTS: 2092 r = KVM_MEMORY_SLOTS; 2093 break; 2094 case KVM_CAP_PV_MMU: /* obsolete */ 2095 r = 0; 2096 break; 2097 case KVM_CAP_IOMMU: 2098 r = iommu_found(); 2099 break; 2100 case KVM_CAP_MCE: 2101 r = KVM_MAX_MCE_BANKS; 2102 break; 2103 case KVM_CAP_XCRS: 2104 r = cpu_has_xsave; 2105 break; 2106 case KVM_CAP_TSC_CONTROL: 2107 r = kvm_has_tsc_control; 2108 break; 2109 default: 2110 r = 0; 2111 break; 2112 } 2113 return r; 2114 2115 } 2116 2117 long kvm_arch_dev_ioctl(struct file *filp, 2118 unsigned int ioctl, unsigned long arg) 2119 { 2120 void __user *argp = (void __user *)arg; 2121 long r; 2122 2123 switch (ioctl) { 2124 case KVM_GET_MSR_INDEX_LIST: { 2125 struct kvm_msr_list __user *user_msr_list = argp; 2126 struct kvm_msr_list msr_list; 2127 unsigned n; 2128 2129 r = -EFAULT; 2130 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list)) 2131 goto out; 2132 n = msr_list.nmsrs; 2133 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs); 2134 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list)) 2135 goto out; 2136 r = -E2BIG; 2137 if (n < msr_list.nmsrs) 2138 goto out; 2139 r = -EFAULT; 2140 if (copy_to_user(user_msr_list->indices, &msrs_to_save, 2141 num_msrs_to_save * sizeof(u32))) 2142 goto out; 2143 if (copy_to_user(user_msr_list->indices + num_msrs_to_save, 2144 &emulated_msrs, 2145 ARRAY_SIZE(emulated_msrs) * sizeof(u32))) 2146 goto out; 2147 r = 0; 2148 break; 2149 } 2150 case KVM_GET_SUPPORTED_CPUID: { 2151 struct kvm_cpuid2 __user *cpuid_arg = argp; 2152 struct kvm_cpuid2 cpuid; 2153 2154 r = -EFAULT; 2155 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) 2156 goto out; 2157 r = kvm_dev_ioctl_get_supported_cpuid(&cpuid, 2158 cpuid_arg->entries); 2159 if (r) 2160 goto out; 2161 2162 r = -EFAULT; 2163 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) 2164 goto out; 2165 r = 0; 2166 break; 2167 } 2168 case KVM_X86_GET_MCE_CAP_SUPPORTED: { 2169 u64 mce_cap; 2170 2171 mce_cap = KVM_MCE_CAP_SUPPORTED; 2172 r = -EFAULT; 2173 if (copy_to_user(argp, &mce_cap, sizeof mce_cap)) 2174 goto out; 2175 r = 0; 2176 break; 2177 } 2178 default: 2179 r = -EINVAL; 2180 } 2181 out: 2182 return r; 2183 } 2184 2185 static void wbinvd_ipi(void *garbage) 2186 { 2187 wbinvd(); 2188 } 2189 2190 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu) 2191 { 2192 return vcpu->kvm->arch.iommu_domain && 2193 !(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY); 2194 } 2195 2196 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 2197 { 2198 /* Address WBINVD may be executed by guest */ 2199 if (need_emulate_wbinvd(vcpu)) { 2200 if (kvm_x86_ops->has_wbinvd_exit()) 2201 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 2202 else if (vcpu->cpu != -1 && vcpu->cpu != cpu) 2203 smp_call_function_single(vcpu->cpu, 2204 wbinvd_ipi, NULL, 1); 2205 } 2206 2207 kvm_x86_ops->vcpu_load(vcpu, cpu); 2208 if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) { 2209 /* Make sure TSC doesn't go backwards */ 2210 s64 tsc_delta; 2211 u64 tsc; 2212 2213 kvm_get_msr(vcpu, MSR_IA32_TSC, &tsc); 2214 tsc_delta = !vcpu->arch.last_guest_tsc ? 0 : 2215 tsc - vcpu->arch.last_guest_tsc; 2216 2217 if (tsc_delta < 0) 2218 mark_tsc_unstable("KVM discovered backwards TSC"); 2219 if (check_tsc_unstable()) { 2220 kvm_x86_ops->adjust_tsc_offset(vcpu, -tsc_delta); 2221 vcpu->arch.tsc_catchup = 1; 2222 } 2223 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 2224 if (vcpu->cpu != cpu) 2225 kvm_migrate_timers(vcpu); 2226 vcpu->cpu = cpu; 2227 } 2228 2229 accumulate_steal_time(vcpu); 2230 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 2231 } 2232 2233 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 2234 { 2235 kvm_x86_ops->vcpu_put(vcpu); 2236 kvm_put_guest_fpu(vcpu); 2237 kvm_get_msr(vcpu, MSR_IA32_TSC, &vcpu->arch.last_guest_tsc); 2238 } 2239 2240 static int is_efer_nx(void) 2241 { 2242 unsigned long long efer = 0; 2243 2244 rdmsrl_safe(MSR_EFER, &efer); 2245 return efer & EFER_NX; 2246 } 2247 2248 static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) 2249 { 2250 int i; 2251 struct kvm_cpuid_entry2 *e, *entry; 2252 2253 entry = NULL; 2254 for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { 2255 e = &vcpu->arch.cpuid_entries[i]; 2256 if (e->function == 0x80000001) { 2257 entry = e; 2258 break; 2259 } 2260 } 2261 if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) { 2262 entry->edx &= ~(1 << 20); 2263 printk(KERN_INFO "kvm: guest NX capability removed\n"); 2264 } 2265 } 2266 2267 /* when an old userspace process fills a new kernel module */ 2268 static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, 2269 struct kvm_cpuid *cpuid, 2270 struct kvm_cpuid_entry __user *entries) 2271 { 2272 int r, i; 2273 struct kvm_cpuid_entry *cpuid_entries; 2274 2275 r = -E2BIG; 2276 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) 2277 goto out; 2278 r = -ENOMEM; 2279 cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent); 2280 if (!cpuid_entries) 2281 goto out; 2282 r = -EFAULT; 2283 if (copy_from_user(cpuid_entries, entries, 2284 cpuid->nent * sizeof(struct kvm_cpuid_entry))) 2285 goto out_free; 2286 for (i = 0; i < cpuid->nent; i++) { 2287 vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function; 2288 vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax; 2289 vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx; 2290 vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx; 2291 vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx; 2292 vcpu->arch.cpuid_entries[i].index = 0; 2293 vcpu->arch.cpuid_entries[i].flags = 0; 2294 vcpu->arch.cpuid_entries[i].padding[0] = 0; 2295 vcpu->arch.cpuid_entries[i].padding[1] = 0; 2296 vcpu->arch.cpuid_entries[i].padding[2] = 0; 2297 } 2298 vcpu->arch.cpuid_nent = cpuid->nent; 2299 cpuid_fix_nx_cap(vcpu); 2300 r = 0; 2301 kvm_apic_set_version(vcpu); 2302 kvm_x86_ops->cpuid_update(vcpu); 2303 update_cpuid(vcpu); 2304 2305 out_free: 2306 vfree(cpuid_entries); 2307 out: 2308 return r; 2309 } 2310 2311 static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, 2312 struct kvm_cpuid2 *cpuid, 2313 struct kvm_cpuid_entry2 __user *entries) 2314 { 2315 int r; 2316 2317 r = -E2BIG; 2318 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) 2319 goto out; 2320 r = -EFAULT; 2321 if (copy_from_user(&vcpu->arch.cpuid_entries, entries, 2322 cpuid->nent * sizeof(struct kvm_cpuid_entry2))) 2323 goto out; 2324 vcpu->arch.cpuid_nent = cpuid->nent; 2325 kvm_apic_set_version(vcpu); 2326 kvm_x86_ops->cpuid_update(vcpu); 2327 update_cpuid(vcpu); 2328 return 0; 2329 2330 out: 2331 return r; 2332 } 2333 2334 static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, 2335 struct kvm_cpuid2 *cpuid, 2336 struct kvm_cpuid_entry2 __user *entries) 2337 { 2338 int r; 2339 2340 r = -E2BIG; 2341 if (cpuid->nent < vcpu->arch.cpuid_nent) 2342 goto out; 2343 r = -EFAULT; 2344 if (copy_to_user(entries, &vcpu->arch.cpuid_entries, 2345 vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2))) 2346 goto out; 2347 return 0; 2348 2349 out: 2350 cpuid->nent = vcpu->arch.cpuid_nent; 2351 return r; 2352 } 2353 2354 static void cpuid_mask(u32 *word, int wordnum) 2355 { 2356 *word &= boot_cpu_data.x86_capability[wordnum]; 2357 } 2358 2359 static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, 2360 u32 index) 2361 { 2362 entry->function = function; 2363 entry->index = index; 2364 cpuid_count(entry->function, entry->index, 2365 &entry->eax, &entry->ebx, &entry->ecx, &entry->edx); 2366 entry->flags = 0; 2367 } 2368 2369 static bool supported_xcr0_bit(unsigned bit) 2370 { 2371 u64 mask = ((u64)1 << bit); 2372 2373 return mask & (XSTATE_FP | XSTATE_SSE | XSTATE_YMM) & host_xcr0; 2374 } 2375 2376 #define F(x) bit(X86_FEATURE_##x) 2377 2378 static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, 2379 u32 index, int *nent, int maxnent) 2380 { 2381 unsigned f_nx = is_efer_nx() ? F(NX) : 0; 2382 #ifdef CONFIG_X86_64 2383 unsigned f_gbpages = (kvm_x86_ops->get_lpage_level() == PT_PDPE_LEVEL) 2384 ? F(GBPAGES) : 0; 2385 unsigned f_lm = F(LM); 2386 #else 2387 unsigned f_gbpages = 0; 2388 unsigned f_lm = 0; 2389 #endif 2390 unsigned f_rdtscp = kvm_x86_ops->rdtscp_supported() ? F(RDTSCP) : 0; 2391 2392 /* cpuid 1.edx */ 2393 const u32 kvm_supported_word0_x86_features = 2394 F(FPU) | F(VME) | F(DE) | F(PSE) | 2395 F(TSC) | F(MSR) | F(PAE) | F(MCE) | 2396 F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) | 2397 F(MTRR) | F(PGE) | F(MCA) | F(CMOV) | 2398 F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLSH) | 2399 0 /* Reserved, DS, ACPI */ | F(MMX) | 2400 F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) | 2401 0 /* HTT, TM, Reserved, PBE */; 2402 /* cpuid 0x80000001.edx */ 2403 const u32 kvm_supported_word1_x86_features = 2404 F(FPU) | F(VME) | F(DE) | F(PSE) | 2405 F(TSC) | F(MSR) | F(PAE) | F(MCE) | 2406 F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) | 2407 F(MTRR) | F(PGE) | F(MCA) | F(CMOV) | 2408 F(PAT) | F(PSE36) | 0 /* Reserved */ | 2409 f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) | 2410 F(FXSR) | F(FXSR_OPT) | f_gbpages | f_rdtscp | 2411 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW); 2412 /* cpuid 1.ecx */ 2413 const u32 kvm_supported_word4_x86_features = 2414 F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ | 2415 0 /* DS-CPL, VMX, SMX, EST */ | 2416 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ | 2417 0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ | 2418 0 /* Reserved, DCA */ | F(XMM4_1) | 2419 F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) | 2420 0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) | 2421 F(F16C) | F(RDRAND); 2422 /* cpuid 0x80000001.ecx */ 2423 const u32 kvm_supported_word6_x86_features = 2424 F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ | 2425 F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) | 2426 F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(XOP) | 2427 0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM); 2428 2429 /* cpuid 0xC0000001.edx */ 2430 const u32 kvm_supported_word5_x86_features = 2431 F(XSTORE) | F(XSTORE_EN) | F(XCRYPT) | F(XCRYPT_EN) | 2432 F(ACE2) | F(ACE2_EN) | F(PHE) | F(PHE_EN) | 2433 F(PMM) | F(PMM_EN); 2434 2435 /* cpuid 7.0.ebx */ 2436 const u32 kvm_supported_word9_x86_features = 2437 F(SMEP) | F(FSGSBASE) | F(ERMS); 2438 2439 /* all calls to cpuid_count() should be made on the same cpu */ 2440 get_cpu(); 2441 do_cpuid_1_ent(entry, function, index); 2442 ++*nent; 2443 2444 switch (function) { 2445 case 0: 2446 entry->eax = min(entry->eax, (u32)0xd); 2447 break; 2448 case 1: 2449 entry->edx &= kvm_supported_word0_x86_features; 2450 cpuid_mask(&entry->edx, 0); 2451 entry->ecx &= kvm_supported_word4_x86_features; 2452 cpuid_mask(&entry->ecx, 4); 2453 /* we support x2apic emulation even if host does not support 2454 * it since we emulate x2apic in software */ 2455 entry->ecx |= F(X2APIC); 2456 break; 2457 /* function 2 entries are STATEFUL. That is, repeated cpuid commands 2458 * may return different values. This forces us to get_cpu() before 2459 * issuing the first command, and also to emulate this annoying behavior 2460 * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */ 2461 case 2: { 2462 int t, times = entry->eax & 0xff; 2463 2464 entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; 2465 entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; 2466 for (t = 1; t < times && *nent < maxnent; ++t) { 2467 do_cpuid_1_ent(&entry[t], function, 0); 2468 entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; 2469 ++*nent; 2470 } 2471 break; 2472 } 2473 /* function 4 has additional index. */ 2474 case 4: { 2475 int i, cache_type; 2476 2477 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 2478 /* read more entries until cache_type is zero */ 2479 for (i = 1; *nent < maxnent; ++i) { 2480 cache_type = entry[i - 1].eax & 0x1f; 2481 if (!cache_type) 2482 break; 2483 do_cpuid_1_ent(&entry[i], function, i); 2484 entry[i].flags |= 2485 KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 2486 ++*nent; 2487 } 2488 break; 2489 } 2490 case 7: { 2491 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 2492 /* Mask ebx against host capbability word 9 */ 2493 if (index == 0) { 2494 entry->ebx &= kvm_supported_word9_x86_features; 2495 cpuid_mask(&entry->ebx, 9); 2496 } else 2497 entry->ebx = 0; 2498 entry->eax = 0; 2499 entry->ecx = 0; 2500 entry->edx = 0; 2501 break; 2502 } 2503 case 9: 2504 break; 2505 /* function 0xb has additional index. */ 2506 case 0xb: { 2507 int i, level_type; 2508 2509 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 2510 /* read more entries until level_type is zero */ 2511 for (i = 1; *nent < maxnent; ++i) { 2512 level_type = entry[i - 1].ecx & 0xff00; 2513 if (!level_type) 2514 break; 2515 do_cpuid_1_ent(&entry[i], function, i); 2516 entry[i].flags |= 2517 KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 2518 ++*nent; 2519 } 2520 break; 2521 } 2522 case 0xd: { 2523 int idx, i; 2524 2525 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 2526 for (idx = 1, i = 1; *nent < maxnent && idx < 64; ++idx) { 2527 do_cpuid_1_ent(&entry[i], function, idx); 2528 if (entry[i].eax == 0 || !supported_xcr0_bit(idx)) 2529 continue; 2530 entry[i].flags |= 2531 KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 2532 ++*nent; 2533 ++i; 2534 } 2535 break; 2536 } 2537 case KVM_CPUID_SIGNATURE: { 2538 char signature[12] = "KVMKVMKVM\0\0"; 2539 u32 *sigptr = (u32 *)signature; 2540 entry->eax = 0; 2541 entry->ebx = sigptr[0]; 2542 entry->ecx = sigptr[1]; 2543 entry->edx = sigptr[2]; 2544 break; 2545 } 2546 case KVM_CPUID_FEATURES: 2547 entry->eax = (1 << KVM_FEATURE_CLOCKSOURCE) | 2548 (1 << KVM_FEATURE_NOP_IO_DELAY) | 2549 (1 << KVM_FEATURE_CLOCKSOURCE2) | 2550 (1 << KVM_FEATURE_ASYNC_PF) | 2551 (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT); 2552 2553 if (sched_info_on()) 2554 entry->eax |= (1 << KVM_FEATURE_STEAL_TIME); 2555 2556 entry->ebx = 0; 2557 entry->ecx = 0; 2558 entry->edx = 0; 2559 break; 2560 case 0x80000000: 2561 entry->eax = min(entry->eax, 0x8000001a); 2562 break; 2563 case 0x80000001: 2564 entry->edx &= kvm_supported_word1_x86_features; 2565 cpuid_mask(&entry->edx, 1); 2566 entry->ecx &= kvm_supported_word6_x86_features; 2567 cpuid_mask(&entry->ecx, 6); 2568 break; 2569 case 0x80000008: { 2570 unsigned g_phys_as = (entry->eax >> 16) & 0xff; 2571 unsigned virt_as = max((entry->eax >> 8) & 0xff, 48U); 2572 unsigned phys_as = entry->eax & 0xff; 2573 2574 if (!g_phys_as) 2575 g_phys_as = phys_as; 2576 entry->eax = g_phys_as | (virt_as << 8); 2577 entry->ebx = entry->edx = 0; 2578 break; 2579 } 2580 case 0x80000019: 2581 entry->ecx = entry->edx = 0; 2582 break; 2583 case 0x8000001a: 2584 break; 2585 case 0x8000001d: 2586 break; 2587 /*Add support for Centaur's CPUID instruction*/ 2588 case 0xC0000000: 2589 /*Just support up to 0xC0000004 now*/ 2590 entry->eax = min(entry->eax, 0xC0000004); 2591 break; 2592 case 0xC0000001: 2593 entry->edx &= kvm_supported_word5_x86_features; 2594 cpuid_mask(&entry->edx, 5); 2595 break; 2596 case 3: /* Processor serial number */ 2597 case 5: /* MONITOR/MWAIT */ 2598 case 6: /* Thermal management */ 2599 case 0xA: /* Architectural Performance Monitoring */ 2600 case 0x80000007: /* Advanced power management */ 2601 case 0xC0000002: 2602 case 0xC0000003: 2603 case 0xC0000004: 2604 default: 2605 entry->eax = entry->ebx = entry->ecx = entry->edx = 0; 2606 break; 2607 } 2608 2609 kvm_x86_ops->set_supported_cpuid(function, entry); 2610 2611 put_cpu(); 2612 } 2613 2614 #undef F 2615 2616 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, 2617 struct kvm_cpuid_entry2 __user *entries) 2618 { 2619 struct kvm_cpuid_entry2 *cpuid_entries; 2620 int limit, nent = 0, r = -E2BIG; 2621 u32 func; 2622 2623 if (cpuid->nent < 1) 2624 goto out; 2625 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) 2626 cpuid->nent = KVM_MAX_CPUID_ENTRIES; 2627 r = -ENOMEM; 2628 cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent); 2629 if (!cpuid_entries) 2630 goto out; 2631 2632 do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent); 2633 limit = cpuid_entries[0].eax; 2634 for (func = 1; func <= limit && nent < cpuid->nent; ++func) 2635 do_cpuid_ent(&cpuid_entries[nent], func, 0, 2636 &nent, cpuid->nent); 2637 r = -E2BIG; 2638 if (nent >= cpuid->nent) 2639 goto out_free; 2640 2641 do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent); 2642 limit = cpuid_entries[nent - 1].eax; 2643 for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func) 2644 do_cpuid_ent(&cpuid_entries[nent], func, 0, 2645 &nent, cpuid->nent); 2646 2647 2648 2649 r = -E2BIG; 2650 if (nent >= cpuid->nent) 2651 goto out_free; 2652 2653 /* Add support for Centaur's CPUID instruction. */ 2654 if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR) { 2655 do_cpuid_ent(&cpuid_entries[nent], 0xC0000000, 0, 2656 &nent, cpuid->nent); 2657 2658 r = -E2BIG; 2659 if (nent >= cpuid->nent) 2660 goto out_free; 2661 2662 limit = cpuid_entries[nent - 1].eax; 2663 for (func = 0xC0000001; 2664 func <= limit && nent < cpuid->nent; ++func) 2665 do_cpuid_ent(&cpuid_entries[nent], func, 0, 2666 &nent, cpuid->nent); 2667 2668 r = -E2BIG; 2669 if (nent >= cpuid->nent) 2670 goto out_free; 2671 } 2672 2673 do_cpuid_ent(&cpuid_entries[nent], KVM_CPUID_SIGNATURE, 0, &nent, 2674 cpuid->nent); 2675 2676 r = -E2BIG; 2677 if (nent >= cpuid->nent) 2678 goto out_free; 2679 2680 do_cpuid_ent(&cpuid_entries[nent], KVM_CPUID_FEATURES, 0, &nent, 2681 cpuid->nent); 2682 2683 r = -E2BIG; 2684 if (nent >= cpuid->nent) 2685 goto out_free; 2686 2687 r = -EFAULT; 2688 if (copy_to_user(entries, cpuid_entries, 2689 nent * sizeof(struct kvm_cpuid_entry2))) 2690 goto out_free; 2691 cpuid->nent = nent; 2692 r = 0; 2693 2694 out_free: 2695 vfree(cpuid_entries); 2696 out: 2697 return r; 2698 } 2699 2700 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, 2701 struct kvm_lapic_state *s) 2702 { 2703 memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s); 2704 2705 return 0; 2706 } 2707 2708 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, 2709 struct kvm_lapic_state *s) 2710 { 2711 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); 2712 kvm_apic_post_state_restore(vcpu); 2713 update_cr8_intercept(vcpu); 2714 2715 return 0; 2716 } 2717 2718 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, 2719 struct kvm_interrupt *irq) 2720 { 2721 if (irq->irq < 0 || irq->irq >= 256) 2722 return -EINVAL; 2723 if (irqchip_in_kernel(vcpu->kvm)) 2724 return -ENXIO; 2725 2726 kvm_queue_interrupt(vcpu, irq->irq, false); 2727 kvm_make_request(KVM_REQ_EVENT, vcpu); 2728 2729 return 0; 2730 } 2731 2732 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu) 2733 { 2734 kvm_inject_nmi(vcpu); 2735 2736 return 0; 2737 } 2738 2739 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu, 2740 struct kvm_tpr_access_ctl *tac) 2741 { 2742 if (tac->flags) 2743 return -EINVAL; 2744 vcpu->arch.tpr_access_reporting = !!tac->enabled; 2745 return 0; 2746 } 2747 2748 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, 2749 u64 mcg_cap) 2750 { 2751 int r; 2752 unsigned bank_num = mcg_cap & 0xff, bank; 2753 2754 r = -EINVAL; 2755 if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS) 2756 goto out; 2757 if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000)) 2758 goto out; 2759 r = 0; 2760 vcpu->arch.mcg_cap = mcg_cap; 2761 /* Init IA32_MCG_CTL to all 1s */ 2762 if (mcg_cap & MCG_CTL_P) 2763 vcpu->arch.mcg_ctl = ~(u64)0; 2764 /* Init IA32_MCi_CTL to all 1s */ 2765 for (bank = 0; bank < bank_num; bank++) 2766 vcpu->arch.mce_banks[bank*4] = ~(u64)0; 2767 out: 2768 return r; 2769 } 2770 2771 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, 2772 struct kvm_x86_mce *mce) 2773 { 2774 u64 mcg_cap = vcpu->arch.mcg_cap; 2775 unsigned bank_num = mcg_cap & 0xff; 2776 u64 *banks = vcpu->arch.mce_banks; 2777 2778 if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL)) 2779 return -EINVAL; 2780 /* 2781 * if IA32_MCG_CTL is not all 1s, the uncorrected error 2782 * reporting is disabled 2783 */ 2784 if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) && 2785 vcpu->arch.mcg_ctl != ~(u64)0) 2786 return 0; 2787 banks += 4 * mce->bank; 2788 /* 2789 * if IA32_MCi_CTL is not all 1s, the uncorrected error 2790 * reporting is disabled for the bank 2791 */ 2792 if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0) 2793 return 0; 2794 if (mce->status & MCI_STATUS_UC) { 2795 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) || 2796 !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) { 2797 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 2798 return 0; 2799 } 2800 if (banks[1] & MCI_STATUS_VAL) 2801 mce->status |= MCI_STATUS_OVER; 2802 banks[2] = mce->addr; 2803 banks[3] = mce->misc; 2804 vcpu->arch.mcg_status = mce->mcg_status; 2805 banks[1] = mce->status; 2806 kvm_queue_exception(vcpu, MC_VECTOR); 2807 } else if (!(banks[1] & MCI_STATUS_VAL) 2808 || !(banks[1] & MCI_STATUS_UC)) { 2809 if (banks[1] & MCI_STATUS_VAL) 2810 mce->status |= MCI_STATUS_OVER; 2811 banks[2] = mce->addr; 2812 banks[3] = mce->misc; 2813 banks[1] = mce->status; 2814 } else 2815 banks[1] |= MCI_STATUS_OVER; 2816 return 0; 2817 } 2818 2819 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, 2820 struct kvm_vcpu_events *events) 2821 { 2822 events->exception.injected = 2823 vcpu->arch.exception.pending && 2824 !kvm_exception_is_soft(vcpu->arch.exception.nr); 2825 events->exception.nr = vcpu->arch.exception.nr; 2826 events->exception.has_error_code = vcpu->arch.exception.has_error_code; 2827 events->exception.pad = 0; 2828 events->exception.error_code = vcpu->arch.exception.error_code; 2829 2830 events->interrupt.injected = 2831 vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft; 2832 events->interrupt.nr = vcpu->arch.interrupt.nr; 2833 events->interrupt.soft = 0; 2834 events->interrupt.shadow = 2835 kvm_x86_ops->get_interrupt_shadow(vcpu, 2836 KVM_X86_SHADOW_INT_MOV_SS | KVM_X86_SHADOW_INT_STI); 2837 2838 events->nmi.injected = vcpu->arch.nmi_injected; 2839 events->nmi.pending = vcpu->arch.nmi_pending; 2840 events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu); 2841 events->nmi.pad = 0; 2842 2843 events->sipi_vector = vcpu->arch.sipi_vector; 2844 2845 events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING 2846 | KVM_VCPUEVENT_VALID_SIPI_VECTOR 2847 | KVM_VCPUEVENT_VALID_SHADOW); 2848 memset(&events->reserved, 0, sizeof(events->reserved)); 2849 } 2850 2851 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, 2852 struct kvm_vcpu_events *events) 2853 { 2854 if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING 2855 | KVM_VCPUEVENT_VALID_SIPI_VECTOR 2856 | KVM_VCPUEVENT_VALID_SHADOW)) 2857 return -EINVAL; 2858 2859 vcpu->arch.exception.pending = events->exception.injected; 2860 vcpu->arch.exception.nr = events->exception.nr; 2861 vcpu->arch.exception.has_error_code = events->exception.has_error_code; 2862 vcpu->arch.exception.error_code = events->exception.error_code; 2863 2864 vcpu->arch.interrupt.pending = events->interrupt.injected; 2865 vcpu->arch.interrupt.nr = events->interrupt.nr; 2866 vcpu->arch.interrupt.soft = events->interrupt.soft; 2867 if (events->flags & KVM_VCPUEVENT_VALID_SHADOW) 2868 kvm_x86_ops->set_interrupt_shadow(vcpu, 2869 events->interrupt.shadow); 2870 2871 vcpu->arch.nmi_injected = events->nmi.injected; 2872 if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING) 2873 vcpu->arch.nmi_pending = events->nmi.pending; 2874 kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked); 2875 2876 if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR) 2877 vcpu->arch.sipi_vector = events->sipi_vector; 2878 2879 kvm_make_request(KVM_REQ_EVENT, vcpu); 2880 2881 return 0; 2882 } 2883 2884 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, 2885 struct kvm_debugregs *dbgregs) 2886 { 2887 memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db)); 2888 dbgregs->dr6 = vcpu->arch.dr6; 2889 dbgregs->dr7 = vcpu->arch.dr7; 2890 dbgregs->flags = 0; 2891 memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved)); 2892 } 2893 2894 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, 2895 struct kvm_debugregs *dbgregs) 2896 { 2897 if (dbgregs->flags) 2898 return -EINVAL; 2899 2900 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db)); 2901 vcpu->arch.dr6 = dbgregs->dr6; 2902 vcpu->arch.dr7 = dbgregs->dr7; 2903 2904 return 0; 2905 } 2906 2907 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, 2908 struct kvm_xsave *guest_xsave) 2909 { 2910 if (cpu_has_xsave) 2911 memcpy(guest_xsave->region, 2912 &vcpu->arch.guest_fpu.state->xsave, 2913 xstate_size); 2914 else { 2915 memcpy(guest_xsave->region, 2916 &vcpu->arch.guest_fpu.state->fxsave, 2917 sizeof(struct i387_fxsave_struct)); 2918 *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] = 2919 XSTATE_FPSSE; 2920 } 2921 } 2922 2923 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, 2924 struct kvm_xsave *guest_xsave) 2925 { 2926 u64 xstate_bv = 2927 *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)]; 2928 2929 if (cpu_has_xsave) 2930 memcpy(&vcpu->arch.guest_fpu.state->xsave, 2931 guest_xsave->region, xstate_size); 2932 else { 2933 if (xstate_bv & ~XSTATE_FPSSE) 2934 return -EINVAL; 2935 memcpy(&vcpu->arch.guest_fpu.state->fxsave, 2936 guest_xsave->region, sizeof(struct i387_fxsave_struct)); 2937 } 2938 return 0; 2939 } 2940 2941 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu, 2942 struct kvm_xcrs *guest_xcrs) 2943 { 2944 if (!cpu_has_xsave) { 2945 guest_xcrs->nr_xcrs = 0; 2946 return; 2947 } 2948 2949 guest_xcrs->nr_xcrs = 1; 2950 guest_xcrs->flags = 0; 2951 guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK; 2952 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0; 2953 } 2954 2955 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu, 2956 struct kvm_xcrs *guest_xcrs) 2957 { 2958 int i, r = 0; 2959 2960 if (!cpu_has_xsave) 2961 return -EINVAL; 2962 2963 if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags) 2964 return -EINVAL; 2965 2966 for (i = 0; i < guest_xcrs->nr_xcrs; i++) 2967 /* Only support XCR0 currently */ 2968 if (guest_xcrs->xcrs[0].xcr == XCR_XFEATURE_ENABLED_MASK) { 2969 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK, 2970 guest_xcrs->xcrs[0].value); 2971 break; 2972 } 2973 if (r) 2974 r = -EINVAL; 2975 return r; 2976 } 2977 2978 long kvm_arch_vcpu_ioctl(struct file *filp, 2979 unsigned int ioctl, unsigned long arg) 2980 { 2981 struct kvm_vcpu *vcpu = filp->private_data; 2982 void __user *argp = (void __user *)arg; 2983 int r; 2984 union { 2985 struct kvm_lapic_state *lapic; 2986 struct kvm_xsave *xsave; 2987 struct kvm_xcrs *xcrs; 2988 void *buffer; 2989 } u; 2990 2991 u.buffer = NULL; 2992 switch (ioctl) { 2993 case KVM_GET_LAPIC: { 2994 r = -EINVAL; 2995 if (!vcpu->arch.apic) 2996 goto out; 2997 u.lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); 2998 2999 r = -ENOMEM; 3000 if (!u.lapic) 3001 goto out; 3002 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic); 3003 if (r) 3004 goto out; 3005 r = -EFAULT; 3006 if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state))) 3007 goto out; 3008 r = 0; 3009 break; 3010 } 3011 case KVM_SET_LAPIC: { 3012 r = -EINVAL; 3013 if (!vcpu->arch.apic) 3014 goto out; 3015 u.lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); 3016 r = -ENOMEM; 3017 if (!u.lapic) 3018 goto out; 3019 r = -EFAULT; 3020 if (copy_from_user(u.lapic, argp, sizeof(struct kvm_lapic_state))) 3021 goto out; 3022 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic); 3023 if (r) 3024 goto out; 3025 r = 0; 3026 break; 3027 } 3028 case KVM_INTERRUPT: { 3029 struct kvm_interrupt irq; 3030 3031 r = -EFAULT; 3032 if (copy_from_user(&irq, argp, sizeof irq)) 3033 goto out; 3034 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); 3035 if (r) 3036 goto out; 3037 r = 0; 3038 break; 3039 } 3040 case KVM_NMI: { 3041 r = kvm_vcpu_ioctl_nmi(vcpu); 3042 if (r) 3043 goto out; 3044 r = 0; 3045 break; 3046 } 3047 case KVM_SET_CPUID: { 3048 struct kvm_cpuid __user *cpuid_arg = argp; 3049 struct kvm_cpuid cpuid; 3050 3051 r = -EFAULT; 3052 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) 3053 goto out; 3054 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); 3055 if (r) 3056 goto out; 3057 break; 3058 } 3059 case KVM_SET_CPUID2: { 3060 struct kvm_cpuid2 __user *cpuid_arg = argp; 3061 struct kvm_cpuid2 cpuid; 3062 3063 r = -EFAULT; 3064 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) 3065 goto out; 3066 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, 3067 cpuid_arg->entries); 3068 if (r) 3069 goto out; 3070 break; 3071 } 3072 case KVM_GET_CPUID2: { 3073 struct kvm_cpuid2 __user *cpuid_arg = argp; 3074 struct kvm_cpuid2 cpuid; 3075 3076 r = -EFAULT; 3077 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) 3078 goto out; 3079 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, 3080 cpuid_arg->entries); 3081 if (r) 3082 goto out; 3083 r = -EFAULT; 3084 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) 3085 goto out; 3086 r = 0; 3087 break; 3088 } 3089 case KVM_GET_MSRS: 3090 r = msr_io(vcpu, argp, kvm_get_msr, 1); 3091 break; 3092 case KVM_SET_MSRS: 3093 r = msr_io(vcpu, argp, do_set_msr, 0); 3094 break; 3095 case KVM_TPR_ACCESS_REPORTING: { 3096 struct kvm_tpr_access_ctl tac; 3097 3098 r = -EFAULT; 3099 if (copy_from_user(&tac, argp, sizeof tac)) 3100 goto out; 3101 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac); 3102 if (r) 3103 goto out; 3104 r = -EFAULT; 3105 if (copy_to_user(argp, &tac, sizeof tac)) 3106 goto out; 3107 r = 0; 3108 break; 3109 }; 3110 case KVM_SET_VAPIC_ADDR: { 3111 struct kvm_vapic_addr va; 3112 3113 r = -EINVAL; 3114 if (!irqchip_in_kernel(vcpu->kvm)) 3115 goto out; 3116 r = -EFAULT; 3117 if (copy_from_user(&va, argp, sizeof va)) 3118 goto out; 3119 r = 0; 3120 kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr); 3121 break; 3122 } 3123 case KVM_X86_SETUP_MCE: { 3124 u64 mcg_cap; 3125 3126 r = -EFAULT; 3127 if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap)) 3128 goto out; 3129 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap); 3130 break; 3131 } 3132 case KVM_X86_SET_MCE: { 3133 struct kvm_x86_mce mce; 3134 3135 r = -EFAULT; 3136 if (copy_from_user(&mce, argp, sizeof mce)) 3137 goto out; 3138 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce); 3139 break; 3140 } 3141 case KVM_GET_VCPU_EVENTS: { 3142 struct kvm_vcpu_events events; 3143 3144 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events); 3145 3146 r = -EFAULT; 3147 if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events))) 3148 break; 3149 r = 0; 3150 break; 3151 } 3152 case KVM_SET_VCPU_EVENTS: { 3153 struct kvm_vcpu_events events; 3154 3155 r = -EFAULT; 3156 if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events))) 3157 break; 3158 3159 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events); 3160 break; 3161 } 3162 case KVM_GET_DEBUGREGS: { 3163 struct kvm_debugregs dbgregs; 3164 3165 kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs); 3166 3167 r = -EFAULT; 3168 if (copy_to_user(argp, &dbgregs, 3169 sizeof(struct kvm_debugregs))) 3170 break; 3171 r = 0; 3172 break; 3173 } 3174 case KVM_SET_DEBUGREGS: { 3175 struct kvm_debugregs dbgregs; 3176 3177 r = -EFAULT; 3178 if (copy_from_user(&dbgregs, argp, 3179 sizeof(struct kvm_debugregs))) 3180 break; 3181 3182 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs); 3183 break; 3184 } 3185 case KVM_GET_XSAVE: { 3186 u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL); 3187 r = -ENOMEM; 3188 if (!u.xsave) 3189 break; 3190 3191 kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave); 3192 3193 r = -EFAULT; 3194 if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave))) 3195 break; 3196 r = 0; 3197 break; 3198 } 3199 case KVM_SET_XSAVE: { 3200 u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL); 3201 r = -ENOMEM; 3202 if (!u.xsave) 3203 break; 3204 3205 r = -EFAULT; 3206 if (copy_from_user(u.xsave, argp, sizeof(struct kvm_xsave))) 3207 break; 3208 3209 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave); 3210 break; 3211 } 3212 case KVM_GET_XCRS: { 3213 u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL); 3214 r = -ENOMEM; 3215 if (!u.xcrs) 3216 break; 3217 3218 kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs); 3219 3220 r = -EFAULT; 3221 if (copy_to_user(argp, u.xcrs, 3222 sizeof(struct kvm_xcrs))) 3223 break; 3224 r = 0; 3225 break; 3226 } 3227 case KVM_SET_XCRS: { 3228 u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL); 3229 r = -ENOMEM; 3230 if (!u.xcrs) 3231 break; 3232 3233 r = -EFAULT; 3234 if (copy_from_user(u.xcrs, argp, 3235 sizeof(struct kvm_xcrs))) 3236 break; 3237 3238 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs); 3239 break; 3240 } 3241 case KVM_SET_TSC_KHZ: { 3242 u32 user_tsc_khz; 3243 3244 r = -EINVAL; 3245 if (!kvm_has_tsc_control) 3246 break; 3247 3248 user_tsc_khz = (u32)arg; 3249 3250 if (user_tsc_khz >= kvm_max_guest_tsc_khz) 3251 goto out; 3252 3253 kvm_x86_ops->set_tsc_khz(vcpu, user_tsc_khz); 3254 3255 r = 0; 3256 goto out; 3257 } 3258 case KVM_GET_TSC_KHZ: { 3259 r = -EIO; 3260 if (check_tsc_unstable()) 3261 goto out; 3262 3263 r = vcpu_tsc_khz(vcpu); 3264 3265 goto out; 3266 } 3267 default: 3268 r = -EINVAL; 3269 } 3270 out: 3271 kfree(u.buffer); 3272 return r; 3273 } 3274 3275 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) 3276 { 3277 int ret; 3278 3279 if (addr > (unsigned int)(-3 * PAGE_SIZE)) 3280 return -1; 3281 ret = kvm_x86_ops->set_tss_addr(kvm, addr); 3282 return ret; 3283 } 3284 3285 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm, 3286 u64 ident_addr) 3287 { 3288 kvm->arch.ept_identity_map_addr = ident_addr; 3289 return 0; 3290 } 3291 3292 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, 3293 u32 kvm_nr_mmu_pages) 3294 { 3295 if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) 3296 return -EINVAL; 3297 3298 mutex_lock(&kvm->slots_lock); 3299 spin_lock(&kvm->mmu_lock); 3300 3301 kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); 3302 kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; 3303 3304 spin_unlock(&kvm->mmu_lock); 3305 mutex_unlock(&kvm->slots_lock); 3306 return 0; 3307 } 3308 3309 static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) 3310 { 3311 return kvm->arch.n_max_mmu_pages; 3312 } 3313 3314 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) 3315 { 3316 int r; 3317 3318 r = 0; 3319 switch (chip->chip_id) { 3320 case KVM_IRQCHIP_PIC_MASTER: 3321 memcpy(&chip->chip.pic, 3322 &pic_irqchip(kvm)->pics[0], 3323 sizeof(struct kvm_pic_state)); 3324 break; 3325 case KVM_IRQCHIP_PIC_SLAVE: 3326 memcpy(&chip->chip.pic, 3327 &pic_irqchip(kvm)->pics[1], 3328 sizeof(struct kvm_pic_state)); 3329 break; 3330 case KVM_IRQCHIP_IOAPIC: 3331 r = kvm_get_ioapic(kvm, &chip->chip.ioapic); 3332 break; 3333 default: 3334 r = -EINVAL; 3335 break; 3336 } 3337 return r; 3338 } 3339 3340 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) 3341 { 3342 int r; 3343 3344 r = 0; 3345 switch (chip->chip_id) { 3346 case KVM_IRQCHIP_PIC_MASTER: 3347 spin_lock(&pic_irqchip(kvm)->lock); 3348 memcpy(&pic_irqchip(kvm)->pics[0], 3349 &chip->chip.pic, 3350 sizeof(struct kvm_pic_state)); 3351 spin_unlock(&pic_irqchip(kvm)->lock); 3352 break; 3353 case KVM_IRQCHIP_PIC_SLAVE: 3354 spin_lock(&pic_irqchip(kvm)->lock); 3355 memcpy(&pic_irqchip(kvm)->pics[1], 3356 &chip->chip.pic, 3357 sizeof(struct kvm_pic_state)); 3358 spin_unlock(&pic_irqchip(kvm)->lock); 3359 break; 3360 case KVM_IRQCHIP_IOAPIC: 3361 r = kvm_set_ioapic(kvm, &chip->chip.ioapic); 3362 break; 3363 default: 3364 r = -EINVAL; 3365 break; 3366 } 3367 kvm_pic_update_irq(pic_irqchip(kvm)); 3368 return r; 3369 } 3370 3371 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps) 3372 { 3373 int r = 0; 3374 3375 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3376 memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state)); 3377 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3378 return r; 3379 } 3380 3381 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps) 3382 { 3383 int r = 0; 3384 3385 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3386 memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state)); 3387 kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0); 3388 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3389 return r; 3390 } 3391 3392 static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps) 3393 { 3394 int r = 0; 3395 3396 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3397 memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels, 3398 sizeof(ps->channels)); 3399 ps->flags = kvm->arch.vpit->pit_state.flags; 3400 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3401 memset(&ps->reserved, 0, sizeof(ps->reserved)); 3402 return r; 3403 } 3404 3405 static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps) 3406 { 3407 int r = 0, start = 0; 3408 u32 prev_legacy, cur_legacy; 3409 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3410 prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY; 3411 cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY; 3412 if (!prev_legacy && cur_legacy) 3413 start = 1; 3414 memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels, 3415 sizeof(kvm->arch.vpit->pit_state.channels)); 3416 kvm->arch.vpit->pit_state.flags = ps->flags; 3417 kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start); 3418 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3419 return r; 3420 } 3421 3422 static int kvm_vm_ioctl_reinject(struct kvm *kvm, 3423 struct kvm_reinject_control *control) 3424 { 3425 if (!kvm->arch.vpit) 3426 return -ENXIO; 3427 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3428 kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject; 3429 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3430 return 0; 3431 } 3432 3433 /* 3434 * Get (and clear) the dirty memory log for a memory slot. 3435 */ 3436 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 3437 struct kvm_dirty_log *log) 3438 { 3439 int r, i; 3440 struct kvm_memory_slot *memslot; 3441 unsigned long n; 3442 unsigned long is_dirty = 0; 3443 3444 mutex_lock(&kvm->slots_lock); 3445 3446 r = -EINVAL; 3447 if (log->slot >= KVM_MEMORY_SLOTS) 3448 goto out; 3449 3450 memslot = &kvm->memslots->memslots[log->slot]; 3451 r = -ENOENT; 3452 if (!memslot->dirty_bitmap) 3453 goto out; 3454 3455 n = kvm_dirty_bitmap_bytes(memslot); 3456 3457 for (i = 0; !is_dirty && i < n/sizeof(long); i++) 3458 is_dirty = memslot->dirty_bitmap[i]; 3459 3460 /* If nothing is dirty, don't bother messing with page tables. */ 3461 if (is_dirty) { 3462 struct kvm_memslots *slots, *old_slots; 3463 unsigned long *dirty_bitmap; 3464 3465 dirty_bitmap = memslot->dirty_bitmap_head; 3466 if (memslot->dirty_bitmap == dirty_bitmap) 3467 dirty_bitmap += n / sizeof(long); 3468 memset(dirty_bitmap, 0, n); 3469 3470 r = -ENOMEM; 3471 slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 3472 if (!slots) 3473 goto out; 3474 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots)); 3475 slots->memslots[log->slot].dirty_bitmap = dirty_bitmap; 3476 slots->generation++; 3477 3478 old_slots = kvm->memslots; 3479 rcu_assign_pointer(kvm->memslots, slots); 3480 synchronize_srcu_expedited(&kvm->srcu); 3481 dirty_bitmap = old_slots->memslots[log->slot].dirty_bitmap; 3482 kfree(old_slots); 3483 3484 spin_lock(&kvm->mmu_lock); 3485 kvm_mmu_slot_remove_write_access(kvm, log->slot); 3486 spin_unlock(&kvm->mmu_lock); 3487 3488 r = -EFAULT; 3489 if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n)) 3490 goto out; 3491 } else { 3492 r = -EFAULT; 3493 if (clear_user(log->dirty_bitmap, n)) 3494 goto out; 3495 } 3496 3497 r = 0; 3498 out: 3499 mutex_unlock(&kvm->slots_lock); 3500 return r; 3501 } 3502 3503 long kvm_arch_vm_ioctl(struct file *filp, 3504 unsigned int ioctl, unsigned long arg) 3505 { 3506 struct kvm *kvm = filp->private_data; 3507 void __user *argp = (void __user *)arg; 3508 int r = -ENOTTY; 3509 /* 3510 * This union makes it completely explicit to gcc-3.x 3511 * that these two variables' stack usage should be 3512 * combined, not added together. 3513 */ 3514 union { 3515 struct kvm_pit_state ps; 3516 struct kvm_pit_state2 ps2; 3517 struct kvm_pit_config pit_config; 3518 } u; 3519 3520 switch (ioctl) { 3521 case KVM_SET_TSS_ADDR: 3522 r = kvm_vm_ioctl_set_tss_addr(kvm, arg); 3523 if (r < 0) 3524 goto out; 3525 break; 3526 case KVM_SET_IDENTITY_MAP_ADDR: { 3527 u64 ident_addr; 3528 3529 r = -EFAULT; 3530 if (copy_from_user(&ident_addr, argp, sizeof ident_addr)) 3531 goto out; 3532 r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr); 3533 if (r < 0) 3534 goto out; 3535 break; 3536 } 3537 case KVM_SET_NR_MMU_PAGES: 3538 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); 3539 if (r) 3540 goto out; 3541 break; 3542 case KVM_GET_NR_MMU_PAGES: 3543 r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); 3544 break; 3545 case KVM_CREATE_IRQCHIP: { 3546 struct kvm_pic *vpic; 3547 3548 mutex_lock(&kvm->lock); 3549 r = -EEXIST; 3550 if (kvm->arch.vpic) 3551 goto create_irqchip_unlock; 3552 r = -ENOMEM; 3553 vpic = kvm_create_pic(kvm); 3554 if (vpic) { 3555 r = kvm_ioapic_init(kvm); 3556 if (r) { 3557 mutex_lock(&kvm->slots_lock); 3558 kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, 3559 &vpic->dev); 3560 mutex_unlock(&kvm->slots_lock); 3561 kfree(vpic); 3562 goto create_irqchip_unlock; 3563 } 3564 } else 3565 goto create_irqchip_unlock; 3566 smp_wmb(); 3567 kvm->arch.vpic = vpic; 3568 smp_wmb(); 3569 r = kvm_setup_default_irq_routing(kvm); 3570 if (r) { 3571 mutex_lock(&kvm->slots_lock); 3572 mutex_lock(&kvm->irq_lock); 3573 kvm_ioapic_destroy(kvm); 3574 kvm_destroy_pic(kvm); 3575 mutex_unlock(&kvm->irq_lock); 3576 mutex_unlock(&kvm->slots_lock); 3577 } 3578 create_irqchip_unlock: 3579 mutex_unlock(&kvm->lock); 3580 break; 3581 } 3582 case KVM_CREATE_PIT: 3583 u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY; 3584 goto create_pit; 3585 case KVM_CREATE_PIT2: 3586 r = -EFAULT; 3587 if (copy_from_user(&u.pit_config, argp, 3588 sizeof(struct kvm_pit_config))) 3589 goto out; 3590 create_pit: 3591 mutex_lock(&kvm->slots_lock); 3592 r = -EEXIST; 3593 if (kvm->arch.vpit) 3594 goto create_pit_unlock; 3595 r = -ENOMEM; 3596 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags); 3597 if (kvm->arch.vpit) 3598 r = 0; 3599 create_pit_unlock: 3600 mutex_unlock(&kvm->slots_lock); 3601 break; 3602 case KVM_IRQ_LINE_STATUS: 3603 case KVM_IRQ_LINE: { 3604 struct kvm_irq_level irq_event; 3605 3606 r = -EFAULT; 3607 if (copy_from_user(&irq_event, argp, sizeof irq_event)) 3608 goto out; 3609 r = -ENXIO; 3610 if (irqchip_in_kernel(kvm)) { 3611 __s32 status; 3612 status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 3613 irq_event.irq, irq_event.level); 3614 if (ioctl == KVM_IRQ_LINE_STATUS) { 3615 r = -EFAULT; 3616 irq_event.status = status; 3617 if (copy_to_user(argp, &irq_event, 3618 sizeof irq_event)) 3619 goto out; 3620 } 3621 r = 0; 3622 } 3623 break; 3624 } 3625 case KVM_GET_IRQCHIP: { 3626 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 3627 struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL); 3628 3629 r = -ENOMEM; 3630 if (!chip) 3631 goto out; 3632 r = -EFAULT; 3633 if (copy_from_user(chip, argp, sizeof *chip)) 3634 goto get_irqchip_out; 3635 r = -ENXIO; 3636 if (!irqchip_in_kernel(kvm)) 3637 goto get_irqchip_out; 3638 r = kvm_vm_ioctl_get_irqchip(kvm, chip); 3639 if (r) 3640 goto get_irqchip_out; 3641 r = -EFAULT; 3642 if (copy_to_user(argp, chip, sizeof *chip)) 3643 goto get_irqchip_out; 3644 r = 0; 3645 get_irqchip_out: 3646 kfree(chip); 3647 if (r) 3648 goto out; 3649 break; 3650 } 3651 case KVM_SET_IRQCHIP: { 3652 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 3653 struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL); 3654 3655 r = -ENOMEM; 3656 if (!chip) 3657 goto out; 3658 r = -EFAULT; 3659 if (copy_from_user(chip, argp, sizeof *chip)) 3660 goto set_irqchip_out; 3661 r = -ENXIO; 3662 if (!irqchip_in_kernel(kvm)) 3663 goto set_irqchip_out; 3664 r = kvm_vm_ioctl_set_irqchip(kvm, chip); 3665 if (r) 3666 goto set_irqchip_out; 3667 r = 0; 3668 set_irqchip_out: 3669 kfree(chip); 3670 if (r) 3671 goto out; 3672 break; 3673 } 3674 case KVM_GET_PIT: { 3675 r = -EFAULT; 3676 if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state))) 3677 goto out; 3678 r = -ENXIO; 3679 if (!kvm->arch.vpit) 3680 goto out; 3681 r = kvm_vm_ioctl_get_pit(kvm, &u.ps); 3682 if (r) 3683 goto out; 3684 r = -EFAULT; 3685 if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state))) 3686 goto out; 3687 r = 0; 3688 break; 3689 } 3690 case KVM_SET_PIT: { 3691 r = -EFAULT; 3692 if (copy_from_user(&u.ps, argp, sizeof u.ps)) 3693 goto out; 3694 r = -ENXIO; 3695 if (!kvm->arch.vpit) 3696 goto out; 3697 r = kvm_vm_ioctl_set_pit(kvm, &u.ps); 3698 if (r) 3699 goto out; 3700 r = 0; 3701 break; 3702 } 3703 case KVM_GET_PIT2: { 3704 r = -ENXIO; 3705 if (!kvm->arch.vpit) 3706 goto out; 3707 r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2); 3708 if (r) 3709 goto out; 3710 r = -EFAULT; 3711 if (copy_to_user(argp, &u.ps2, sizeof(u.ps2))) 3712 goto out; 3713 r = 0; 3714 break; 3715 } 3716 case KVM_SET_PIT2: { 3717 r = -EFAULT; 3718 if (copy_from_user(&u.ps2, argp, sizeof(u.ps2))) 3719 goto out; 3720 r = -ENXIO; 3721 if (!kvm->arch.vpit) 3722 goto out; 3723 r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2); 3724 if (r) 3725 goto out; 3726 r = 0; 3727 break; 3728 } 3729 case KVM_REINJECT_CONTROL: { 3730 struct kvm_reinject_control control; 3731 r = -EFAULT; 3732 if (copy_from_user(&control, argp, sizeof(control))) 3733 goto out; 3734 r = kvm_vm_ioctl_reinject(kvm, &control); 3735 if (r) 3736 goto out; 3737 r = 0; 3738 break; 3739 } 3740 case KVM_XEN_HVM_CONFIG: { 3741 r = -EFAULT; 3742 if (copy_from_user(&kvm->arch.xen_hvm_config, argp, 3743 sizeof(struct kvm_xen_hvm_config))) 3744 goto out; 3745 r = -EINVAL; 3746 if (kvm->arch.xen_hvm_config.flags) 3747 goto out; 3748 r = 0; 3749 break; 3750 } 3751 case KVM_SET_CLOCK: { 3752 struct kvm_clock_data user_ns; 3753 u64 now_ns; 3754 s64 delta; 3755 3756 r = -EFAULT; 3757 if (copy_from_user(&user_ns, argp, sizeof(user_ns))) 3758 goto out; 3759 3760 r = -EINVAL; 3761 if (user_ns.flags) 3762 goto out; 3763 3764 r = 0; 3765 local_irq_disable(); 3766 now_ns = get_kernel_ns(); 3767 delta = user_ns.clock - now_ns; 3768 local_irq_enable(); 3769 kvm->arch.kvmclock_offset = delta; 3770 break; 3771 } 3772 case KVM_GET_CLOCK: { 3773 struct kvm_clock_data user_ns; 3774 u64 now_ns; 3775 3776 local_irq_disable(); 3777 now_ns = get_kernel_ns(); 3778 user_ns.clock = kvm->arch.kvmclock_offset + now_ns; 3779 local_irq_enable(); 3780 user_ns.flags = 0; 3781 memset(&user_ns.pad, 0, sizeof(user_ns.pad)); 3782 3783 r = -EFAULT; 3784 if (copy_to_user(argp, &user_ns, sizeof(user_ns))) 3785 goto out; 3786 r = 0; 3787 break; 3788 } 3789 3790 default: 3791 ; 3792 } 3793 out: 3794 return r; 3795 } 3796 3797 static void kvm_init_msr_list(void) 3798 { 3799 u32 dummy[2]; 3800 unsigned i, j; 3801 3802 /* skip the first msrs in the list. KVM-specific */ 3803 for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) { 3804 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) 3805 continue; 3806 if (j < i) 3807 msrs_to_save[j] = msrs_to_save[i]; 3808 j++; 3809 } 3810 num_msrs_to_save = j; 3811 } 3812 3813 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len, 3814 const void *v) 3815 { 3816 int handled = 0; 3817 int n; 3818 3819 do { 3820 n = min(len, 8); 3821 if (!(vcpu->arch.apic && 3822 !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, n, v)) 3823 && kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, n, v)) 3824 break; 3825 handled += n; 3826 addr += n; 3827 len -= n; 3828 v += n; 3829 } while (len); 3830 3831 return handled; 3832 } 3833 3834 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v) 3835 { 3836 int handled = 0; 3837 int n; 3838 3839 do { 3840 n = min(len, 8); 3841 if (!(vcpu->arch.apic && 3842 !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, n, v)) 3843 && kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, n, v)) 3844 break; 3845 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v); 3846 handled += n; 3847 addr += n; 3848 len -= n; 3849 v += n; 3850 } while (len); 3851 3852 return handled; 3853 } 3854 3855 static void kvm_set_segment(struct kvm_vcpu *vcpu, 3856 struct kvm_segment *var, int seg) 3857 { 3858 kvm_x86_ops->set_segment(vcpu, var, seg); 3859 } 3860 3861 void kvm_get_segment(struct kvm_vcpu *vcpu, 3862 struct kvm_segment *var, int seg) 3863 { 3864 kvm_x86_ops->get_segment(vcpu, var, seg); 3865 } 3866 3867 static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access) 3868 { 3869 return gpa; 3870 } 3871 3872 static gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access) 3873 { 3874 gpa_t t_gpa; 3875 struct x86_exception exception; 3876 3877 BUG_ON(!mmu_is_nested(vcpu)); 3878 3879 /* NPT walks are always user-walks */ 3880 access |= PFERR_USER_MASK; 3881 t_gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception); 3882 3883 return t_gpa; 3884 } 3885 3886 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, 3887 struct x86_exception *exception) 3888 { 3889 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 3890 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); 3891 } 3892 3893 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, 3894 struct x86_exception *exception) 3895 { 3896 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 3897 access |= PFERR_FETCH_MASK; 3898 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); 3899 } 3900 3901 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, 3902 struct x86_exception *exception) 3903 { 3904 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 3905 access |= PFERR_WRITE_MASK; 3906 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); 3907 } 3908 3909 /* uses this to access any guest's mapped memory without checking CPL */ 3910 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, 3911 struct x86_exception *exception) 3912 { 3913 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception); 3914 } 3915 3916 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, 3917 struct kvm_vcpu *vcpu, u32 access, 3918 struct x86_exception *exception) 3919 { 3920 void *data = val; 3921 int r = X86EMUL_CONTINUE; 3922 3923 while (bytes) { 3924 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access, 3925 exception); 3926 unsigned offset = addr & (PAGE_SIZE-1); 3927 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset); 3928 int ret; 3929 3930 if (gpa == UNMAPPED_GVA) 3931 return X86EMUL_PROPAGATE_FAULT; 3932 ret = kvm_read_guest(vcpu->kvm, gpa, data, toread); 3933 if (ret < 0) { 3934 r = X86EMUL_IO_NEEDED; 3935 goto out; 3936 } 3937 3938 bytes -= toread; 3939 data += toread; 3940 addr += toread; 3941 } 3942 out: 3943 return r; 3944 } 3945 3946 /* used for instruction fetching */ 3947 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, 3948 gva_t addr, void *val, unsigned int bytes, 3949 struct x86_exception *exception) 3950 { 3951 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 3952 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 3953 3954 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 3955 access | PFERR_FETCH_MASK, 3956 exception); 3957 } 3958 3959 int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt, 3960 gva_t addr, void *val, unsigned int bytes, 3961 struct x86_exception *exception) 3962 { 3963 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 3964 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 3965 3966 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, 3967 exception); 3968 } 3969 EXPORT_SYMBOL_GPL(kvm_read_guest_virt); 3970 3971 static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt, 3972 gva_t addr, void *val, unsigned int bytes, 3973 struct x86_exception *exception) 3974 { 3975 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 3976 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception); 3977 } 3978 3979 int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt, 3980 gva_t addr, void *val, 3981 unsigned int bytes, 3982 struct x86_exception *exception) 3983 { 3984 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 3985 void *data = val; 3986 int r = X86EMUL_CONTINUE; 3987 3988 while (bytes) { 3989 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, 3990 PFERR_WRITE_MASK, 3991 exception); 3992 unsigned offset = addr & (PAGE_SIZE-1); 3993 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset); 3994 int ret; 3995 3996 if (gpa == UNMAPPED_GVA) 3997 return X86EMUL_PROPAGATE_FAULT; 3998 ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite); 3999 if (ret < 0) { 4000 r = X86EMUL_IO_NEEDED; 4001 goto out; 4002 } 4003 4004 bytes -= towrite; 4005 data += towrite; 4006 addr += towrite; 4007 } 4008 out: 4009 return r; 4010 } 4011 EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system); 4012 4013 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 4014 gpa_t *gpa, struct x86_exception *exception, 4015 bool write) 4016 { 4017 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 4018 4019 if (vcpu_match_mmio_gva(vcpu, gva) && 4020 check_write_user_access(vcpu, write, access, 4021 vcpu->arch.access)) { 4022 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT | 4023 (gva & (PAGE_SIZE - 1)); 4024 trace_vcpu_match_mmio(gva, *gpa, write, false); 4025 return 1; 4026 } 4027 4028 if (write) 4029 access |= PFERR_WRITE_MASK; 4030 4031 *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); 4032 4033 if (*gpa == UNMAPPED_GVA) 4034 return -1; 4035 4036 /* For APIC access vmexit */ 4037 if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 4038 return 1; 4039 4040 if (vcpu_match_mmio_gpa(vcpu, *gpa)) { 4041 trace_vcpu_match_mmio(gva, *gpa, write, true); 4042 return 1; 4043 } 4044 4045 return 0; 4046 } 4047 4048 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt, 4049 unsigned long addr, 4050 void *val, 4051 unsigned int bytes, 4052 struct x86_exception *exception) 4053 { 4054 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4055 gpa_t gpa; 4056 int handled, ret; 4057 4058 if (vcpu->mmio_read_completed) { 4059 memcpy(val, vcpu->mmio_data, bytes); 4060 trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, 4061 vcpu->mmio_phys_addr, *(u64 *)val); 4062 vcpu->mmio_read_completed = 0; 4063 return X86EMUL_CONTINUE; 4064 } 4065 4066 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, false); 4067 4068 if (ret < 0) 4069 return X86EMUL_PROPAGATE_FAULT; 4070 4071 if (ret) 4072 goto mmio; 4073 4074 if (kvm_read_guest_virt(ctxt, addr, val, bytes, exception) 4075 == X86EMUL_CONTINUE) 4076 return X86EMUL_CONTINUE; 4077 4078 mmio: 4079 /* 4080 * Is this MMIO handled locally? 4081 */ 4082 handled = vcpu_mmio_read(vcpu, gpa, bytes, val); 4083 4084 if (handled == bytes) 4085 return X86EMUL_CONTINUE; 4086 4087 gpa += handled; 4088 bytes -= handled; 4089 val += handled; 4090 4091 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0); 4092 4093 vcpu->mmio_needed = 1; 4094 vcpu->run->exit_reason = KVM_EXIT_MMIO; 4095 vcpu->run->mmio.phys_addr = vcpu->mmio_phys_addr = gpa; 4096 vcpu->mmio_size = bytes; 4097 vcpu->run->mmio.len = min(vcpu->mmio_size, 8); 4098 vcpu->run->mmio.is_write = vcpu->mmio_is_write = 0; 4099 vcpu->mmio_index = 0; 4100 4101 return X86EMUL_IO_NEEDED; 4102 } 4103 4104 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, 4105 const void *val, int bytes) 4106 { 4107 int ret; 4108 4109 ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); 4110 if (ret < 0) 4111 return 0; 4112 kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1); 4113 return 1; 4114 } 4115 4116 static int emulator_write_emulated_onepage(unsigned long addr, 4117 const void *val, 4118 unsigned int bytes, 4119 struct x86_exception *exception, 4120 struct kvm_vcpu *vcpu) 4121 { 4122 gpa_t gpa; 4123 int handled, ret; 4124 4125 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, true); 4126 4127 if (ret < 0) 4128 return X86EMUL_PROPAGATE_FAULT; 4129 4130 /* For APIC access vmexit */ 4131 if (ret) 4132 goto mmio; 4133 4134 if (emulator_write_phys(vcpu, gpa, val, bytes)) 4135 return X86EMUL_CONTINUE; 4136 4137 mmio: 4138 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val); 4139 /* 4140 * Is this MMIO handled locally? 4141 */ 4142 handled = vcpu_mmio_write(vcpu, gpa, bytes, val); 4143 if (handled == bytes) 4144 return X86EMUL_CONTINUE; 4145 4146 gpa += handled; 4147 bytes -= handled; 4148 val += handled; 4149 4150 vcpu->mmio_needed = 1; 4151 memcpy(vcpu->mmio_data, val, bytes); 4152 vcpu->run->exit_reason = KVM_EXIT_MMIO; 4153 vcpu->run->mmio.phys_addr = vcpu->mmio_phys_addr = gpa; 4154 vcpu->mmio_size = bytes; 4155 vcpu->run->mmio.len = min(vcpu->mmio_size, 8); 4156 vcpu->run->mmio.is_write = vcpu->mmio_is_write = 1; 4157 memcpy(vcpu->run->mmio.data, vcpu->mmio_data, 8); 4158 vcpu->mmio_index = 0; 4159 4160 return X86EMUL_CONTINUE; 4161 } 4162 4163 int emulator_write_emulated(struct x86_emulate_ctxt *ctxt, 4164 unsigned long addr, 4165 const void *val, 4166 unsigned int bytes, 4167 struct x86_exception *exception) 4168 { 4169 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4170 4171 /* Crossing a page boundary? */ 4172 if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { 4173 int rc, now; 4174 4175 now = -addr & ~PAGE_MASK; 4176 rc = emulator_write_emulated_onepage(addr, val, now, exception, 4177 vcpu); 4178 if (rc != X86EMUL_CONTINUE) 4179 return rc; 4180 addr += now; 4181 val += now; 4182 bytes -= now; 4183 } 4184 return emulator_write_emulated_onepage(addr, val, bytes, exception, 4185 vcpu); 4186 } 4187 4188 #define CMPXCHG_TYPE(t, ptr, old, new) \ 4189 (cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old)) 4190 4191 #ifdef CONFIG_X86_64 4192 # define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new) 4193 #else 4194 # define CMPXCHG64(ptr, old, new) \ 4195 (cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old)) 4196 #endif 4197 4198 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, 4199 unsigned long addr, 4200 const void *old, 4201 const void *new, 4202 unsigned int bytes, 4203 struct x86_exception *exception) 4204 { 4205 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4206 gpa_t gpa; 4207 struct page *page; 4208 char *kaddr; 4209 bool exchanged; 4210 4211 /* guests cmpxchg8b have to be emulated atomically */ 4212 if (bytes > 8 || (bytes & (bytes - 1))) 4213 goto emul_write; 4214 4215 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL); 4216 4217 if (gpa == UNMAPPED_GVA || 4218 (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 4219 goto emul_write; 4220 4221 if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK)) 4222 goto emul_write; 4223 4224 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); 4225 if (is_error_page(page)) { 4226 kvm_release_page_clean(page); 4227 goto emul_write; 4228 } 4229 4230 kaddr = kmap_atomic(page, KM_USER0); 4231 kaddr += offset_in_page(gpa); 4232 switch (bytes) { 4233 case 1: 4234 exchanged = CMPXCHG_TYPE(u8, kaddr, old, new); 4235 break; 4236 case 2: 4237 exchanged = CMPXCHG_TYPE(u16, kaddr, old, new); 4238 break; 4239 case 4: 4240 exchanged = CMPXCHG_TYPE(u32, kaddr, old, new); 4241 break; 4242 case 8: 4243 exchanged = CMPXCHG64(kaddr, old, new); 4244 break; 4245 default: 4246 BUG(); 4247 } 4248 kunmap_atomic(kaddr, KM_USER0); 4249 kvm_release_page_dirty(page); 4250 4251 if (!exchanged) 4252 return X86EMUL_CMPXCHG_FAILED; 4253 4254 kvm_mmu_pte_write(vcpu, gpa, new, bytes, 1); 4255 4256 return X86EMUL_CONTINUE; 4257 4258 emul_write: 4259 printk_once(KERN_WARNING "kvm: emulating exchange as write\n"); 4260 4261 return emulator_write_emulated(ctxt, addr, new, bytes, exception); 4262 } 4263 4264 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd) 4265 { 4266 /* TODO: String I/O for in kernel device */ 4267 int r; 4268 4269 if (vcpu->arch.pio.in) 4270 r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port, 4271 vcpu->arch.pio.size, pd); 4272 else 4273 r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS, 4274 vcpu->arch.pio.port, vcpu->arch.pio.size, 4275 pd); 4276 return r; 4277 } 4278 4279 4280 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt, 4281 int size, unsigned short port, void *val, 4282 unsigned int count) 4283 { 4284 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4285 4286 if (vcpu->arch.pio.count) 4287 goto data_avail; 4288 4289 trace_kvm_pio(0, port, size, count); 4290 4291 vcpu->arch.pio.port = port; 4292 vcpu->arch.pio.in = 1; 4293 vcpu->arch.pio.count = count; 4294 vcpu->arch.pio.size = size; 4295 4296 if (!kernel_pio(vcpu, vcpu->arch.pio_data)) { 4297 data_avail: 4298 memcpy(val, vcpu->arch.pio_data, size * count); 4299 vcpu->arch.pio.count = 0; 4300 return 1; 4301 } 4302 4303 vcpu->run->exit_reason = KVM_EXIT_IO; 4304 vcpu->run->io.direction = KVM_EXIT_IO_IN; 4305 vcpu->run->io.size = size; 4306 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; 4307 vcpu->run->io.count = count; 4308 vcpu->run->io.port = port; 4309 4310 return 0; 4311 } 4312 4313 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, 4314 int size, unsigned short port, 4315 const void *val, unsigned int count) 4316 { 4317 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4318 4319 trace_kvm_pio(1, port, size, count); 4320 4321 vcpu->arch.pio.port = port; 4322 vcpu->arch.pio.in = 0; 4323 vcpu->arch.pio.count = count; 4324 vcpu->arch.pio.size = size; 4325 4326 memcpy(vcpu->arch.pio_data, val, size * count); 4327 4328 if (!kernel_pio(vcpu, vcpu->arch.pio_data)) { 4329 vcpu->arch.pio.count = 0; 4330 return 1; 4331 } 4332 4333 vcpu->run->exit_reason = KVM_EXIT_IO; 4334 vcpu->run->io.direction = KVM_EXIT_IO_OUT; 4335 vcpu->run->io.size = size; 4336 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; 4337 vcpu->run->io.count = count; 4338 vcpu->run->io.port = port; 4339 4340 return 0; 4341 } 4342 4343 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) 4344 { 4345 return kvm_x86_ops->get_segment_base(vcpu, seg); 4346 } 4347 4348 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address) 4349 { 4350 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); 4351 } 4352 4353 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) 4354 { 4355 if (!need_emulate_wbinvd(vcpu)) 4356 return X86EMUL_CONTINUE; 4357 4358 if (kvm_x86_ops->has_wbinvd_exit()) { 4359 int cpu = get_cpu(); 4360 4361 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 4362 smp_call_function_many(vcpu->arch.wbinvd_dirty_mask, 4363 wbinvd_ipi, NULL, 1); 4364 put_cpu(); 4365 cpumask_clear(vcpu->arch.wbinvd_dirty_mask); 4366 } else 4367 wbinvd(); 4368 return X86EMUL_CONTINUE; 4369 } 4370 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd); 4371 4372 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) 4373 { 4374 kvm_emulate_wbinvd(emul_to_vcpu(ctxt)); 4375 } 4376 4377 int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) 4378 { 4379 return _kvm_get_dr(emul_to_vcpu(ctxt), dr, dest); 4380 } 4381 4382 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) 4383 { 4384 4385 return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value); 4386 } 4387 4388 static u64 mk_cr_64(u64 curr_cr, u32 new_val) 4389 { 4390 return (curr_cr & ~((1ULL << 32) - 1)) | new_val; 4391 } 4392 4393 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr) 4394 { 4395 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4396 unsigned long value; 4397 4398 switch (cr) { 4399 case 0: 4400 value = kvm_read_cr0(vcpu); 4401 break; 4402 case 2: 4403 value = vcpu->arch.cr2; 4404 break; 4405 case 3: 4406 value = kvm_read_cr3(vcpu); 4407 break; 4408 case 4: 4409 value = kvm_read_cr4(vcpu); 4410 break; 4411 case 8: 4412 value = kvm_get_cr8(vcpu); 4413 break; 4414 default: 4415 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr); 4416 return 0; 4417 } 4418 4419 return value; 4420 } 4421 4422 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val) 4423 { 4424 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4425 int res = 0; 4426 4427 switch (cr) { 4428 case 0: 4429 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val)); 4430 break; 4431 case 2: 4432 vcpu->arch.cr2 = val; 4433 break; 4434 case 3: 4435 res = kvm_set_cr3(vcpu, val); 4436 break; 4437 case 4: 4438 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val)); 4439 break; 4440 case 8: 4441 res = kvm_set_cr8(vcpu, val); 4442 break; 4443 default: 4444 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr); 4445 res = -1; 4446 } 4447 4448 return res; 4449 } 4450 4451 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt) 4452 { 4453 return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt)); 4454 } 4455 4456 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 4457 { 4458 kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt); 4459 } 4460 4461 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 4462 { 4463 kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt); 4464 } 4465 4466 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 4467 { 4468 kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt); 4469 } 4470 4471 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 4472 { 4473 kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt); 4474 } 4475 4476 static unsigned long emulator_get_cached_segment_base( 4477 struct x86_emulate_ctxt *ctxt, int seg) 4478 { 4479 return get_segment_base(emul_to_vcpu(ctxt), seg); 4480 } 4481 4482 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector, 4483 struct desc_struct *desc, u32 *base3, 4484 int seg) 4485 { 4486 struct kvm_segment var; 4487 4488 kvm_get_segment(emul_to_vcpu(ctxt), &var, seg); 4489 *selector = var.selector; 4490 4491 if (var.unusable) 4492 return false; 4493 4494 if (var.g) 4495 var.limit >>= 12; 4496 set_desc_limit(desc, var.limit); 4497 set_desc_base(desc, (unsigned long)var.base); 4498 #ifdef CONFIG_X86_64 4499 if (base3) 4500 *base3 = var.base >> 32; 4501 #endif 4502 desc->type = var.type; 4503 desc->s = var.s; 4504 desc->dpl = var.dpl; 4505 desc->p = var.present; 4506 desc->avl = var.avl; 4507 desc->l = var.l; 4508 desc->d = var.db; 4509 desc->g = var.g; 4510 4511 return true; 4512 } 4513 4514 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector, 4515 struct desc_struct *desc, u32 base3, 4516 int seg) 4517 { 4518 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4519 struct kvm_segment var; 4520 4521 var.selector = selector; 4522 var.base = get_desc_base(desc); 4523 #ifdef CONFIG_X86_64 4524 var.base |= ((u64)base3) << 32; 4525 #endif 4526 var.limit = get_desc_limit(desc); 4527 if (desc->g) 4528 var.limit = (var.limit << 12) | 0xfff; 4529 var.type = desc->type; 4530 var.present = desc->p; 4531 var.dpl = desc->dpl; 4532 var.db = desc->d; 4533 var.s = desc->s; 4534 var.l = desc->l; 4535 var.g = desc->g; 4536 var.avl = desc->avl; 4537 var.present = desc->p; 4538 var.unusable = !var.present; 4539 var.padding = 0; 4540 4541 kvm_set_segment(vcpu, &var, seg); 4542 return; 4543 } 4544 4545 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, 4546 u32 msr_index, u64 *pdata) 4547 { 4548 return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata); 4549 } 4550 4551 static int emulator_set_msr(struct x86_emulate_ctxt *ctxt, 4552 u32 msr_index, u64 data) 4553 { 4554 return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data); 4555 } 4556 4557 static void emulator_halt(struct x86_emulate_ctxt *ctxt) 4558 { 4559 emul_to_vcpu(ctxt)->arch.halt_request = 1; 4560 } 4561 4562 static void emulator_get_fpu(struct x86_emulate_ctxt *ctxt) 4563 { 4564 preempt_disable(); 4565 kvm_load_guest_fpu(emul_to_vcpu(ctxt)); 4566 /* 4567 * CR0.TS may reference the host fpu state, not the guest fpu state, 4568 * so it may be clear at this point. 4569 */ 4570 clts(); 4571 } 4572 4573 static void emulator_put_fpu(struct x86_emulate_ctxt *ctxt) 4574 { 4575 preempt_enable(); 4576 } 4577 4578 static int emulator_intercept(struct x86_emulate_ctxt *ctxt, 4579 struct x86_instruction_info *info, 4580 enum x86_intercept_stage stage) 4581 { 4582 return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage); 4583 } 4584 4585 static struct x86_emulate_ops emulate_ops = { 4586 .read_std = kvm_read_guest_virt_system, 4587 .write_std = kvm_write_guest_virt_system, 4588 .fetch = kvm_fetch_guest_virt, 4589 .read_emulated = emulator_read_emulated, 4590 .write_emulated = emulator_write_emulated, 4591 .cmpxchg_emulated = emulator_cmpxchg_emulated, 4592 .invlpg = emulator_invlpg, 4593 .pio_in_emulated = emulator_pio_in_emulated, 4594 .pio_out_emulated = emulator_pio_out_emulated, 4595 .get_segment = emulator_get_segment, 4596 .set_segment = emulator_set_segment, 4597 .get_cached_segment_base = emulator_get_cached_segment_base, 4598 .get_gdt = emulator_get_gdt, 4599 .get_idt = emulator_get_idt, 4600 .set_gdt = emulator_set_gdt, 4601 .set_idt = emulator_set_idt, 4602 .get_cr = emulator_get_cr, 4603 .set_cr = emulator_set_cr, 4604 .cpl = emulator_get_cpl, 4605 .get_dr = emulator_get_dr, 4606 .set_dr = emulator_set_dr, 4607 .set_msr = emulator_set_msr, 4608 .get_msr = emulator_get_msr, 4609 .halt = emulator_halt, 4610 .wbinvd = emulator_wbinvd, 4611 .fix_hypercall = emulator_fix_hypercall, 4612 .get_fpu = emulator_get_fpu, 4613 .put_fpu = emulator_put_fpu, 4614 .intercept = emulator_intercept, 4615 }; 4616 4617 static void cache_all_regs(struct kvm_vcpu *vcpu) 4618 { 4619 kvm_register_read(vcpu, VCPU_REGS_RAX); 4620 kvm_register_read(vcpu, VCPU_REGS_RSP); 4621 kvm_register_read(vcpu, VCPU_REGS_RIP); 4622 vcpu->arch.regs_dirty = ~0; 4623 } 4624 4625 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) 4626 { 4627 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask); 4628 /* 4629 * an sti; sti; sequence only disable interrupts for the first 4630 * instruction. So, if the last instruction, be it emulated or 4631 * not, left the system with the INT_STI flag enabled, it 4632 * means that the last instruction is an sti. We should not 4633 * leave the flag on in this case. The same goes for mov ss 4634 */ 4635 if (!(int_shadow & mask)) 4636 kvm_x86_ops->set_interrupt_shadow(vcpu, mask); 4637 } 4638 4639 static void inject_emulated_exception(struct kvm_vcpu *vcpu) 4640 { 4641 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 4642 if (ctxt->exception.vector == PF_VECTOR) 4643 kvm_propagate_fault(vcpu, &ctxt->exception); 4644 else if (ctxt->exception.error_code_valid) 4645 kvm_queue_exception_e(vcpu, ctxt->exception.vector, 4646 ctxt->exception.error_code); 4647 else 4648 kvm_queue_exception(vcpu, ctxt->exception.vector); 4649 } 4650 4651 static void init_decode_cache(struct x86_emulate_ctxt *ctxt, 4652 const unsigned long *regs) 4653 { 4654 memset(&ctxt->twobyte, 0, 4655 (void *)&ctxt->regs - (void *)&ctxt->twobyte); 4656 memcpy(ctxt->regs, regs, sizeof(ctxt->regs)); 4657 4658 ctxt->fetch.start = 0; 4659 ctxt->fetch.end = 0; 4660 ctxt->io_read.pos = 0; 4661 ctxt->io_read.end = 0; 4662 ctxt->mem_read.pos = 0; 4663 ctxt->mem_read.end = 0; 4664 } 4665 4666 static void init_emulate_ctxt(struct kvm_vcpu *vcpu) 4667 { 4668 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 4669 int cs_db, cs_l; 4670 4671 /* 4672 * TODO: fix emulate.c to use guest_read/write_register 4673 * instead of direct ->regs accesses, can save hundred cycles 4674 * on Intel for instructions that don't read/change RSP, for 4675 * for example. 4676 */ 4677 cache_all_regs(vcpu); 4678 4679 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); 4680 4681 ctxt->eflags = kvm_get_rflags(vcpu); 4682 ctxt->eip = kvm_rip_read(vcpu); 4683 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL : 4684 (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 : 4685 cs_l ? X86EMUL_MODE_PROT64 : 4686 cs_db ? X86EMUL_MODE_PROT32 : 4687 X86EMUL_MODE_PROT16; 4688 ctxt->guest_mode = is_guest_mode(vcpu); 4689 4690 init_decode_cache(ctxt, vcpu->arch.regs); 4691 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 4692 } 4693 4694 int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) 4695 { 4696 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 4697 int ret; 4698 4699 init_emulate_ctxt(vcpu); 4700 4701 ctxt->op_bytes = 2; 4702 ctxt->ad_bytes = 2; 4703 ctxt->_eip = ctxt->eip + inc_eip; 4704 ret = emulate_int_real(ctxt, irq); 4705 4706 if (ret != X86EMUL_CONTINUE) 4707 return EMULATE_FAIL; 4708 4709 ctxt->eip = ctxt->_eip; 4710 memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs); 4711 kvm_rip_write(vcpu, ctxt->eip); 4712 kvm_set_rflags(vcpu, ctxt->eflags); 4713 4714 if (irq == NMI_VECTOR) 4715 vcpu->arch.nmi_pending = false; 4716 else 4717 vcpu->arch.interrupt.pending = false; 4718 4719 return EMULATE_DONE; 4720 } 4721 EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt); 4722 4723 static int handle_emulation_failure(struct kvm_vcpu *vcpu) 4724 { 4725 int r = EMULATE_DONE; 4726 4727 ++vcpu->stat.insn_emulation_fail; 4728 trace_kvm_emulate_insn_failed(vcpu); 4729 if (!is_guest_mode(vcpu)) { 4730 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 4731 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 4732 vcpu->run->internal.ndata = 0; 4733 r = EMULATE_FAIL; 4734 } 4735 kvm_queue_exception(vcpu, UD_VECTOR); 4736 4737 return r; 4738 } 4739 4740 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva) 4741 { 4742 gpa_t gpa; 4743 4744 if (tdp_enabled) 4745 return false; 4746 4747 /* 4748 * if emulation was due to access to shadowed page table 4749 * and it failed try to unshadow page and re-entetr the 4750 * guest to let CPU execute the instruction. 4751 */ 4752 if (kvm_mmu_unprotect_page_virt(vcpu, gva)) 4753 return true; 4754 4755 gpa = kvm_mmu_gva_to_gpa_system(vcpu, gva, NULL); 4756 4757 if (gpa == UNMAPPED_GVA) 4758 return true; /* let cpu generate fault */ 4759 4760 if (!kvm_is_error_hva(gfn_to_hva(vcpu->kvm, gpa >> PAGE_SHIFT))) 4761 return true; 4762 4763 return false; 4764 } 4765 4766 int x86_emulate_instruction(struct kvm_vcpu *vcpu, 4767 unsigned long cr2, 4768 int emulation_type, 4769 void *insn, 4770 int insn_len) 4771 { 4772 int r; 4773 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 4774 bool writeback = true; 4775 4776 kvm_clear_exception_queue(vcpu); 4777 4778 if (!(emulation_type & EMULTYPE_NO_DECODE)) { 4779 init_emulate_ctxt(vcpu); 4780 ctxt->interruptibility = 0; 4781 ctxt->have_exception = false; 4782 ctxt->perm_ok = false; 4783 4784 ctxt->only_vendor_specific_insn 4785 = emulation_type & EMULTYPE_TRAP_UD; 4786 4787 r = x86_decode_insn(ctxt, insn, insn_len); 4788 4789 trace_kvm_emulate_insn_start(vcpu); 4790 ++vcpu->stat.insn_emulation; 4791 if (r) { 4792 if (emulation_type & EMULTYPE_TRAP_UD) 4793 return EMULATE_FAIL; 4794 if (reexecute_instruction(vcpu, cr2)) 4795 return EMULATE_DONE; 4796 if (emulation_type & EMULTYPE_SKIP) 4797 return EMULATE_FAIL; 4798 return handle_emulation_failure(vcpu); 4799 } 4800 } 4801 4802 if (emulation_type & EMULTYPE_SKIP) { 4803 kvm_rip_write(vcpu, ctxt->_eip); 4804 return EMULATE_DONE; 4805 } 4806 4807 /* this is needed for vmware backdoor interface to work since it 4808 changes registers values during IO operation */ 4809 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) { 4810 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 4811 memcpy(ctxt->regs, vcpu->arch.regs, sizeof ctxt->regs); 4812 } 4813 4814 restart: 4815 r = x86_emulate_insn(ctxt); 4816 4817 if (r == EMULATION_INTERCEPTED) 4818 return EMULATE_DONE; 4819 4820 if (r == EMULATION_FAILED) { 4821 if (reexecute_instruction(vcpu, cr2)) 4822 return EMULATE_DONE; 4823 4824 return handle_emulation_failure(vcpu); 4825 } 4826 4827 if (ctxt->have_exception) { 4828 inject_emulated_exception(vcpu); 4829 r = EMULATE_DONE; 4830 } else if (vcpu->arch.pio.count) { 4831 if (!vcpu->arch.pio.in) 4832 vcpu->arch.pio.count = 0; 4833 else 4834 writeback = false; 4835 r = EMULATE_DO_MMIO; 4836 } else if (vcpu->mmio_needed) { 4837 if (!vcpu->mmio_is_write) 4838 writeback = false; 4839 r = EMULATE_DO_MMIO; 4840 } else if (r == EMULATION_RESTART) 4841 goto restart; 4842 else 4843 r = EMULATE_DONE; 4844 4845 if (writeback) { 4846 toggle_interruptibility(vcpu, ctxt->interruptibility); 4847 kvm_set_rflags(vcpu, ctxt->eflags); 4848 kvm_make_request(KVM_REQ_EVENT, vcpu); 4849 memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs); 4850 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 4851 kvm_rip_write(vcpu, ctxt->eip); 4852 } else 4853 vcpu->arch.emulate_regs_need_sync_to_vcpu = true; 4854 4855 return r; 4856 } 4857 EXPORT_SYMBOL_GPL(x86_emulate_instruction); 4858 4859 int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port) 4860 { 4861 unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX); 4862 int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt, 4863 size, port, &val, 1); 4864 /* do not return to emulator after return from userspace */ 4865 vcpu->arch.pio.count = 0; 4866 return ret; 4867 } 4868 EXPORT_SYMBOL_GPL(kvm_fast_pio_out); 4869 4870 static void tsc_bad(void *info) 4871 { 4872 __this_cpu_write(cpu_tsc_khz, 0); 4873 } 4874 4875 static void tsc_khz_changed(void *data) 4876 { 4877 struct cpufreq_freqs *freq = data; 4878 unsigned long khz = 0; 4879 4880 if (data) 4881 khz = freq->new; 4882 else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 4883 khz = cpufreq_quick_get(raw_smp_processor_id()); 4884 if (!khz) 4885 khz = tsc_khz; 4886 __this_cpu_write(cpu_tsc_khz, khz); 4887 } 4888 4889 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 4890 void *data) 4891 { 4892 struct cpufreq_freqs *freq = data; 4893 struct kvm *kvm; 4894 struct kvm_vcpu *vcpu; 4895 int i, send_ipi = 0; 4896 4897 /* 4898 * We allow guests to temporarily run on slowing clocks, 4899 * provided we notify them after, or to run on accelerating 4900 * clocks, provided we notify them before. Thus time never 4901 * goes backwards. 4902 * 4903 * However, we have a problem. We can't atomically update 4904 * the frequency of a given CPU from this function; it is 4905 * merely a notifier, which can be called from any CPU. 4906 * Changing the TSC frequency at arbitrary points in time 4907 * requires a recomputation of local variables related to 4908 * the TSC for each VCPU. We must flag these local variables 4909 * to be updated and be sure the update takes place with the 4910 * new frequency before any guests proceed. 4911 * 4912 * Unfortunately, the combination of hotplug CPU and frequency 4913 * change creates an intractable locking scenario; the order 4914 * of when these callouts happen is undefined with respect to 4915 * CPU hotplug, and they can race with each other. As such, 4916 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is 4917 * undefined; you can actually have a CPU frequency change take 4918 * place in between the computation of X and the setting of the 4919 * variable. To protect against this problem, all updates of 4920 * the per_cpu tsc_khz variable are done in an interrupt 4921 * protected IPI, and all callers wishing to update the value 4922 * must wait for a synchronous IPI to complete (which is trivial 4923 * if the caller is on the CPU already). This establishes the 4924 * necessary total order on variable updates. 4925 * 4926 * Note that because a guest time update may take place 4927 * anytime after the setting of the VCPU's request bit, the 4928 * correct TSC value must be set before the request. However, 4929 * to ensure the update actually makes it to any guest which 4930 * starts running in hardware virtualization between the set 4931 * and the acquisition of the spinlock, we must also ping the 4932 * CPU after setting the request bit. 4933 * 4934 */ 4935 4936 if (val == CPUFREQ_PRECHANGE && freq->old > freq->new) 4937 return 0; 4938 if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new) 4939 return 0; 4940 4941 smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1); 4942 4943 raw_spin_lock(&kvm_lock); 4944 list_for_each_entry(kvm, &vm_list, vm_list) { 4945 kvm_for_each_vcpu(i, vcpu, kvm) { 4946 if (vcpu->cpu != freq->cpu) 4947 continue; 4948 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 4949 if (vcpu->cpu != smp_processor_id()) 4950 send_ipi = 1; 4951 } 4952 } 4953 raw_spin_unlock(&kvm_lock); 4954 4955 if (freq->old < freq->new && send_ipi) { 4956 /* 4957 * We upscale the frequency. Must make the guest 4958 * doesn't see old kvmclock values while running with 4959 * the new frequency, otherwise we risk the guest sees 4960 * time go backwards. 4961 * 4962 * In case we update the frequency for another cpu 4963 * (which might be in guest context) send an interrupt 4964 * to kick the cpu out of guest context. Next time 4965 * guest context is entered kvmclock will be updated, 4966 * so the guest will not see stale values. 4967 */ 4968 smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1); 4969 } 4970 return 0; 4971 } 4972 4973 static struct notifier_block kvmclock_cpufreq_notifier_block = { 4974 .notifier_call = kvmclock_cpufreq_notifier 4975 }; 4976 4977 static int kvmclock_cpu_notifier(struct notifier_block *nfb, 4978 unsigned long action, void *hcpu) 4979 { 4980 unsigned int cpu = (unsigned long)hcpu; 4981 4982 switch (action) { 4983 case CPU_ONLINE: 4984 case CPU_DOWN_FAILED: 4985 smp_call_function_single(cpu, tsc_khz_changed, NULL, 1); 4986 break; 4987 case CPU_DOWN_PREPARE: 4988 smp_call_function_single(cpu, tsc_bad, NULL, 1); 4989 break; 4990 } 4991 return NOTIFY_OK; 4992 } 4993 4994 static struct notifier_block kvmclock_cpu_notifier_block = { 4995 .notifier_call = kvmclock_cpu_notifier, 4996 .priority = -INT_MAX 4997 }; 4998 4999 static void kvm_timer_init(void) 5000 { 5001 int cpu; 5002 5003 max_tsc_khz = tsc_khz; 5004 register_hotcpu_notifier(&kvmclock_cpu_notifier_block); 5005 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 5006 #ifdef CONFIG_CPU_FREQ 5007 struct cpufreq_policy policy; 5008 memset(&policy, 0, sizeof(policy)); 5009 cpu = get_cpu(); 5010 cpufreq_get_policy(&policy, cpu); 5011 if (policy.cpuinfo.max_freq) 5012 max_tsc_khz = policy.cpuinfo.max_freq; 5013 put_cpu(); 5014 #endif 5015 cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block, 5016 CPUFREQ_TRANSITION_NOTIFIER); 5017 } 5018 pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz); 5019 for_each_online_cpu(cpu) 5020 smp_call_function_single(cpu, tsc_khz_changed, NULL, 1); 5021 } 5022 5023 static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu); 5024 5025 static int kvm_is_in_guest(void) 5026 { 5027 return percpu_read(current_vcpu) != NULL; 5028 } 5029 5030 static int kvm_is_user_mode(void) 5031 { 5032 int user_mode = 3; 5033 5034 if (percpu_read(current_vcpu)) 5035 user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu)); 5036 5037 return user_mode != 0; 5038 } 5039 5040 static unsigned long kvm_get_guest_ip(void) 5041 { 5042 unsigned long ip = 0; 5043 5044 if (percpu_read(current_vcpu)) 5045 ip = kvm_rip_read(percpu_read(current_vcpu)); 5046 5047 return ip; 5048 } 5049 5050 static struct perf_guest_info_callbacks kvm_guest_cbs = { 5051 .is_in_guest = kvm_is_in_guest, 5052 .is_user_mode = kvm_is_user_mode, 5053 .get_guest_ip = kvm_get_guest_ip, 5054 }; 5055 5056 void kvm_before_handle_nmi(struct kvm_vcpu *vcpu) 5057 { 5058 percpu_write(current_vcpu, vcpu); 5059 } 5060 EXPORT_SYMBOL_GPL(kvm_before_handle_nmi); 5061 5062 void kvm_after_handle_nmi(struct kvm_vcpu *vcpu) 5063 { 5064 percpu_write(current_vcpu, NULL); 5065 } 5066 EXPORT_SYMBOL_GPL(kvm_after_handle_nmi); 5067 5068 static void kvm_set_mmio_spte_mask(void) 5069 { 5070 u64 mask; 5071 int maxphyaddr = boot_cpu_data.x86_phys_bits; 5072 5073 /* 5074 * Set the reserved bits and the present bit of an paging-structure 5075 * entry to generate page fault with PFER.RSV = 1. 5076 */ 5077 mask = ((1ull << (62 - maxphyaddr + 1)) - 1) << maxphyaddr; 5078 mask |= 1ull; 5079 5080 #ifdef CONFIG_X86_64 5081 /* 5082 * If reserved bit is not supported, clear the present bit to disable 5083 * mmio page fault. 5084 */ 5085 if (maxphyaddr == 52) 5086 mask &= ~1ull; 5087 #endif 5088 5089 kvm_mmu_set_mmio_spte_mask(mask); 5090 } 5091 5092 int kvm_arch_init(void *opaque) 5093 { 5094 int r; 5095 struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; 5096 5097 if (kvm_x86_ops) { 5098 printk(KERN_ERR "kvm: already loaded the other module\n"); 5099 r = -EEXIST; 5100 goto out; 5101 } 5102 5103 if (!ops->cpu_has_kvm_support()) { 5104 printk(KERN_ERR "kvm: no hardware support\n"); 5105 r = -EOPNOTSUPP; 5106 goto out; 5107 } 5108 if (ops->disabled_by_bios()) { 5109 printk(KERN_ERR "kvm: disabled by bios\n"); 5110 r = -EOPNOTSUPP; 5111 goto out; 5112 } 5113 5114 r = kvm_mmu_module_init(); 5115 if (r) 5116 goto out; 5117 5118 kvm_set_mmio_spte_mask(); 5119 kvm_init_msr_list(); 5120 5121 kvm_x86_ops = ops; 5122 kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, 5123 PT_DIRTY_MASK, PT64_NX_MASK, 0); 5124 5125 kvm_timer_init(); 5126 5127 perf_register_guest_info_callbacks(&kvm_guest_cbs); 5128 5129 if (cpu_has_xsave) 5130 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 5131 5132 return 0; 5133 5134 out: 5135 return r; 5136 } 5137 5138 void kvm_arch_exit(void) 5139 { 5140 perf_unregister_guest_info_callbacks(&kvm_guest_cbs); 5141 5142 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 5143 cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block, 5144 CPUFREQ_TRANSITION_NOTIFIER); 5145 unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block); 5146 kvm_x86_ops = NULL; 5147 kvm_mmu_module_exit(); 5148 } 5149 5150 int kvm_emulate_halt(struct kvm_vcpu *vcpu) 5151 { 5152 ++vcpu->stat.halt_exits; 5153 if (irqchip_in_kernel(vcpu->kvm)) { 5154 vcpu->arch.mp_state = KVM_MP_STATE_HALTED; 5155 return 1; 5156 } else { 5157 vcpu->run->exit_reason = KVM_EXIT_HLT; 5158 return 0; 5159 } 5160 } 5161 EXPORT_SYMBOL_GPL(kvm_emulate_halt); 5162 5163 static inline gpa_t hc_gpa(struct kvm_vcpu *vcpu, unsigned long a0, 5164 unsigned long a1) 5165 { 5166 if (is_long_mode(vcpu)) 5167 return a0; 5168 else 5169 return a0 | ((gpa_t)a1 << 32); 5170 } 5171 5172 int kvm_hv_hypercall(struct kvm_vcpu *vcpu) 5173 { 5174 u64 param, ingpa, outgpa, ret; 5175 uint16_t code, rep_idx, rep_cnt, res = HV_STATUS_SUCCESS, rep_done = 0; 5176 bool fast, longmode; 5177 int cs_db, cs_l; 5178 5179 /* 5180 * hypercall generates UD from non zero cpl and real mode 5181 * per HYPER-V spec 5182 */ 5183 if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) { 5184 kvm_queue_exception(vcpu, UD_VECTOR); 5185 return 0; 5186 } 5187 5188 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); 5189 longmode = is_long_mode(vcpu) && cs_l == 1; 5190 5191 if (!longmode) { 5192 param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) | 5193 (kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff); 5194 ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) | 5195 (kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff); 5196 outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) | 5197 (kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff); 5198 } 5199 #ifdef CONFIG_X86_64 5200 else { 5201 param = kvm_register_read(vcpu, VCPU_REGS_RCX); 5202 ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX); 5203 outgpa = kvm_register_read(vcpu, VCPU_REGS_R8); 5204 } 5205 #endif 5206 5207 code = param & 0xffff; 5208 fast = (param >> 16) & 0x1; 5209 rep_cnt = (param >> 32) & 0xfff; 5210 rep_idx = (param >> 48) & 0xfff; 5211 5212 trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa); 5213 5214 switch (code) { 5215 case HV_X64_HV_NOTIFY_LONG_SPIN_WAIT: 5216 kvm_vcpu_on_spin(vcpu); 5217 break; 5218 default: 5219 res = HV_STATUS_INVALID_HYPERCALL_CODE; 5220 break; 5221 } 5222 5223 ret = res | (((u64)rep_done & 0xfff) << 32); 5224 if (longmode) { 5225 kvm_register_write(vcpu, VCPU_REGS_RAX, ret); 5226 } else { 5227 kvm_register_write(vcpu, VCPU_REGS_RDX, ret >> 32); 5228 kvm_register_write(vcpu, VCPU_REGS_RAX, ret & 0xffffffff); 5229 } 5230 5231 return 1; 5232 } 5233 5234 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) 5235 { 5236 unsigned long nr, a0, a1, a2, a3, ret; 5237 int r = 1; 5238 5239 if (kvm_hv_hypercall_enabled(vcpu->kvm)) 5240 return kvm_hv_hypercall(vcpu); 5241 5242 nr = kvm_register_read(vcpu, VCPU_REGS_RAX); 5243 a0 = kvm_register_read(vcpu, VCPU_REGS_RBX); 5244 a1 = kvm_register_read(vcpu, VCPU_REGS_RCX); 5245 a2 = kvm_register_read(vcpu, VCPU_REGS_RDX); 5246 a3 = kvm_register_read(vcpu, VCPU_REGS_RSI); 5247 5248 trace_kvm_hypercall(nr, a0, a1, a2, a3); 5249 5250 if (!is_long_mode(vcpu)) { 5251 nr &= 0xFFFFFFFF; 5252 a0 &= 0xFFFFFFFF; 5253 a1 &= 0xFFFFFFFF; 5254 a2 &= 0xFFFFFFFF; 5255 a3 &= 0xFFFFFFFF; 5256 } 5257 5258 if (kvm_x86_ops->get_cpl(vcpu) != 0) { 5259 ret = -KVM_EPERM; 5260 goto out; 5261 } 5262 5263 switch (nr) { 5264 case KVM_HC_VAPIC_POLL_IRQ: 5265 ret = 0; 5266 break; 5267 case KVM_HC_MMU_OP: 5268 r = kvm_pv_mmu_op(vcpu, a0, hc_gpa(vcpu, a1, a2), &ret); 5269 break; 5270 default: 5271 ret = -KVM_ENOSYS; 5272 break; 5273 } 5274 out: 5275 kvm_register_write(vcpu, VCPU_REGS_RAX, ret); 5276 ++vcpu->stat.hypercalls; 5277 return r; 5278 } 5279 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); 5280 5281 int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt) 5282 { 5283 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5284 char instruction[3]; 5285 unsigned long rip = kvm_rip_read(vcpu); 5286 5287 /* 5288 * Blow out the MMU to ensure that no other VCPU has an active mapping 5289 * to ensure that the updated hypercall appears atomically across all 5290 * VCPUs. 5291 */ 5292 kvm_mmu_zap_all(vcpu->kvm); 5293 5294 kvm_x86_ops->patch_hypercall(vcpu, instruction); 5295 5296 return emulator_write_emulated(ctxt, rip, instruction, 3, NULL); 5297 } 5298 5299 static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i) 5300 { 5301 struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i]; 5302 int j, nent = vcpu->arch.cpuid_nent; 5303 5304 e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT; 5305 /* when no next entry is found, the current entry[i] is reselected */ 5306 for (j = i + 1; ; j = (j + 1) % nent) { 5307 struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j]; 5308 if (ej->function == e->function) { 5309 ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; 5310 return j; 5311 } 5312 } 5313 return 0; /* silence gcc, even though control never reaches here */ 5314 } 5315 5316 /* find an entry with matching function, matching index (if needed), and that 5317 * should be read next (if it's stateful) */ 5318 static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e, 5319 u32 function, u32 index) 5320 { 5321 if (e->function != function) 5322 return 0; 5323 if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index) 5324 return 0; 5325 if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) && 5326 !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT)) 5327 return 0; 5328 return 1; 5329 } 5330 5331 struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu, 5332 u32 function, u32 index) 5333 { 5334 int i; 5335 struct kvm_cpuid_entry2 *best = NULL; 5336 5337 for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { 5338 struct kvm_cpuid_entry2 *e; 5339 5340 e = &vcpu->arch.cpuid_entries[i]; 5341 if (is_matching_cpuid_entry(e, function, index)) { 5342 if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) 5343 move_to_next_stateful_cpuid_entry(vcpu, i); 5344 best = e; 5345 break; 5346 } 5347 } 5348 return best; 5349 } 5350 EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry); 5351 5352 int cpuid_maxphyaddr(struct kvm_vcpu *vcpu) 5353 { 5354 struct kvm_cpuid_entry2 *best; 5355 5356 best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0); 5357 if (!best || best->eax < 0x80000008) 5358 goto not_found; 5359 best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0); 5360 if (best) 5361 return best->eax & 0xff; 5362 not_found: 5363 return 36; 5364 } 5365 5366 /* 5367 * If no match is found, check whether we exceed the vCPU's limit 5368 * and return the content of the highest valid _standard_ leaf instead. 5369 * This is to satisfy the CPUID specification. 5370 */ 5371 static struct kvm_cpuid_entry2* check_cpuid_limit(struct kvm_vcpu *vcpu, 5372 u32 function, u32 index) 5373 { 5374 struct kvm_cpuid_entry2 *maxlevel; 5375 5376 maxlevel = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0); 5377 if (!maxlevel || maxlevel->eax >= function) 5378 return NULL; 5379 if (function & 0x80000000) { 5380 maxlevel = kvm_find_cpuid_entry(vcpu, 0, 0); 5381 if (!maxlevel) 5382 return NULL; 5383 } 5384 return kvm_find_cpuid_entry(vcpu, maxlevel->eax, index); 5385 } 5386 5387 void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) 5388 { 5389 u32 function, index; 5390 struct kvm_cpuid_entry2 *best; 5391 5392 function = kvm_register_read(vcpu, VCPU_REGS_RAX); 5393 index = kvm_register_read(vcpu, VCPU_REGS_RCX); 5394 kvm_register_write(vcpu, VCPU_REGS_RAX, 0); 5395 kvm_register_write(vcpu, VCPU_REGS_RBX, 0); 5396 kvm_register_write(vcpu, VCPU_REGS_RCX, 0); 5397 kvm_register_write(vcpu, VCPU_REGS_RDX, 0); 5398 best = kvm_find_cpuid_entry(vcpu, function, index); 5399 5400 if (!best) 5401 best = check_cpuid_limit(vcpu, function, index); 5402 5403 if (best) { 5404 kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax); 5405 kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx); 5406 kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx); 5407 kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx); 5408 } 5409 kvm_x86_ops->skip_emulated_instruction(vcpu); 5410 trace_kvm_cpuid(function, 5411 kvm_register_read(vcpu, VCPU_REGS_RAX), 5412 kvm_register_read(vcpu, VCPU_REGS_RBX), 5413 kvm_register_read(vcpu, VCPU_REGS_RCX), 5414 kvm_register_read(vcpu, VCPU_REGS_RDX)); 5415 } 5416 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); 5417 5418 /* 5419 * Check if userspace requested an interrupt window, and that the 5420 * interrupt window is open. 5421 * 5422 * No need to exit to userspace if we already have an interrupt queued. 5423 */ 5424 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu) 5425 { 5426 return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) && 5427 vcpu->run->request_interrupt_window && 5428 kvm_arch_interrupt_allowed(vcpu)); 5429 } 5430 5431 static void post_kvm_run_save(struct kvm_vcpu *vcpu) 5432 { 5433 struct kvm_run *kvm_run = vcpu->run; 5434 5435 kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0; 5436 kvm_run->cr8 = kvm_get_cr8(vcpu); 5437 kvm_run->apic_base = kvm_get_apic_base(vcpu); 5438 if (irqchip_in_kernel(vcpu->kvm)) 5439 kvm_run->ready_for_interrupt_injection = 1; 5440 else 5441 kvm_run->ready_for_interrupt_injection = 5442 kvm_arch_interrupt_allowed(vcpu) && 5443 !kvm_cpu_has_interrupt(vcpu) && 5444 !kvm_event_needs_reinjection(vcpu); 5445 } 5446 5447 static void vapic_enter(struct kvm_vcpu *vcpu) 5448 { 5449 struct kvm_lapic *apic = vcpu->arch.apic; 5450 struct page *page; 5451 5452 if (!apic || !apic->vapic_addr) 5453 return; 5454 5455 page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); 5456 5457 vcpu->arch.apic->vapic_page = page; 5458 } 5459 5460 static void vapic_exit(struct kvm_vcpu *vcpu) 5461 { 5462 struct kvm_lapic *apic = vcpu->arch.apic; 5463 int idx; 5464 5465 if (!apic || !apic->vapic_addr) 5466 return; 5467 5468 idx = srcu_read_lock(&vcpu->kvm->srcu); 5469 kvm_release_page_dirty(apic->vapic_page); 5470 mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); 5471 srcu_read_unlock(&vcpu->kvm->srcu, idx); 5472 } 5473 5474 static void update_cr8_intercept(struct kvm_vcpu *vcpu) 5475 { 5476 int max_irr, tpr; 5477 5478 if (!kvm_x86_ops->update_cr8_intercept) 5479 return; 5480 5481 if (!vcpu->arch.apic) 5482 return; 5483 5484 if (!vcpu->arch.apic->vapic_addr) 5485 max_irr = kvm_lapic_find_highest_irr(vcpu); 5486 else 5487 max_irr = -1; 5488 5489 if (max_irr != -1) 5490 max_irr >>= 4; 5491 5492 tpr = kvm_lapic_get_cr8(vcpu); 5493 5494 kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr); 5495 } 5496 5497 static void inject_pending_event(struct kvm_vcpu *vcpu) 5498 { 5499 /* try to reinject previous events if any */ 5500 if (vcpu->arch.exception.pending) { 5501 trace_kvm_inj_exception(vcpu->arch.exception.nr, 5502 vcpu->arch.exception.has_error_code, 5503 vcpu->arch.exception.error_code); 5504 kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr, 5505 vcpu->arch.exception.has_error_code, 5506 vcpu->arch.exception.error_code, 5507 vcpu->arch.exception.reinject); 5508 return; 5509 } 5510 5511 if (vcpu->arch.nmi_injected) { 5512 kvm_x86_ops->set_nmi(vcpu); 5513 return; 5514 } 5515 5516 if (vcpu->arch.interrupt.pending) { 5517 kvm_x86_ops->set_irq(vcpu); 5518 return; 5519 } 5520 5521 /* try to inject new event if pending */ 5522 if (vcpu->arch.nmi_pending) { 5523 if (kvm_x86_ops->nmi_allowed(vcpu)) { 5524 vcpu->arch.nmi_pending = false; 5525 vcpu->arch.nmi_injected = true; 5526 kvm_x86_ops->set_nmi(vcpu); 5527 } 5528 } else if (kvm_cpu_has_interrupt(vcpu)) { 5529 if (kvm_x86_ops->interrupt_allowed(vcpu)) { 5530 kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), 5531 false); 5532 kvm_x86_ops->set_irq(vcpu); 5533 } 5534 } 5535 } 5536 5537 static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu) 5538 { 5539 if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) && 5540 !vcpu->guest_xcr0_loaded) { 5541 /* kvm_set_xcr() also depends on this */ 5542 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0); 5543 vcpu->guest_xcr0_loaded = 1; 5544 } 5545 } 5546 5547 static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu) 5548 { 5549 if (vcpu->guest_xcr0_loaded) { 5550 if (vcpu->arch.xcr0 != host_xcr0) 5551 xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0); 5552 vcpu->guest_xcr0_loaded = 0; 5553 } 5554 } 5555 5556 static int vcpu_enter_guest(struct kvm_vcpu *vcpu) 5557 { 5558 int r; 5559 bool nmi_pending; 5560 bool req_int_win = !irqchip_in_kernel(vcpu->kvm) && 5561 vcpu->run->request_interrupt_window; 5562 5563 if (vcpu->requests) { 5564 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) 5565 kvm_mmu_unload(vcpu); 5566 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu)) 5567 __kvm_migrate_timers(vcpu); 5568 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) { 5569 r = kvm_guest_time_update(vcpu); 5570 if (unlikely(r)) 5571 goto out; 5572 } 5573 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu)) 5574 kvm_mmu_sync_roots(vcpu); 5575 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 5576 kvm_x86_ops->tlb_flush(vcpu); 5577 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { 5578 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; 5579 r = 0; 5580 goto out; 5581 } 5582 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 5583 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 5584 r = 0; 5585 goto out; 5586 } 5587 if (kvm_check_request(KVM_REQ_DEACTIVATE_FPU, vcpu)) { 5588 vcpu->fpu_active = 0; 5589 kvm_x86_ops->fpu_deactivate(vcpu); 5590 } 5591 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) { 5592 /* Page is swapped out. Do synthetic halt */ 5593 vcpu->arch.apf.halted = true; 5594 r = 1; 5595 goto out; 5596 } 5597 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 5598 record_steal_time(vcpu); 5599 5600 } 5601 5602 r = kvm_mmu_reload(vcpu); 5603 if (unlikely(r)) 5604 goto out; 5605 5606 /* 5607 * An NMI can be injected between local nmi_pending read and 5608 * vcpu->arch.nmi_pending read inside inject_pending_event(). 5609 * But in that case, KVM_REQ_EVENT will be set, which makes 5610 * the race described above benign. 5611 */ 5612 nmi_pending = ACCESS_ONCE(vcpu->arch.nmi_pending); 5613 5614 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) { 5615 inject_pending_event(vcpu); 5616 5617 /* enable NMI/IRQ window open exits if needed */ 5618 if (nmi_pending) 5619 kvm_x86_ops->enable_nmi_window(vcpu); 5620 else if (kvm_cpu_has_interrupt(vcpu) || req_int_win) 5621 kvm_x86_ops->enable_irq_window(vcpu); 5622 5623 if (kvm_lapic_enabled(vcpu)) { 5624 update_cr8_intercept(vcpu); 5625 kvm_lapic_sync_to_vapic(vcpu); 5626 } 5627 } 5628 5629 preempt_disable(); 5630 5631 kvm_x86_ops->prepare_guest_switch(vcpu); 5632 if (vcpu->fpu_active) 5633 kvm_load_guest_fpu(vcpu); 5634 kvm_load_guest_xcr0(vcpu); 5635 5636 vcpu->mode = IN_GUEST_MODE; 5637 5638 /* We should set ->mode before check ->requests, 5639 * see the comment in make_all_cpus_request. 5640 */ 5641 smp_mb(); 5642 5643 local_irq_disable(); 5644 5645 if (vcpu->mode == EXITING_GUEST_MODE || vcpu->requests 5646 || need_resched() || signal_pending(current)) { 5647 vcpu->mode = OUTSIDE_GUEST_MODE; 5648 smp_wmb(); 5649 local_irq_enable(); 5650 preempt_enable(); 5651 kvm_x86_ops->cancel_injection(vcpu); 5652 r = 1; 5653 goto out; 5654 } 5655 5656 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 5657 5658 kvm_guest_enter(); 5659 5660 if (unlikely(vcpu->arch.switch_db_regs)) { 5661 set_debugreg(0, 7); 5662 set_debugreg(vcpu->arch.eff_db[0], 0); 5663 set_debugreg(vcpu->arch.eff_db[1], 1); 5664 set_debugreg(vcpu->arch.eff_db[2], 2); 5665 set_debugreg(vcpu->arch.eff_db[3], 3); 5666 } 5667 5668 trace_kvm_entry(vcpu->vcpu_id); 5669 kvm_x86_ops->run(vcpu); 5670 5671 /* 5672 * If the guest has used debug registers, at least dr7 5673 * will be disabled while returning to the host. 5674 * If we don't have active breakpoints in the host, we don't 5675 * care about the messed up debug address registers. But if 5676 * we have some of them active, restore the old state. 5677 */ 5678 if (hw_breakpoint_active()) 5679 hw_breakpoint_restore(); 5680 5681 kvm_get_msr(vcpu, MSR_IA32_TSC, &vcpu->arch.last_guest_tsc); 5682 5683 vcpu->mode = OUTSIDE_GUEST_MODE; 5684 smp_wmb(); 5685 local_irq_enable(); 5686 5687 ++vcpu->stat.exits; 5688 5689 /* 5690 * We must have an instruction between local_irq_enable() and 5691 * kvm_guest_exit(), so the timer interrupt isn't delayed by 5692 * the interrupt shadow. The stat.exits increment will do nicely. 5693 * But we need to prevent reordering, hence this barrier(): 5694 */ 5695 barrier(); 5696 5697 kvm_guest_exit(); 5698 5699 preempt_enable(); 5700 5701 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 5702 5703 /* 5704 * Profile KVM exit RIPs: 5705 */ 5706 if (unlikely(prof_on == KVM_PROFILING)) { 5707 unsigned long rip = kvm_rip_read(vcpu); 5708 profile_hit(KVM_PROFILING, (void *)rip); 5709 } 5710 5711 5712 kvm_lapic_sync_from_vapic(vcpu); 5713 5714 r = kvm_x86_ops->handle_exit(vcpu); 5715 out: 5716 return r; 5717 } 5718 5719 5720 static int __vcpu_run(struct kvm_vcpu *vcpu) 5721 { 5722 int r; 5723 struct kvm *kvm = vcpu->kvm; 5724 5725 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) { 5726 pr_debug("vcpu %d received sipi with vector # %x\n", 5727 vcpu->vcpu_id, vcpu->arch.sipi_vector); 5728 kvm_lapic_reset(vcpu); 5729 r = kvm_arch_vcpu_reset(vcpu); 5730 if (r) 5731 return r; 5732 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 5733 } 5734 5735 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); 5736 vapic_enter(vcpu); 5737 5738 r = 1; 5739 while (r > 0) { 5740 if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && 5741 !vcpu->arch.apf.halted) 5742 r = vcpu_enter_guest(vcpu); 5743 else { 5744 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 5745 kvm_vcpu_block(vcpu); 5746 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); 5747 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) 5748 { 5749 switch(vcpu->arch.mp_state) { 5750 case KVM_MP_STATE_HALTED: 5751 vcpu->arch.mp_state = 5752 KVM_MP_STATE_RUNNABLE; 5753 case KVM_MP_STATE_RUNNABLE: 5754 vcpu->arch.apf.halted = false; 5755 break; 5756 case KVM_MP_STATE_SIPI_RECEIVED: 5757 default: 5758 r = -EINTR; 5759 break; 5760 } 5761 } 5762 } 5763 5764 if (r <= 0) 5765 break; 5766 5767 clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests); 5768 if (kvm_cpu_has_pending_timer(vcpu)) 5769 kvm_inject_pending_timer_irqs(vcpu); 5770 5771 if (dm_request_for_irq_injection(vcpu)) { 5772 r = -EINTR; 5773 vcpu->run->exit_reason = KVM_EXIT_INTR; 5774 ++vcpu->stat.request_irq_exits; 5775 } 5776 5777 kvm_check_async_pf_completion(vcpu); 5778 5779 if (signal_pending(current)) { 5780 r = -EINTR; 5781 vcpu->run->exit_reason = KVM_EXIT_INTR; 5782 ++vcpu->stat.signal_exits; 5783 } 5784 if (need_resched()) { 5785 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 5786 kvm_resched(vcpu); 5787 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); 5788 } 5789 } 5790 5791 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 5792 5793 vapic_exit(vcpu); 5794 5795 return r; 5796 } 5797 5798 static int complete_mmio(struct kvm_vcpu *vcpu) 5799 { 5800 struct kvm_run *run = vcpu->run; 5801 int r; 5802 5803 if (!(vcpu->arch.pio.count || vcpu->mmio_needed)) 5804 return 1; 5805 5806 if (vcpu->mmio_needed) { 5807 vcpu->mmio_needed = 0; 5808 if (!vcpu->mmio_is_write) 5809 memcpy(vcpu->mmio_data + vcpu->mmio_index, 5810 run->mmio.data, 8); 5811 vcpu->mmio_index += 8; 5812 if (vcpu->mmio_index < vcpu->mmio_size) { 5813 run->exit_reason = KVM_EXIT_MMIO; 5814 run->mmio.phys_addr = vcpu->mmio_phys_addr + vcpu->mmio_index; 5815 memcpy(run->mmio.data, vcpu->mmio_data + vcpu->mmio_index, 8); 5816 run->mmio.len = min(vcpu->mmio_size - vcpu->mmio_index, 8); 5817 run->mmio.is_write = vcpu->mmio_is_write; 5818 vcpu->mmio_needed = 1; 5819 return 0; 5820 } 5821 if (vcpu->mmio_is_write) 5822 return 1; 5823 vcpu->mmio_read_completed = 1; 5824 } 5825 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 5826 r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE); 5827 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 5828 if (r != EMULATE_DONE) 5829 return 0; 5830 return 1; 5831 } 5832 5833 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 5834 { 5835 int r; 5836 sigset_t sigsaved; 5837 5838 if (!tsk_used_math(current) && init_fpu(current)) 5839 return -ENOMEM; 5840 5841 if (vcpu->sigset_active) 5842 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); 5843 5844 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { 5845 kvm_vcpu_block(vcpu); 5846 clear_bit(KVM_REQ_UNHALT, &vcpu->requests); 5847 r = -EAGAIN; 5848 goto out; 5849 } 5850 5851 /* re-sync apic's tpr */ 5852 if (!irqchip_in_kernel(vcpu->kvm)) { 5853 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) { 5854 r = -EINVAL; 5855 goto out; 5856 } 5857 } 5858 5859 r = complete_mmio(vcpu); 5860 if (r <= 0) 5861 goto out; 5862 5863 if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) 5864 kvm_register_write(vcpu, VCPU_REGS_RAX, 5865 kvm_run->hypercall.ret); 5866 5867 r = __vcpu_run(vcpu); 5868 5869 out: 5870 post_kvm_run_save(vcpu); 5871 if (vcpu->sigset_active) 5872 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 5873 5874 return r; 5875 } 5876 5877 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 5878 { 5879 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { 5880 /* 5881 * We are here if userspace calls get_regs() in the middle of 5882 * instruction emulation. Registers state needs to be copied 5883 * back from emulation context to vcpu. Usrapace shouldn't do 5884 * that usually, but some bad designed PV devices (vmware 5885 * backdoor interface) need this to work 5886 */ 5887 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 5888 memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs); 5889 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 5890 } 5891 regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX); 5892 regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX); 5893 regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX); 5894 regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX); 5895 regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI); 5896 regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI); 5897 regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP); 5898 regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP); 5899 #ifdef CONFIG_X86_64 5900 regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8); 5901 regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9); 5902 regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10); 5903 regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11); 5904 regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12); 5905 regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13); 5906 regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14); 5907 regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15); 5908 #endif 5909 5910 regs->rip = kvm_rip_read(vcpu); 5911 regs->rflags = kvm_get_rflags(vcpu); 5912 5913 return 0; 5914 } 5915 5916 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 5917 { 5918 vcpu->arch.emulate_regs_need_sync_from_vcpu = true; 5919 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 5920 5921 kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax); 5922 kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx); 5923 kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx); 5924 kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx); 5925 kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi); 5926 kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi); 5927 kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp); 5928 kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp); 5929 #ifdef CONFIG_X86_64 5930 kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8); 5931 kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9); 5932 kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10); 5933 kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11); 5934 kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12); 5935 kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13); 5936 kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14); 5937 kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15); 5938 #endif 5939 5940 kvm_rip_write(vcpu, regs->rip); 5941 kvm_set_rflags(vcpu, regs->rflags); 5942 5943 vcpu->arch.exception.pending = false; 5944 5945 kvm_make_request(KVM_REQ_EVENT, vcpu); 5946 5947 return 0; 5948 } 5949 5950 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 5951 { 5952 struct kvm_segment cs; 5953 5954 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS); 5955 *db = cs.db; 5956 *l = cs.l; 5957 } 5958 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); 5959 5960 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 5961 struct kvm_sregs *sregs) 5962 { 5963 struct desc_ptr dt; 5964 5965 kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); 5966 kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); 5967 kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); 5968 kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); 5969 kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); 5970 kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); 5971 5972 kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); 5973 kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); 5974 5975 kvm_x86_ops->get_idt(vcpu, &dt); 5976 sregs->idt.limit = dt.size; 5977 sregs->idt.base = dt.address; 5978 kvm_x86_ops->get_gdt(vcpu, &dt); 5979 sregs->gdt.limit = dt.size; 5980 sregs->gdt.base = dt.address; 5981 5982 sregs->cr0 = kvm_read_cr0(vcpu); 5983 sregs->cr2 = vcpu->arch.cr2; 5984 sregs->cr3 = kvm_read_cr3(vcpu); 5985 sregs->cr4 = kvm_read_cr4(vcpu); 5986 sregs->cr8 = kvm_get_cr8(vcpu); 5987 sregs->efer = vcpu->arch.efer; 5988 sregs->apic_base = kvm_get_apic_base(vcpu); 5989 5990 memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap); 5991 5992 if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft) 5993 set_bit(vcpu->arch.interrupt.nr, 5994 (unsigned long *)sregs->interrupt_bitmap); 5995 5996 return 0; 5997 } 5998 5999 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 6000 struct kvm_mp_state *mp_state) 6001 { 6002 mp_state->mp_state = vcpu->arch.mp_state; 6003 return 0; 6004 } 6005 6006 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 6007 struct kvm_mp_state *mp_state) 6008 { 6009 vcpu->arch.mp_state = mp_state->mp_state; 6010 kvm_make_request(KVM_REQ_EVENT, vcpu); 6011 return 0; 6012 } 6013 6014 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason, 6015 bool has_error_code, u32 error_code) 6016 { 6017 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 6018 int ret; 6019 6020 init_emulate_ctxt(vcpu); 6021 6022 ret = emulator_task_switch(ctxt, tss_selector, reason, 6023 has_error_code, error_code); 6024 6025 if (ret) 6026 return EMULATE_FAIL; 6027 6028 memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs); 6029 kvm_rip_write(vcpu, ctxt->eip); 6030 kvm_set_rflags(vcpu, ctxt->eflags); 6031 kvm_make_request(KVM_REQ_EVENT, vcpu); 6032 return EMULATE_DONE; 6033 } 6034 EXPORT_SYMBOL_GPL(kvm_task_switch); 6035 6036 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 6037 struct kvm_sregs *sregs) 6038 { 6039 int mmu_reset_needed = 0; 6040 int pending_vec, max_bits, idx; 6041 struct desc_ptr dt; 6042 6043 dt.size = sregs->idt.limit; 6044 dt.address = sregs->idt.base; 6045 kvm_x86_ops->set_idt(vcpu, &dt); 6046 dt.size = sregs->gdt.limit; 6047 dt.address = sregs->gdt.base; 6048 kvm_x86_ops->set_gdt(vcpu, &dt); 6049 6050 vcpu->arch.cr2 = sregs->cr2; 6051 mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; 6052 vcpu->arch.cr3 = sregs->cr3; 6053 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 6054 6055 kvm_set_cr8(vcpu, sregs->cr8); 6056 6057 mmu_reset_needed |= vcpu->arch.efer != sregs->efer; 6058 kvm_x86_ops->set_efer(vcpu, sregs->efer); 6059 kvm_set_apic_base(vcpu, sregs->apic_base); 6060 6061 mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; 6062 kvm_x86_ops->set_cr0(vcpu, sregs->cr0); 6063 vcpu->arch.cr0 = sregs->cr0; 6064 6065 mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; 6066 kvm_x86_ops->set_cr4(vcpu, sregs->cr4); 6067 if (sregs->cr4 & X86_CR4_OSXSAVE) 6068 update_cpuid(vcpu); 6069 6070 idx = srcu_read_lock(&vcpu->kvm->srcu); 6071 if (!is_long_mode(vcpu) && is_pae(vcpu)) { 6072 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); 6073 mmu_reset_needed = 1; 6074 } 6075 srcu_read_unlock(&vcpu->kvm->srcu, idx); 6076 6077 if (mmu_reset_needed) 6078 kvm_mmu_reset_context(vcpu); 6079 6080 max_bits = (sizeof sregs->interrupt_bitmap) << 3; 6081 pending_vec = find_first_bit( 6082 (const unsigned long *)sregs->interrupt_bitmap, max_bits); 6083 if (pending_vec < max_bits) { 6084 kvm_queue_interrupt(vcpu, pending_vec, false); 6085 pr_debug("Set back pending irq %d\n", pending_vec); 6086 } 6087 6088 kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); 6089 kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); 6090 kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); 6091 kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); 6092 kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); 6093 kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); 6094 6095 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); 6096 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); 6097 6098 update_cr8_intercept(vcpu); 6099 6100 /* Older userspace won't unhalt the vcpu on reset. */ 6101 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && 6102 sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && 6103 !is_protmode(vcpu)) 6104 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 6105 6106 kvm_make_request(KVM_REQ_EVENT, vcpu); 6107 6108 return 0; 6109 } 6110 6111 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 6112 struct kvm_guest_debug *dbg) 6113 { 6114 unsigned long rflags; 6115 int i, r; 6116 6117 if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) { 6118 r = -EBUSY; 6119 if (vcpu->arch.exception.pending) 6120 goto out; 6121 if (dbg->control & KVM_GUESTDBG_INJECT_DB) 6122 kvm_queue_exception(vcpu, DB_VECTOR); 6123 else 6124 kvm_queue_exception(vcpu, BP_VECTOR); 6125 } 6126 6127 /* 6128 * Read rflags as long as potentially injected trace flags are still 6129 * filtered out. 6130 */ 6131 rflags = kvm_get_rflags(vcpu); 6132 6133 vcpu->guest_debug = dbg->control; 6134 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE)) 6135 vcpu->guest_debug = 0; 6136 6137 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 6138 for (i = 0; i < KVM_NR_DB_REGS; ++i) 6139 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; 6140 vcpu->arch.switch_db_regs = 6141 (dbg->arch.debugreg[7] & DR7_BP_EN_MASK); 6142 } else { 6143 for (i = 0; i < KVM_NR_DB_REGS; i++) 6144 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 6145 vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK); 6146 } 6147 6148 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 6149 vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) + 6150 get_segment_base(vcpu, VCPU_SREG_CS); 6151 6152 /* 6153 * Trigger an rflags update that will inject or remove the trace 6154 * flags. 6155 */ 6156 kvm_set_rflags(vcpu, rflags); 6157 6158 kvm_x86_ops->set_guest_debug(vcpu, dbg); 6159 6160 r = 0; 6161 6162 out: 6163 6164 return r; 6165 } 6166 6167 /* 6168 * Translate a guest virtual address to a guest physical address. 6169 */ 6170 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 6171 struct kvm_translation *tr) 6172 { 6173 unsigned long vaddr = tr->linear_address; 6174 gpa_t gpa; 6175 int idx; 6176 6177 idx = srcu_read_lock(&vcpu->kvm->srcu); 6178 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL); 6179 srcu_read_unlock(&vcpu->kvm->srcu, idx); 6180 tr->physical_address = gpa; 6181 tr->valid = gpa != UNMAPPED_GVA; 6182 tr->writeable = 1; 6183 tr->usermode = 0; 6184 6185 return 0; 6186 } 6187 6188 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 6189 { 6190 struct i387_fxsave_struct *fxsave = 6191 &vcpu->arch.guest_fpu.state->fxsave; 6192 6193 memcpy(fpu->fpr, fxsave->st_space, 128); 6194 fpu->fcw = fxsave->cwd; 6195 fpu->fsw = fxsave->swd; 6196 fpu->ftwx = fxsave->twd; 6197 fpu->last_opcode = fxsave->fop; 6198 fpu->last_ip = fxsave->rip; 6199 fpu->last_dp = fxsave->rdp; 6200 memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space); 6201 6202 return 0; 6203 } 6204 6205 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 6206 { 6207 struct i387_fxsave_struct *fxsave = 6208 &vcpu->arch.guest_fpu.state->fxsave; 6209 6210 memcpy(fxsave->st_space, fpu->fpr, 128); 6211 fxsave->cwd = fpu->fcw; 6212 fxsave->swd = fpu->fsw; 6213 fxsave->twd = fpu->ftwx; 6214 fxsave->fop = fpu->last_opcode; 6215 fxsave->rip = fpu->last_ip; 6216 fxsave->rdp = fpu->last_dp; 6217 memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space); 6218 6219 return 0; 6220 } 6221 6222 int fx_init(struct kvm_vcpu *vcpu) 6223 { 6224 int err; 6225 6226 err = fpu_alloc(&vcpu->arch.guest_fpu); 6227 if (err) 6228 return err; 6229 6230 fpu_finit(&vcpu->arch.guest_fpu); 6231 6232 /* 6233 * Ensure guest xcr0 is valid for loading 6234 */ 6235 vcpu->arch.xcr0 = XSTATE_FP; 6236 6237 vcpu->arch.cr0 |= X86_CR0_ET; 6238 6239 return 0; 6240 } 6241 EXPORT_SYMBOL_GPL(fx_init); 6242 6243 static void fx_free(struct kvm_vcpu *vcpu) 6244 { 6245 fpu_free(&vcpu->arch.guest_fpu); 6246 } 6247 6248 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) 6249 { 6250 if (vcpu->guest_fpu_loaded) 6251 return; 6252 6253 /* 6254 * Restore all possible states in the guest, 6255 * and assume host would use all available bits. 6256 * Guest xcr0 would be loaded later. 6257 */ 6258 kvm_put_guest_xcr0(vcpu); 6259 vcpu->guest_fpu_loaded = 1; 6260 unlazy_fpu(current); 6261 fpu_restore_checking(&vcpu->arch.guest_fpu); 6262 trace_kvm_fpu(1); 6263 } 6264 6265 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) 6266 { 6267 kvm_put_guest_xcr0(vcpu); 6268 6269 if (!vcpu->guest_fpu_loaded) 6270 return; 6271 6272 vcpu->guest_fpu_loaded = 0; 6273 fpu_save_init(&vcpu->arch.guest_fpu); 6274 ++vcpu->stat.fpu_reload; 6275 kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu); 6276 trace_kvm_fpu(0); 6277 } 6278 6279 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) 6280 { 6281 kvmclock_reset(vcpu); 6282 6283 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); 6284 fx_free(vcpu); 6285 kvm_x86_ops->vcpu_free(vcpu); 6286 } 6287 6288 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, 6289 unsigned int id) 6290 { 6291 if (check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0) 6292 printk_once(KERN_WARNING 6293 "kvm: SMP vm created on host with unstable TSC; " 6294 "guest TSC will not be reliable\n"); 6295 return kvm_x86_ops->vcpu_create(kvm, id); 6296 } 6297 6298 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 6299 { 6300 int r; 6301 6302 vcpu->arch.mtrr_state.have_fixed = 1; 6303 vcpu_load(vcpu); 6304 r = kvm_arch_vcpu_reset(vcpu); 6305 if (r == 0) 6306 r = kvm_mmu_setup(vcpu); 6307 vcpu_put(vcpu); 6308 6309 return r; 6310 } 6311 6312 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 6313 { 6314 vcpu->arch.apf.msr_val = 0; 6315 6316 vcpu_load(vcpu); 6317 kvm_mmu_unload(vcpu); 6318 vcpu_put(vcpu); 6319 6320 fx_free(vcpu); 6321 kvm_x86_ops->vcpu_free(vcpu); 6322 } 6323 6324 int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) 6325 { 6326 vcpu->arch.nmi_pending = false; 6327 vcpu->arch.nmi_injected = false; 6328 6329 vcpu->arch.switch_db_regs = 0; 6330 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); 6331 vcpu->arch.dr6 = DR6_FIXED_1; 6332 vcpu->arch.dr7 = DR7_FIXED_1; 6333 6334 kvm_make_request(KVM_REQ_EVENT, vcpu); 6335 vcpu->arch.apf.msr_val = 0; 6336 vcpu->arch.st.msr_val = 0; 6337 6338 kvmclock_reset(vcpu); 6339 6340 kvm_clear_async_pf_completion_queue(vcpu); 6341 kvm_async_pf_hash_reset(vcpu); 6342 vcpu->arch.apf.halted = false; 6343 6344 return kvm_x86_ops->vcpu_reset(vcpu); 6345 } 6346 6347 int kvm_arch_hardware_enable(void *garbage) 6348 { 6349 struct kvm *kvm; 6350 struct kvm_vcpu *vcpu; 6351 int i; 6352 6353 kvm_shared_msr_cpu_online(); 6354 list_for_each_entry(kvm, &vm_list, vm_list) 6355 kvm_for_each_vcpu(i, vcpu, kvm) 6356 if (vcpu->cpu == smp_processor_id()) 6357 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 6358 return kvm_x86_ops->hardware_enable(garbage); 6359 } 6360 6361 void kvm_arch_hardware_disable(void *garbage) 6362 { 6363 kvm_x86_ops->hardware_disable(garbage); 6364 drop_user_return_notifiers(garbage); 6365 } 6366 6367 int kvm_arch_hardware_setup(void) 6368 { 6369 return kvm_x86_ops->hardware_setup(); 6370 } 6371 6372 void kvm_arch_hardware_unsetup(void) 6373 { 6374 kvm_x86_ops->hardware_unsetup(); 6375 } 6376 6377 void kvm_arch_check_processor_compat(void *rtn) 6378 { 6379 kvm_x86_ops->check_processor_compatibility(rtn); 6380 } 6381 6382 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 6383 { 6384 struct page *page; 6385 struct kvm *kvm; 6386 int r; 6387 6388 BUG_ON(vcpu->kvm == NULL); 6389 kvm = vcpu->kvm; 6390 6391 vcpu->arch.emulate_ctxt.ops = &emulate_ops; 6392 vcpu->arch.walk_mmu = &vcpu->arch.mmu; 6393 vcpu->arch.mmu.root_hpa = INVALID_PAGE; 6394 vcpu->arch.mmu.translate_gpa = translate_gpa; 6395 vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa; 6396 if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu)) 6397 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 6398 else 6399 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED; 6400 6401 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 6402 if (!page) { 6403 r = -ENOMEM; 6404 goto fail; 6405 } 6406 vcpu->arch.pio_data = page_address(page); 6407 6408 kvm_init_tsc_catchup(vcpu, max_tsc_khz); 6409 6410 r = kvm_mmu_create(vcpu); 6411 if (r < 0) 6412 goto fail_free_pio_data; 6413 6414 if (irqchip_in_kernel(kvm)) { 6415 r = kvm_create_lapic(vcpu); 6416 if (r < 0) 6417 goto fail_mmu_destroy; 6418 } 6419 6420 vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4, 6421 GFP_KERNEL); 6422 if (!vcpu->arch.mce_banks) { 6423 r = -ENOMEM; 6424 goto fail_free_lapic; 6425 } 6426 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS; 6427 6428 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL)) 6429 goto fail_free_mce_banks; 6430 6431 kvm_async_pf_hash_reset(vcpu); 6432 6433 return 0; 6434 fail_free_mce_banks: 6435 kfree(vcpu->arch.mce_banks); 6436 fail_free_lapic: 6437 kvm_free_lapic(vcpu); 6438 fail_mmu_destroy: 6439 kvm_mmu_destroy(vcpu); 6440 fail_free_pio_data: 6441 free_page((unsigned long)vcpu->arch.pio_data); 6442 fail: 6443 return r; 6444 } 6445 6446 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) 6447 { 6448 int idx; 6449 6450 kfree(vcpu->arch.mce_banks); 6451 kvm_free_lapic(vcpu); 6452 idx = srcu_read_lock(&vcpu->kvm->srcu); 6453 kvm_mmu_destroy(vcpu); 6454 srcu_read_unlock(&vcpu->kvm->srcu, idx); 6455 free_page((unsigned long)vcpu->arch.pio_data); 6456 } 6457 6458 int kvm_arch_init_vm(struct kvm *kvm) 6459 { 6460 INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); 6461 INIT_LIST_HEAD(&kvm->arch.assigned_dev_head); 6462 6463 /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ 6464 set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); 6465 6466 raw_spin_lock_init(&kvm->arch.tsc_write_lock); 6467 6468 return 0; 6469 } 6470 6471 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) 6472 { 6473 vcpu_load(vcpu); 6474 kvm_mmu_unload(vcpu); 6475 vcpu_put(vcpu); 6476 } 6477 6478 static void kvm_free_vcpus(struct kvm *kvm) 6479 { 6480 unsigned int i; 6481 struct kvm_vcpu *vcpu; 6482 6483 /* 6484 * Unpin any mmu pages first. 6485 */ 6486 kvm_for_each_vcpu(i, vcpu, kvm) { 6487 kvm_clear_async_pf_completion_queue(vcpu); 6488 kvm_unload_vcpu_mmu(vcpu); 6489 } 6490 kvm_for_each_vcpu(i, vcpu, kvm) 6491 kvm_arch_vcpu_free(vcpu); 6492 6493 mutex_lock(&kvm->lock); 6494 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 6495 kvm->vcpus[i] = NULL; 6496 6497 atomic_set(&kvm->online_vcpus, 0); 6498 mutex_unlock(&kvm->lock); 6499 } 6500 6501 void kvm_arch_sync_events(struct kvm *kvm) 6502 { 6503 kvm_free_all_assigned_devices(kvm); 6504 kvm_free_pit(kvm); 6505 } 6506 6507 void kvm_arch_destroy_vm(struct kvm *kvm) 6508 { 6509 kvm_iommu_unmap_guest(kvm); 6510 kfree(kvm->arch.vpic); 6511 kfree(kvm->arch.vioapic); 6512 kvm_free_vcpus(kvm); 6513 if (kvm->arch.apic_access_page) 6514 put_page(kvm->arch.apic_access_page); 6515 if (kvm->arch.ept_identity_pagetable) 6516 put_page(kvm->arch.ept_identity_pagetable); 6517 } 6518 6519 int kvm_arch_prepare_memory_region(struct kvm *kvm, 6520 struct kvm_memory_slot *memslot, 6521 struct kvm_memory_slot old, 6522 struct kvm_userspace_memory_region *mem, 6523 int user_alloc) 6524 { 6525 int npages = memslot->npages; 6526 int map_flags = MAP_PRIVATE | MAP_ANONYMOUS; 6527 6528 /* Prevent internal slot pages from being moved by fork()/COW. */ 6529 if (memslot->id >= KVM_MEMORY_SLOTS) 6530 map_flags = MAP_SHARED | MAP_ANONYMOUS; 6531 6532 /*To keep backward compatibility with older userspace, 6533 *x86 needs to hanlde !user_alloc case. 6534 */ 6535 if (!user_alloc) { 6536 if (npages && !old.rmap) { 6537 unsigned long userspace_addr; 6538 6539 down_write(¤t->mm->mmap_sem); 6540 userspace_addr = do_mmap(NULL, 0, 6541 npages * PAGE_SIZE, 6542 PROT_READ | PROT_WRITE, 6543 map_flags, 6544 0); 6545 up_write(¤t->mm->mmap_sem); 6546 6547 if (IS_ERR((void *)userspace_addr)) 6548 return PTR_ERR((void *)userspace_addr); 6549 6550 memslot->userspace_addr = userspace_addr; 6551 } 6552 } 6553 6554 6555 return 0; 6556 } 6557 6558 void kvm_arch_commit_memory_region(struct kvm *kvm, 6559 struct kvm_userspace_memory_region *mem, 6560 struct kvm_memory_slot old, 6561 int user_alloc) 6562 { 6563 6564 int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT; 6565 6566 if (!user_alloc && !old.user_alloc && old.rmap && !npages) { 6567 int ret; 6568 6569 down_write(¤t->mm->mmap_sem); 6570 ret = do_munmap(current->mm, old.userspace_addr, 6571 old.npages * PAGE_SIZE); 6572 up_write(¤t->mm->mmap_sem); 6573 if (ret < 0) 6574 printk(KERN_WARNING 6575 "kvm_vm_ioctl_set_memory_region: " 6576 "failed to munmap memory\n"); 6577 } 6578 6579 if (!kvm->arch.n_requested_mmu_pages) 6580 nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); 6581 6582 spin_lock(&kvm->mmu_lock); 6583 if (nr_mmu_pages) 6584 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); 6585 kvm_mmu_slot_remove_write_access(kvm, mem->slot); 6586 spin_unlock(&kvm->mmu_lock); 6587 } 6588 6589 void kvm_arch_flush_shadow(struct kvm *kvm) 6590 { 6591 kvm_mmu_zap_all(kvm); 6592 kvm_reload_remote_mmus(kvm); 6593 } 6594 6595 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 6596 { 6597 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && 6598 !vcpu->arch.apf.halted) 6599 || !list_empty_careful(&vcpu->async_pf.done) 6600 || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED 6601 || vcpu->arch.nmi_pending || 6602 (kvm_arch_interrupt_allowed(vcpu) && 6603 kvm_cpu_has_interrupt(vcpu)); 6604 } 6605 6606 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 6607 { 6608 int me; 6609 int cpu = vcpu->cpu; 6610 6611 if (waitqueue_active(&vcpu->wq)) { 6612 wake_up_interruptible(&vcpu->wq); 6613 ++vcpu->stat.halt_wakeup; 6614 } 6615 6616 me = get_cpu(); 6617 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 6618 if (kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE) 6619 smp_send_reschedule(cpu); 6620 put_cpu(); 6621 } 6622 6623 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu) 6624 { 6625 return kvm_x86_ops->interrupt_allowed(vcpu); 6626 } 6627 6628 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip) 6629 { 6630 unsigned long current_rip = kvm_rip_read(vcpu) + 6631 get_segment_base(vcpu, VCPU_SREG_CS); 6632 6633 return current_rip == linear_rip; 6634 } 6635 EXPORT_SYMBOL_GPL(kvm_is_linear_rip); 6636 6637 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) 6638 { 6639 unsigned long rflags; 6640 6641 rflags = kvm_x86_ops->get_rflags(vcpu); 6642 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 6643 rflags &= ~X86_EFLAGS_TF; 6644 return rflags; 6645 } 6646 EXPORT_SYMBOL_GPL(kvm_get_rflags); 6647 6648 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 6649 { 6650 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && 6651 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) 6652 rflags |= X86_EFLAGS_TF; 6653 kvm_x86_ops->set_rflags(vcpu, rflags); 6654 kvm_make_request(KVM_REQ_EVENT, vcpu); 6655 } 6656 EXPORT_SYMBOL_GPL(kvm_set_rflags); 6657 6658 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) 6659 { 6660 int r; 6661 6662 if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) || 6663 is_error_page(work->page)) 6664 return; 6665 6666 r = kvm_mmu_reload(vcpu); 6667 if (unlikely(r)) 6668 return; 6669 6670 if (!vcpu->arch.mmu.direct_map && 6671 work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu)) 6672 return; 6673 6674 vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true); 6675 } 6676 6677 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) 6678 { 6679 return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU)); 6680 } 6681 6682 static inline u32 kvm_async_pf_next_probe(u32 key) 6683 { 6684 return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1); 6685 } 6686 6687 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 6688 { 6689 u32 key = kvm_async_pf_hash_fn(gfn); 6690 6691 while (vcpu->arch.apf.gfns[key] != ~0) 6692 key = kvm_async_pf_next_probe(key); 6693 6694 vcpu->arch.apf.gfns[key] = gfn; 6695 } 6696 6697 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn) 6698 { 6699 int i; 6700 u32 key = kvm_async_pf_hash_fn(gfn); 6701 6702 for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) && 6703 (vcpu->arch.apf.gfns[key] != gfn && 6704 vcpu->arch.apf.gfns[key] != ~0); i++) 6705 key = kvm_async_pf_next_probe(key); 6706 6707 return key; 6708 } 6709 6710 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 6711 { 6712 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn; 6713 } 6714 6715 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 6716 { 6717 u32 i, j, k; 6718 6719 i = j = kvm_async_pf_gfn_slot(vcpu, gfn); 6720 while (true) { 6721 vcpu->arch.apf.gfns[i] = ~0; 6722 do { 6723 j = kvm_async_pf_next_probe(j); 6724 if (vcpu->arch.apf.gfns[j] == ~0) 6725 return; 6726 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]); 6727 /* 6728 * k lies cyclically in ]i,j] 6729 * | i.k.j | 6730 * |....j i.k.| or |.k..j i...| 6731 */ 6732 } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j)); 6733 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j]; 6734 i = j; 6735 } 6736 } 6737 6738 static int apf_put_user(struct kvm_vcpu *vcpu, u32 val) 6739 { 6740 6741 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val, 6742 sizeof(val)); 6743 } 6744 6745 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 6746 struct kvm_async_pf *work) 6747 { 6748 struct x86_exception fault; 6749 6750 trace_kvm_async_pf_not_present(work->arch.token, work->gva); 6751 kvm_add_async_pf_gfn(vcpu, work->arch.gfn); 6752 6753 if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) || 6754 (vcpu->arch.apf.send_user_only && 6755 kvm_x86_ops->get_cpl(vcpu) == 0)) 6756 kvm_make_request(KVM_REQ_APF_HALT, vcpu); 6757 else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) { 6758 fault.vector = PF_VECTOR; 6759 fault.error_code_valid = true; 6760 fault.error_code = 0; 6761 fault.nested_page_fault = false; 6762 fault.address = work->arch.token; 6763 kvm_inject_page_fault(vcpu, &fault); 6764 } 6765 } 6766 6767 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 6768 struct kvm_async_pf *work) 6769 { 6770 struct x86_exception fault; 6771 6772 trace_kvm_async_pf_ready(work->arch.token, work->gva); 6773 if (is_error_page(work->page)) 6774 work->arch.token = ~0; /* broadcast wakeup */ 6775 else 6776 kvm_del_async_pf_gfn(vcpu, work->arch.gfn); 6777 6778 if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) && 6779 !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) { 6780 fault.vector = PF_VECTOR; 6781 fault.error_code_valid = true; 6782 fault.error_code = 0; 6783 fault.nested_page_fault = false; 6784 fault.address = work->arch.token; 6785 kvm_inject_page_fault(vcpu, &fault); 6786 } 6787 vcpu->arch.apf.halted = false; 6788 } 6789 6790 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) 6791 { 6792 if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED)) 6793 return true; 6794 else 6795 return !kvm_event_needs_reinjection(vcpu) && 6796 kvm_x86_ops->interrupt_allowed(vcpu); 6797 } 6798 6799 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit); 6800 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); 6801 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); 6802 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr); 6803 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr); 6804 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun); 6805 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit); 6806 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject); 6807 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit); 6808 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga); 6809 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit); 6810 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts); 6811