1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include <kvm/iodev.h> 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 #include <linux/sort.h> 51 #include <linux/bsearch.h> 52 53 #include <asm/processor.h> 54 #include <asm/io.h> 55 #include <asm/ioctl.h> 56 #include <asm/uaccess.h> 57 #include <asm/pgtable.h> 58 59 #include "coalesced_mmio.h" 60 #include "async_pf.h" 61 #include "vfio.h" 62 63 #define CREATE_TRACE_POINTS 64 #include <trace/events/kvm.h> 65 66 MODULE_AUTHOR("Qumranet"); 67 MODULE_LICENSE("GPL"); 68 69 /* Architectures should define their poll value according to the halt latency */ 70 static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; 71 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR); 72 73 /* Default doubles per-vcpu halt_poll_ns. */ 74 static unsigned int halt_poll_ns_grow = 2; 75 module_param(halt_poll_ns_grow, int, S_IRUGO); 76 77 /* Default resets per-vcpu halt_poll_ns . */ 78 static unsigned int halt_poll_ns_shrink; 79 module_param(halt_poll_ns_shrink, int, S_IRUGO); 80 81 /* 82 * Ordering of locks: 83 * 84 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 85 */ 86 87 DEFINE_SPINLOCK(kvm_lock); 88 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 89 LIST_HEAD(vm_list); 90 91 static cpumask_var_t cpus_hardware_enabled; 92 static int kvm_usage_count; 93 static atomic_t hardware_enable_failed; 94 95 struct kmem_cache *kvm_vcpu_cache; 96 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 97 98 static __read_mostly struct preempt_ops kvm_preempt_ops; 99 100 struct dentry *kvm_debugfs_dir; 101 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 102 103 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 104 unsigned long arg); 105 #ifdef CONFIG_KVM_COMPAT 106 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 107 unsigned long arg); 108 #endif 109 static int hardware_enable_all(void); 110 static void hardware_disable_all(void); 111 112 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 113 114 static void kvm_release_pfn_dirty(pfn_t pfn); 115 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn); 116 117 __visible bool kvm_rebooting; 118 EXPORT_SYMBOL_GPL(kvm_rebooting); 119 120 static bool largepages_enabled = true; 121 122 bool kvm_is_reserved_pfn(pfn_t pfn) 123 { 124 if (pfn_valid(pfn)) 125 return PageReserved(pfn_to_page(pfn)); 126 127 return true; 128 } 129 130 /* 131 * Switches to specified vcpu, until a matching vcpu_put() 132 */ 133 int vcpu_load(struct kvm_vcpu *vcpu) 134 { 135 int cpu; 136 137 if (mutex_lock_killable(&vcpu->mutex)) 138 return -EINTR; 139 cpu = get_cpu(); 140 preempt_notifier_register(&vcpu->preempt_notifier); 141 kvm_arch_vcpu_load(vcpu, cpu); 142 put_cpu(); 143 return 0; 144 } 145 146 void vcpu_put(struct kvm_vcpu *vcpu) 147 { 148 preempt_disable(); 149 kvm_arch_vcpu_put(vcpu); 150 preempt_notifier_unregister(&vcpu->preempt_notifier); 151 preempt_enable(); 152 mutex_unlock(&vcpu->mutex); 153 } 154 155 static void ack_flush(void *_completed) 156 { 157 } 158 159 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 160 { 161 int i, cpu, me; 162 cpumask_var_t cpus; 163 bool called = true; 164 struct kvm_vcpu *vcpu; 165 166 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 167 168 me = get_cpu(); 169 kvm_for_each_vcpu(i, vcpu, kvm) { 170 kvm_make_request(req, vcpu); 171 cpu = vcpu->cpu; 172 173 /* Set ->requests bit before we read ->mode */ 174 smp_mb(); 175 176 if (cpus != NULL && cpu != -1 && cpu != me && 177 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 178 cpumask_set_cpu(cpu, cpus); 179 } 180 if (unlikely(cpus == NULL)) 181 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 182 else if (!cpumask_empty(cpus)) 183 smp_call_function_many(cpus, ack_flush, NULL, 1); 184 else 185 called = false; 186 put_cpu(); 187 free_cpumask_var(cpus); 188 return called; 189 } 190 191 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 192 void kvm_flush_remote_tlbs(struct kvm *kvm) 193 { 194 long dirty_count = kvm->tlbs_dirty; 195 196 smp_mb(); 197 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 198 ++kvm->stat.remote_tlb_flush; 199 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 200 } 201 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 202 #endif 203 204 void kvm_reload_remote_mmus(struct kvm *kvm) 205 { 206 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 207 } 208 209 void kvm_make_mclock_inprogress_request(struct kvm *kvm) 210 { 211 kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS); 212 } 213 214 void kvm_make_scan_ioapic_request(struct kvm *kvm) 215 { 216 kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC); 217 } 218 219 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 220 { 221 struct page *page; 222 int r; 223 224 mutex_init(&vcpu->mutex); 225 vcpu->cpu = -1; 226 vcpu->kvm = kvm; 227 vcpu->vcpu_id = id; 228 vcpu->pid = NULL; 229 vcpu->halt_poll_ns = 0; 230 init_waitqueue_head(&vcpu->wq); 231 kvm_async_pf_vcpu_init(vcpu); 232 233 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 234 if (!page) { 235 r = -ENOMEM; 236 goto fail; 237 } 238 vcpu->run = page_address(page); 239 240 kvm_vcpu_set_in_spin_loop(vcpu, false); 241 kvm_vcpu_set_dy_eligible(vcpu, false); 242 vcpu->preempted = false; 243 244 r = kvm_arch_vcpu_init(vcpu); 245 if (r < 0) 246 goto fail_free_run; 247 return 0; 248 249 fail_free_run: 250 free_page((unsigned long)vcpu->run); 251 fail: 252 return r; 253 } 254 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 255 256 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 257 { 258 put_pid(vcpu->pid); 259 kvm_arch_vcpu_uninit(vcpu); 260 free_page((unsigned long)vcpu->run); 261 } 262 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 263 264 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 265 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 266 { 267 return container_of(mn, struct kvm, mmu_notifier); 268 } 269 270 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 271 struct mm_struct *mm, 272 unsigned long address) 273 { 274 struct kvm *kvm = mmu_notifier_to_kvm(mn); 275 int need_tlb_flush, idx; 276 277 /* 278 * When ->invalidate_page runs, the linux pte has been zapped 279 * already but the page is still allocated until 280 * ->invalidate_page returns. So if we increase the sequence 281 * here the kvm page fault will notice if the spte can't be 282 * established because the page is going to be freed. If 283 * instead the kvm page fault establishes the spte before 284 * ->invalidate_page runs, kvm_unmap_hva will release it 285 * before returning. 286 * 287 * The sequence increase only need to be seen at spin_unlock 288 * time, and not at spin_lock time. 289 * 290 * Increasing the sequence after the spin_unlock would be 291 * unsafe because the kvm page fault could then establish the 292 * pte after kvm_unmap_hva returned, without noticing the page 293 * is going to be freed. 294 */ 295 idx = srcu_read_lock(&kvm->srcu); 296 spin_lock(&kvm->mmu_lock); 297 298 kvm->mmu_notifier_seq++; 299 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 300 /* we've to flush the tlb before the pages can be freed */ 301 if (need_tlb_flush) 302 kvm_flush_remote_tlbs(kvm); 303 304 spin_unlock(&kvm->mmu_lock); 305 306 kvm_arch_mmu_notifier_invalidate_page(kvm, address); 307 308 srcu_read_unlock(&kvm->srcu, idx); 309 } 310 311 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 312 struct mm_struct *mm, 313 unsigned long address, 314 pte_t pte) 315 { 316 struct kvm *kvm = mmu_notifier_to_kvm(mn); 317 int idx; 318 319 idx = srcu_read_lock(&kvm->srcu); 320 spin_lock(&kvm->mmu_lock); 321 kvm->mmu_notifier_seq++; 322 kvm_set_spte_hva(kvm, address, pte); 323 spin_unlock(&kvm->mmu_lock); 324 srcu_read_unlock(&kvm->srcu, idx); 325 } 326 327 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 328 struct mm_struct *mm, 329 unsigned long start, 330 unsigned long end) 331 { 332 struct kvm *kvm = mmu_notifier_to_kvm(mn); 333 int need_tlb_flush = 0, idx; 334 335 idx = srcu_read_lock(&kvm->srcu); 336 spin_lock(&kvm->mmu_lock); 337 /* 338 * The count increase must become visible at unlock time as no 339 * spte can be established without taking the mmu_lock and 340 * count is also read inside the mmu_lock critical section. 341 */ 342 kvm->mmu_notifier_count++; 343 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 344 need_tlb_flush |= kvm->tlbs_dirty; 345 /* we've to flush the tlb before the pages can be freed */ 346 if (need_tlb_flush) 347 kvm_flush_remote_tlbs(kvm); 348 349 spin_unlock(&kvm->mmu_lock); 350 srcu_read_unlock(&kvm->srcu, idx); 351 } 352 353 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 354 struct mm_struct *mm, 355 unsigned long start, 356 unsigned long end) 357 { 358 struct kvm *kvm = mmu_notifier_to_kvm(mn); 359 360 spin_lock(&kvm->mmu_lock); 361 /* 362 * This sequence increase will notify the kvm page fault that 363 * the page that is going to be mapped in the spte could have 364 * been freed. 365 */ 366 kvm->mmu_notifier_seq++; 367 smp_wmb(); 368 /* 369 * The above sequence increase must be visible before the 370 * below count decrease, which is ensured by the smp_wmb above 371 * in conjunction with the smp_rmb in mmu_notifier_retry(). 372 */ 373 kvm->mmu_notifier_count--; 374 spin_unlock(&kvm->mmu_lock); 375 376 BUG_ON(kvm->mmu_notifier_count < 0); 377 } 378 379 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 380 struct mm_struct *mm, 381 unsigned long start, 382 unsigned long end) 383 { 384 struct kvm *kvm = mmu_notifier_to_kvm(mn); 385 int young, idx; 386 387 idx = srcu_read_lock(&kvm->srcu); 388 spin_lock(&kvm->mmu_lock); 389 390 young = kvm_age_hva(kvm, start, end); 391 if (young) 392 kvm_flush_remote_tlbs(kvm); 393 394 spin_unlock(&kvm->mmu_lock); 395 srcu_read_unlock(&kvm->srcu, idx); 396 397 return young; 398 } 399 400 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, 401 struct mm_struct *mm, 402 unsigned long start, 403 unsigned long end) 404 { 405 struct kvm *kvm = mmu_notifier_to_kvm(mn); 406 int young, idx; 407 408 idx = srcu_read_lock(&kvm->srcu); 409 spin_lock(&kvm->mmu_lock); 410 /* 411 * Even though we do not flush TLB, this will still adversely 412 * affect performance on pre-Haswell Intel EPT, where there is 413 * no EPT Access Bit to clear so that we have to tear down EPT 414 * tables instead. If we find this unacceptable, we can always 415 * add a parameter to kvm_age_hva so that it effectively doesn't 416 * do anything on clear_young. 417 * 418 * Also note that currently we never issue secondary TLB flushes 419 * from clear_young, leaving this job up to the regular system 420 * cadence. If we find this inaccurate, we might come up with a 421 * more sophisticated heuristic later. 422 */ 423 young = kvm_age_hva(kvm, start, end); 424 spin_unlock(&kvm->mmu_lock); 425 srcu_read_unlock(&kvm->srcu, idx); 426 427 return young; 428 } 429 430 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 431 struct mm_struct *mm, 432 unsigned long address) 433 { 434 struct kvm *kvm = mmu_notifier_to_kvm(mn); 435 int young, idx; 436 437 idx = srcu_read_lock(&kvm->srcu); 438 spin_lock(&kvm->mmu_lock); 439 young = kvm_test_age_hva(kvm, address); 440 spin_unlock(&kvm->mmu_lock); 441 srcu_read_unlock(&kvm->srcu, idx); 442 443 return young; 444 } 445 446 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 447 struct mm_struct *mm) 448 { 449 struct kvm *kvm = mmu_notifier_to_kvm(mn); 450 int idx; 451 452 idx = srcu_read_lock(&kvm->srcu); 453 kvm_arch_flush_shadow_all(kvm); 454 srcu_read_unlock(&kvm->srcu, idx); 455 } 456 457 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 458 .invalidate_page = kvm_mmu_notifier_invalidate_page, 459 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 460 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 461 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 462 .clear_young = kvm_mmu_notifier_clear_young, 463 .test_young = kvm_mmu_notifier_test_young, 464 .change_pte = kvm_mmu_notifier_change_pte, 465 .release = kvm_mmu_notifier_release, 466 }; 467 468 static int kvm_init_mmu_notifier(struct kvm *kvm) 469 { 470 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 471 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 472 } 473 474 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 475 476 static int kvm_init_mmu_notifier(struct kvm *kvm) 477 { 478 return 0; 479 } 480 481 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 482 483 static struct kvm_memslots *kvm_alloc_memslots(void) 484 { 485 int i; 486 struct kvm_memslots *slots; 487 488 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 489 if (!slots) 490 return NULL; 491 492 /* 493 * Init kvm generation close to the maximum to easily test the 494 * code of handling generation number wrap-around. 495 */ 496 slots->generation = -150; 497 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 498 slots->id_to_index[i] = slots->memslots[i].id = i; 499 500 return slots; 501 } 502 503 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 504 { 505 if (!memslot->dirty_bitmap) 506 return; 507 508 kvfree(memslot->dirty_bitmap); 509 memslot->dirty_bitmap = NULL; 510 } 511 512 /* 513 * Free any memory in @free but not in @dont. 514 */ 515 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 516 struct kvm_memory_slot *dont) 517 { 518 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 519 kvm_destroy_dirty_bitmap(free); 520 521 kvm_arch_free_memslot(kvm, free, dont); 522 523 free->npages = 0; 524 } 525 526 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 527 { 528 struct kvm_memory_slot *memslot; 529 530 if (!slots) 531 return; 532 533 kvm_for_each_memslot(memslot, slots) 534 kvm_free_memslot(kvm, memslot, NULL); 535 536 kvfree(slots); 537 } 538 539 static struct kvm *kvm_create_vm(unsigned long type) 540 { 541 int r, i; 542 struct kvm *kvm = kvm_arch_alloc_vm(); 543 544 if (!kvm) 545 return ERR_PTR(-ENOMEM); 546 547 r = kvm_arch_init_vm(kvm, type); 548 if (r) 549 goto out_err_no_disable; 550 551 r = hardware_enable_all(); 552 if (r) 553 goto out_err_no_disable; 554 555 #ifdef CONFIG_HAVE_KVM_IRQFD 556 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 557 #endif 558 559 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 560 561 r = -ENOMEM; 562 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 563 kvm->memslots[i] = kvm_alloc_memslots(); 564 if (!kvm->memslots[i]) 565 goto out_err_no_srcu; 566 } 567 568 if (init_srcu_struct(&kvm->srcu)) 569 goto out_err_no_srcu; 570 if (init_srcu_struct(&kvm->irq_srcu)) 571 goto out_err_no_irq_srcu; 572 for (i = 0; i < KVM_NR_BUSES; i++) { 573 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 574 GFP_KERNEL); 575 if (!kvm->buses[i]) 576 goto out_err; 577 } 578 579 spin_lock_init(&kvm->mmu_lock); 580 kvm->mm = current->mm; 581 atomic_inc(&kvm->mm->mm_count); 582 kvm_eventfd_init(kvm); 583 mutex_init(&kvm->lock); 584 mutex_init(&kvm->irq_lock); 585 mutex_init(&kvm->slots_lock); 586 atomic_set(&kvm->users_count, 1); 587 INIT_LIST_HEAD(&kvm->devices); 588 589 r = kvm_init_mmu_notifier(kvm); 590 if (r) 591 goto out_err; 592 593 spin_lock(&kvm_lock); 594 list_add(&kvm->vm_list, &vm_list); 595 spin_unlock(&kvm_lock); 596 597 preempt_notifier_inc(); 598 599 return kvm; 600 601 out_err: 602 cleanup_srcu_struct(&kvm->irq_srcu); 603 out_err_no_irq_srcu: 604 cleanup_srcu_struct(&kvm->srcu); 605 out_err_no_srcu: 606 hardware_disable_all(); 607 out_err_no_disable: 608 for (i = 0; i < KVM_NR_BUSES; i++) 609 kfree(kvm->buses[i]); 610 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 611 kvm_free_memslots(kvm, kvm->memslots[i]); 612 kvm_arch_free_vm(kvm); 613 return ERR_PTR(r); 614 } 615 616 /* 617 * Avoid using vmalloc for a small buffer. 618 * Should not be used when the size is statically known. 619 */ 620 void *kvm_kvzalloc(unsigned long size) 621 { 622 if (size > PAGE_SIZE) 623 return vzalloc(size); 624 else 625 return kzalloc(size, GFP_KERNEL); 626 } 627 628 static void kvm_destroy_devices(struct kvm *kvm) 629 { 630 struct list_head *node, *tmp; 631 632 list_for_each_safe(node, tmp, &kvm->devices) { 633 struct kvm_device *dev = 634 list_entry(node, struct kvm_device, vm_node); 635 636 list_del(node); 637 dev->ops->destroy(dev); 638 } 639 } 640 641 static void kvm_destroy_vm(struct kvm *kvm) 642 { 643 int i; 644 struct mm_struct *mm = kvm->mm; 645 646 kvm_arch_sync_events(kvm); 647 spin_lock(&kvm_lock); 648 list_del(&kvm->vm_list); 649 spin_unlock(&kvm_lock); 650 kvm_free_irq_routing(kvm); 651 for (i = 0; i < KVM_NR_BUSES; i++) 652 kvm_io_bus_destroy(kvm->buses[i]); 653 kvm_coalesced_mmio_free(kvm); 654 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 655 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 656 #else 657 kvm_arch_flush_shadow_all(kvm); 658 #endif 659 kvm_arch_destroy_vm(kvm); 660 kvm_destroy_devices(kvm); 661 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 662 kvm_free_memslots(kvm, kvm->memslots[i]); 663 cleanup_srcu_struct(&kvm->irq_srcu); 664 cleanup_srcu_struct(&kvm->srcu); 665 kvm_arch_free_vm(kvm); 666 preempt_notifier_dec(); 667 hardware_disable_all(); 668 mmdrop(mm); 669 } 670 671 void kvm_get_kvm(struct kvm *kvm) 672 { 673 atomic_inc(&kvm->users_count); 674 } 675 EXPORT_SYMBOL_GPL(kvm_get_kvm); 676 677 void kvm_put_kvm(struct kvm *kvm) 678 { 679 if (atomic_dec_and_test(&kvm->users_count)) 680 kvm_destroy_vm(kvm); 681 } 682 EXPORT_SYMBOL_GPL(kvm_put_kvm); 683 684 685 static int kvm_vm_release(struct inode *inode, struct file *filp) 686 { 687 struct kvm *kvm = filp->private_data; 688 689 kvm_irqfd_release(kvm); 690 691 kvm_put_kvm(kvm); 692 return 0; 693 } 694 695 /* 696 * Allocation size is twice as large as the actual dirty bitmap size. 697 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 698 */ 699 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 700 { 701 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 702 703 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes); 704 if (!memslot->dirty_bitmap) 705 return -ENOMEM; 706 707 return 0; 708 } 709 710 /* 711 * Insert memslot and re-sort memslots based on their GFN, 712 * so binary search could be used to lookup GFN. 713 * Sorting algorithm takes advantage of having initially 714 * sorted array and known changed memslot position. 715 */ 716 static void update_memslots(struct kvm_memslots *slots, 717 struct kvm_memory_slot *new) 718 { 719 int id = new->id; 720 int i = slots->id_to_index[id]; 721 struct kvm_memory_slot *mslots = slots->memslots; 722 723 WARN_ON(mslots[i].id != id); 724 if (!new->npages) { 725 WARN_ON(!mslots[i].npages); 726 if (mslots[i].npages) 727 slots->used_slots--; 728 } else { 729 if (!mslots[i].npages) 730 slots->used_slots++; 731 } 732 733 while (i < KVM_MEM_SLOTS_NUM - 1 && 734 new->base_gfn <= mslots[i + 1].base_gfn) { 735 if (!mslots[i + 1].npages) 736 break; 737 mslots[i] = mslots[i + 1]; 738 slots->id_to_index[mslots[i].id] = i; 739 i++; 740 } 741 742 /* 743 * The ">=" is needed when creating a slot with base_gfn == 0, 744 * so that it moves before all those with base_gfn == npages == 0. 745 * 746 * On the other hand, if new->npages is zero, the above loop has 747 * already left i pointing to the beginning of the empty part of 748 * mslots, and the ">=" would move the hole backwards in this 749 * case---which is wrong. So skip the loop when deleting a slot. 750 */ 751 if (new->npages) { 752 while (i > 0 && 753 new->base_gfn >= mslots[i - 1].base_gfn) { 754 mslots[i] = mslots[i - 1]; 755 slots->id_to_index[mslots[i].id] = i; 756 i--; 757 } 758 } else 759 WARN_ON_ONCE(i != slots->used_slots); 760 761 mslots[i] = *new; 762 slots->id_to_index[mslots[i].id] = i; 763 } 764 765 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 766 { 767 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 768 769 #ifdef __KVM_HAVE_READONLY_MEM 770 valid_flags |= KVM_MEM_READONLY; 771 #endif 772 773 if (mem->flags & ~valid_flags) 774 return -EINVAL; 775 776 return 0; 777 } 778 779 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 780 int as_id, struct kvm_memslots *slots) 781 { 782 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); 783 784 /* 785 * Set the low bit in the generation, which disables SPTE caching 786 * until the end of synchronize_srcu_expedited. 787 */ 788 WARN_ON(old_memslots->generation & 1); 789 slots->generation = old_memslots->generation + 1; 790 791 rcu_assign_pointer(kvm->memslots[as_id], slots); 792 synchronize_srcu_expedited(&kvm->srcu); 793 794 /* 795 * Increment the new memslot generation a second time. This prevents 796 * vm exits that race with memslot updates from caching a memslot 797 * generation that will (potentially) be valid forever. 798 */ 799 slots->generation++; 800 801 kvm_arch_memslots_updated(kvm, slots); 802 803 return old_memslots; 804 } 805 806 /* 807 * Allocate some memory and give it an address in the guest physical address 808 * space. 809 * 810 * Discontiguous memory is allowed, mostly for framebuffers. 811 * 812 * Must be called holding kvm->slots_lock for write. 813 */ 814 int __kvm_set_memory_region(struct kvm *kvm, 815 const struct kvm_userspace_memory_region *mem) 816 { 817 int r; 818 gfn_t base_gfn; 819 unsigned long npages; 820 struct kvm_memory_slot *slot; 821 struct kvm_memory_slot old, new; 822 struct kvm_memslots *slots = NULL, *old_memslots; 823 int as_id, id; 824 enum kvm_mr_change change; 825 826 r = check_memory_region_flags(mem); 827 if (r) 828 goto out; 829 830 r = -EINVAL; 831 as_id = mem->slot >> 16; 832 id = (u16)mem->slot; 833 834 /* General sanity checks */ 835 if (mem->memory_size & (PAGE_SIZE - 1)) 836 goto out; 837 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 838 goto out; 839 /* We can read the guest memory with __xxx_user() later on. */ 840 if ((id < KVM_USER_MEM_SLOTS) && 841 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 842 !access_ok(VERIFY_WRITE, 843 (void __user *)(unsigned long)mem->userspace_addr, 844 mem->memory_size))) 845 goto out; 846 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 847 goto out; 848 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 849 goto out; 850 851 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); 852 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 853 npages = mem->memory_size >> PAGE_SHIFT; 854 855 if (npages > KVM_MEM_MAX_NR_PAGES) 856 goto out; 857 858 new = old = *slot; 859 860 new.id = id; 861 new.base_gfn = base_gfn; 862 new.npages = npages; 863 new.flags = mem->flags; 864 865 if (npages) { 866 if (!old.npages) 867 change = KVM_MR_CREATE; 868 else { /* Modify an existing slot. */ 869 if ((mem->userspace_addr != old.userspace_addr) || 870 (npages != old.npages) || 871 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 872 goto out; 873 874 if (base_gfn != old.base_gfn) 875 change = KVM_MR_MOVE; 876 else if (new.flags != old.flags) 877 change = KVM_MR_FLAGS_ONLY; 878 else { /* Nothing to change. */ 879 r = 0; 880 goto out; 881 } 882 } 883 } else { 884 if (!old.npages) 885 goto out; 886 887 change = KVM_MR_DELETE; 888 new.base_gfn = 0; 889 new.flags = 0; 890 } 891 892 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 893 /* Check for overlaps */ 894 r = -EEXIST; 895 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { 896 if ((slot->id >= KVM_USER_MEM_SLOTS) || 897 (slot->id == id)) 898 continue; 899 if (!((base_gfn + npages <= slot->base_gfn) || 900 (base_gfn >= slot->base_gfn + slot->npages))) 901 goto out; 902 } 903 } 904 905 /* Free page dirty bitmap if unneeded */ 906 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 907 new.dirty_bitmap = NULL; 908 909 r = -ENOMEM; 910 if (change == KVM_MR_CREATE) { 911 new.userspace_addr = mem->userspace_addr; 912 913 if (kvm_arch_create_memslot(kvm, &new, npages)) 914 goto out_free; 915 } 916 917 /* Allocate page dirty bitmap if needed */ 918 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 919 if (kvm_create_dirty_bitmap(&new) < 0) 920 goto out_free; 921 } 922 923 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 924 if (!slots) 925 goto out_free; 926 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); 927 928 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 929 slot = id_to_memslot(slots, id); 930 slot->flags |= KVM_MEMSLOT_INVALID; 931 932 old_memslots = install_new_memslots(kvm, as_id, slots); 933 934 /* slot was deleted or moved, clear iommu mapping */ 935 kvm_iommu_unmap_pages(kvm, &old); 936 /* From this point no new shadow pages pointing to a deleted, 937 * or moved, memslot will be created. 938 * 939 * validation of sp->gfn happens in: 940 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 941 * - kvm_is_visible_gfn (mmu_check_roots) 942 */ 943 kvm_arch_flush_shadow_memslot(kvm, slot); 944 945 /* 946 * We can re-use the old_memslots from above, the only difference 947 * from the currently installed memslots is the invalid flag. This 948 * will get overwritten by update_memslots anyway. 949 */ 950 slots = old_memslots; 951 } 952 953 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 954 if (r) 955 goto out_slots; 956 957 /* actual memory is freed via old in kvm_free_memslot below */ 958 if (change == KVM_MR_DELETE) { 959 new.dirty_bitmap = NULL; 960 memset(&new.arch, 0, sizeof(new.arch)); 961 } 962 963 update_memslots(slots, &new); 964 old_memslots = install_new_memslots(kvm, as_id, slots); 965 966 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); 967 968 kvm_free_memslot(kvm, &old, &new); 969 kvfree(old_memslots); 970 971 /* 972 * IOMMU mapping: New slots need to be mapped. Old slots need to be 973 * un-mapped and re-mapped if their base changes. Since base change 974 * unmapping is handled above with slot deletion, mapping alone is 975 * needed here. Anything else the iommu might care about for existing 976 * slots (size changes, userspace addr changes and read-only flag 977 * changes) is disallowed above, so any other attribute changes getting 978 * here can be skipped. 979 */ 980 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 981 r = kvm_iommu_map_pages(kvm, &new); 982 return r; 983 } 984 985 return 0; 986 987 out_slots: 988 kvfree(slots); 989 out_free: 990 kvm_free_memslot(kvm, &new, &old); 991 out: 992 return r; 993 } 994 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 995 996 int kvm_set_memory_region(struct kvm *kvm, 997 const struct kvm_userspace_memory_region *mem) 998 { 999 int r; 1000 1001 mutex_lock(&kvm->slots_lock); 1002 r = __kvm_set_memory_region(kvm, mem); 1003 mutex_unlock(&kvm->slots_lock); 1004 return r; 1005 } 1006 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 1007 1008 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 1009 struct kvm_userspace_memory_region *mem) 1010 { 1011 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 1012 return -EINVAL; 1013 1014 return kvm_set_memory_region(kvm, mem); 1015 } 1016 1017 int kvm_get_dirty_log(struct kvm *kvm, 1018 struct kvm_dirty_log *log, int *is_dirty) 1019 { 1020 struct kvm_memslots *slots; 1021 struct kvm_memory_slot *memslot; 1022 int r, i, as_id, id; 1023 unsigned long n; 1024 unsigned long any = 0; 1025 1026 r = -EINVAL; 1027 as_id = log->slot >> 16; 1028 id = (u16)log->slot; 1029 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1030 goto out; 1031 1032 slots = __kvm_memslots(kvm, as_id); 1033 memslot = id_to_memslot(slots, id); 1034 r = -ENOENT; 1035 if (!memslot->dirty_bitmap) 1036 goto out; 1037 1038 n = kvm_dirty_bitmap_bytes(memslot); 1039 1040 for (i = 0; !any && i < n/sizeof(long); ++i) 1041 any = memslot->dirty_bitmap[i]; 1042 1043 r = -EFAULT; 1044 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 1045 goto out; 1046 1047 if (any) 1048 *is_dirty = 1; 1049 1050 r = 0; 1051 out: 1052 return r; 1053 } 1054 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1055 1056 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1057 /** 1058 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages 1059 * are dirty write protect them for next write. 1060 * @kvm: pointer to kvm instance 1061 * @log: slot id and address to which we copy the log 1062 * @is_dirty: flag set if any page is dirty 1063 * 1064 * We need to keep it in mind that VCPU threads can write to the bitmap 1065 * concurrently. So, to avoid losing track of dirty pages we keep the 1066 * following order: 1067 * 1068 * 1. Take a snapshot of the bit and clear it if needed. 1069 * 2. Write protect the corresponding page. 1070 * 3. Copy the snapshot to the userspace. 1071 * 4. Upon return caller flushes TLB's if needed. 1072 * 1073 * Between 2 and 4, the guest may write to the page using the remaining TLB 1074 * entry. This is not a problem because the page is reported dirty using 1075 * the snapshot taken before and step 4 ensures that writes done after 1076 * exiting to userspace will be logged for the next call. 1077 * 1078 */ 1079 int kvm_get_dirty_log_protect(struct kvm *kvm, 1080 struct kvm_dirty_log *log, bool *is_dirty) 1081 { 1082 struct kvm_memslots *slots; 1083 struct kvm_memory_slot *memslot; 1084 int r, i, as_id, id; 1085 unsigned long n; 1086 unsigned long *dirty_bitmap; 1087 unsigned long *dirty_bitmap_buffer; 1088 1089 r = -EINVAL; 1090 as_id = log->slot >> 16; 1091 id = (u16)log->slot; 1092 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1093 goto out; 1094 1095 slots = __kvm_memslots(kvm, as_id); 1096 memslot = id_to_memslot(slots, id); 1097 1098 dirty_bitmap = memslot->dirty_bitmap; 1099 r = -ENOENT; 1100 if (!dirty_bitmap) 1101 goto out; 1102 1103 n = kvm_dirty_bitmap_bytes(memslot); 1104 1105 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long); 1106 memset(dirty_bitmap_buffer, 0, n); 1107 1108 spin_lock(&kvm->mmu_lock); 1109 *is_dirty = false; 1110 for (i = 0; i < n / sizeof(long); i++) { 1111 unsigned long mask; 1112 gfn_t offset; 1113 1114 if (!dirty_bitmap[i]) 1115 continue; 1116 1117 *is_dirty = true; 1118 1119 mask = xchg(&dirty_bitmap[i], 0); 1120 dirty_bitmap_buffer[i] = mask; 1121 1122 if (mask) { 1123 offset = i * BITS_PER_LONG; 1124 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1125 offset, mask); 1126 } 1127 } 1128 1129 spin_unlock(&kvm->mmu_lock); 1130 1131 r = -EFAULT; 1132 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 1133 goto out; 1134 1135 r = 0; 1136 out: 1137 return r; 1138 } 1139 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect); 1140 #endif 1141 1142 bool kvm_largepages_enabled(void) 1143 { 1144 return largepages_enabled; 1145 } 1146 1147 void kvm_disable_largepages(void) 1148 { 1149 largepages_enabled = false; 1150 } 1151 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1152 1153 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1154 { 1155 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1156 } 1157 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1158 1159 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 1160 { 1161 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); 1162 } 1163 1164 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1165 { 1166 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1167 1168 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1169 memslot->flags & KVM_MEMSLOT_INVALID) 1170 return 0; 1171 1172 return 1; 1173 } 1174 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1175 1176 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1177 { 1178 struct vm_area_struct *vma; 1179 unsigned long addr, size; 1180 1181 size = PAGE_SIZE; 1182 1183 addr = gfn_to_hva(kvm, gfn); 1184 if (kvm_is_error_hva(addr)) 1185 return PAGE_SIZE; 1186 1187 down_read(¤t->mm->mmap_sem); 1188 vma = find_vma(current->mm, addr); 1189 if (!vma) 1190 goto out; 1191 1192 size = vma_kernel_pagesize(vma); 1193 1194 out: 1195 up_read(¤t->mm->mmap_sem); 1196 1197 return size; 1198 } 1199 1200 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1201 { 1202 return slot->flags & KVM_MEM_READONLY; 1203 } 1204 1205 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1206 gfn_t *nr_pages, bool write) 1207 { 1208 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1209 return KVM_HVA_ERR_BAD; 1210 1211 if (memslot_is_readonly(slot) && write) 1212 return KVM_HVA_ERR_RO_BAD; 1213 1214 if (nr_pages) 1215 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1216 1217 return __gfn_to_hva_memslot(slot, gfn); 1218 } 1219 1220 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1221 gfn_t *nr_pages) 1222 { 1223 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1224 } 1225 1226 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1227 gfn_t gfn) 1228 { 1229 return gfn_to_hva_many(slot, gfn, NULL); 1230 } 1231 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1232 1233 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1234 { 1235 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1236 } 1237 EXPORT_SYMBOL_GPL(gfn_to_hva); 1238 1239 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 1240 { 1241 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 1242 } 1243 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 1244 1245 /* 1246 * If writable is set to false, the hva returned by this function is only 1247 * allowed to be read. 1248 */ 1249 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1250 gfn_t gfn, bool *writable) 1251 { 1252 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1253 1254 if (!kvm_is_error_hva(hva) && writable) 1255 *writable = !memslot_is_readonly(slot); 1256 1257 return hva; 1258 } 1259 1260 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1261 { 1262 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1263 1264 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1265 } 1266 1267 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 1268 { 1269 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1270 1271 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1272 } 1273 1274 static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm, 1275 unsigned long start, int write, struct page **page) 1276 { 1277 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1278 1279 if (write) 1280 flags |= FOLL_WRITE; 1281 1282 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL); 1283 } 1284 1285 static inline int check_user_page_hwpoison(unsigned long addr) 1286 { 1287 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1288 1289 rc = __get_user_pages(current, current->mm, addr, 1, 1290 flags, NULL, NULL, NULL); 1291 return rc == -EHWPOISON; 1292 } 1293 1294 /* 1295 * The atomic path to get the writable pfn which will be stored in @pfn, 1296 * true indicates success, otherwise false is returned. 1297 */ 1298 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1299 bool write_fault, bool *writable, pfn_t *pfn) 1300 { 1301 struct page *page[1]; 1302 int npages; 1303 1304 if (!(async || atomic)) 1305 return false; 1306 1307 /* 1308 * Fast pin a writable pfn only if it is a write fault request 1309 * or the caller allows to map a writable pfn for a read fault 1310 * request. 1311 */ 1312 if (!(write_fault || writable)) 1313 return false; 1314 1315 npages = __get_user_pages_fast(addr, 1, 1, page); 1316 if (npages == 1) { 1317 *pfn = page_to_pfn(page[0]); 1318 1319 if (writable) 1320 *writable = true; 1321 return true; 1322 } 1323 1324 return false; 1325 } 1326 1327 /* 1328 * The slow path to get the pfn of the specified host virtual address, 1329 * 1 indicates success, -errno is returned if error is detected. 1330 */ 1331 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1332 bool *writable, pfn_t *pfn) 1333 { 1334 struct page *page[1]; 1335 int npages = 0; 1336 1337 might_sleep(); 1338 1339 if (writable) 1340 *writable = write_fault; 1341 1342 if (async) { 1343 down_read(¤t->mm->mmap_sem); 1344 npages = get_user_page_nowait(current, current->mm, 1345 addr, write_fault, page); 1346 up_read(¤t->mm->mmap_sem); 1347 } else 1348 npages = __get_user_pages_unlocked(current, current->mm, addr, 1, 1349 write_fault, 0, page, 1350 FOLL_TOUCH|FOLL_HWPOISON); 1351 if (npages != 1) 1352 return npages; 1353 1354 /* map read fault as writable if possible */ 1355 if (unlikely(!write_fault) && writable) { 1356 struct page *wpage[1]; 1357 1358 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1359 if (npages == 1) { 1360 *writable = true; 1361 put_page(page[0]); 1362 page[0] = wpage[0]; 1363 } 1364 1365 npages = 1; 1366 } 1367 *pfn = page_to_pfn(page[0]); 1368 return npages; 1369 } 1370 1371 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1372 { 1373 if (unlikely(!(vma->vm_flags & VM_READ))) 1374 return false; 1375 1376 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1377 return false; 1378 1379 return true; 1380 } 1381 1382 /* 1383 * Pin guest page in memory and return its pfn. 1384 * @addr: host virtual address which maps memory to the guest 1385 * @atomic: whether this function can sleep 1386 * @async: whether this function need to wait IO complete if the 1387 * host page is not in the memory 1388 * @write_fault: whether we should get a writable host page 1389 * @writable: whether it allows to map a writable host page for !@write_fault 1390 * 1391 * The function will map a writable host page for these two cases: 1392 * 1): @write_fault = true 1393 * 2): @write_fault = false && @writable, @writable will tell the caller 1394 * whether the mapping is writable. 1395 */ 1396 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1397 bool write_fault, bool *writable) 1398 { 1399 struct vm_area_struct *vma; 1400 pfn_t pfn = 0; 1401 int npages; 1402 1403 /* we can do it either atomically or asynchronously, not both */ 1404 BUG_ON(atomic && async); 1405 1406 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1407 return pfn; 1408 1409 if (atomic) 1410 return KVM_PFN_ERR_FAULT; 1411 1412 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1413 if (npages == 1) 1414 return pfn; 1415 1416 down_read(¤t->mm->mmap_sem); 1417 if (npages == -EHWPOISON || 1418 (!async && check_user_page_hwpoison(addr))) { 1419 pfn = KVM_PFN_ERR_HWPOISON; 1420 goto exit; 1421 } 1422 1423 vma = find_vma_intersection(current->mm, addr, addr + 1); 1424 1425 if (vma == NULL) 1426 pfn = KVM_PFN_ERR_FAULT; 1427 else if ((vma->vm_flags & VM_PFNMAP)) { 1428 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1429 vma->vm_pgoff; 1430 BUG_ON(!kvm_is_reserved_pfn(pfn)); 1431 } else { 1432 if (async && vma_is_valid(vma, write_fault)) 1433 *async = true; 1434 pfn = KVM_PFN_ERR_FAULT; 1435 } 1436 exit: 1437 up_read(¤t->mm->mmap_sem); 1438 return pfn; 1439 } 1440 1441 pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic, 1442 bool *async, bool write_fault, bool *writable) 1443 { 1444 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1445 1446 if (addr == KVM_HVA_ERR_RO_BAD) 1447 return KVM_PFN_ERR_RO_FAULT; 1448 1449 if (kvm_is_error_hva(addr)) 1450 return KVM_PFN_NOSLOT; 1451 1452 /* Do not map writable pfn in the readonly memslot. */ 1453 if (writable && memslot_is_readonly(slot)) { 1454 *writable = false; 1455 writable = NULL; 1456 } 1457 1458 return hva_to_pfn(addr, atomic, async, write_fault, 1459 writable); 1460 } 1461 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 1462 1463 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1464 bool *writable) 1465 { 1466 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, 1467 write_fault, writable); 1468 } 1469 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1470 1471 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1472 { 1473 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1474 } 1475 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 1476 1477 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1478 { 1479 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1480 } 1481 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1482 1483 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1484 { 1485 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); 1486 } 1487 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1488 1489 pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 1490 { 1491 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1492 } 1493 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 1494 1495 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1496 { 1497 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 1498 } 1499 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1500 1501 pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 1502 { 1503 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1504 } 1505 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 1506 1507 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1508 struct page **pages, int nr_pages) 1509 { 1510 unsigned long addr; 1511 gfn_t entry; 1512 1513 addr = gfn_to_hva_many(slot, gfn, &entry); 1514 if (kvm_is_error_hva(addr)) 1515 return -1; 1516 1517 if (entry < nr_pages) 1518 return 0; 1519 1520 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1521 } 1522 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1523 1524 static struct page *kvm_pfn_to_page(pfn_t pfn) 1525 { 1526 if (is_error_noslot_pfn(pfn)) 1527 return KVM_ERR_PTR_BAD_PAGE; 1528 1529 if (kvm_is_reserved_pfn(pfn)) { 1530 WARN_ON(1); 1531 return KVM_ERR_PTR_BAD_PAGE; 1532 } 1533 1534 return pfn_to_page(pfn); 1535 } 1536 1537 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1538 { 1539 pfn_t pfn; 1540 1541 pfn = gfn_to_pfn(kvm, gfn); 1542 1543 return kvm_pfn_to_page(pfn); 1544 } 1545 EXPORT_SYMBOL_GPL(gfn_to_page); 1546 1547 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) 1548 { 1549 pfn_t pfn; 1550 1551 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); 1552 1553 return kvm_pfn_to_page(pfn); 1554 } 1555 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); 1556 1557 void kvm_release_page_clean(struct page *page) 1558 { 1559 WARN_ON(is_error_page(page)); 1560 1561 kvm_release_pfn_clean(page_to_pfn(page)); 1562 } 1563 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1564 1565 void kvm_release_pfn_clean(pfn_t pfn) 1566 { 1567 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) 1568 put_page(pfn_to_page(pfn)); 1569 } 1570 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1571 1572 void kvm_release_page_dirty(struct page *page) 1573 { 1574 WARN_ON(is_error_page(page)); 1575 1576 kvm_release_pfn_dirty(page_to_pfn(page)); 1577 } 1578 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1579 1580 static void kvm_release_pfn_dirty(pfn_t pfn) 1581 { 1582 kvm_set_pfn_dirty(pfn); 1583 kvm_release_pfn_clean(pfn); 1584 } 1585 1586 void kvm_set_pfn_dirty(pfn_t pfn) 1587 { 1588 if (!kvm_is_reserved_pfn(pfn)) { 1589 struct page *page = pfn_to_page(pfn); 1590 1591 if (!PageReserved(page)) 1592 SetPageDirty(page); 1593 } 1594 } 1595 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1596 1597 void kvm_set_pfn_accessed(pfn_t pfn) 1598 { 1599 if (!kvm_is_reserved_pfn(pfn)) 1600 mark_page_accessed(pfn_to_page(pfn)); 1601 } 1602 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1603 1604 void kvm_get_pfn(pfn_t pfn) 1605 { 1606 if (!kvm_is_reserved_pfn(pfn)) 1607 get_page(pfn_to_page(pfn)); 1608 } 1609 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1610 1611 static int next_segment(unsigned long len, int offset) 1612 { 1613 if (len > PAGE_SIZE - offset) 1614 return PAGE_SIZE - offset; 1615 else 1616 return len; 1617 } 1618 1619 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 1620 void *data, int offset, int len) 1621 { 1622 int r; 1623 unsigned long addr; 1624 1625 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1626 if (kvm_is_error_hva(addr)) 1627 return -EFAULT; 1628 r = __copy_from_user(data, (void __user *)addr + offset, len); 1629 if (r) 1630 return -EFAULT; 1631 return 0; 1632 } 1633 1634 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1635 int len) 1636 { 1637 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1638 1639 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1640 } 1641 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1642 1643 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 1644 int offset, int len) 1645 { 1646 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1647 1648 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1649 } 1650 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 1651 1652 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1653 { 1654 gfn_t gfn = gpa >> PAGE_SHIFT; 1655 int seg; 1656 int offset = offset_in_page(gpa); 1657 int ret; 1658 1659 while ((seg = next_segment(len, offset)) != 0) { 1660 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1661 if (ret < 0) 1662 return ret; 1663 offset = 0; 1664 len -= seg; 1665 data += seg; 1666 ++gfn; 1667 } 1668 return 0; 1669 } 1670 EXPORT_SYMBOL_GPL(kvm_read_guest); 1671 1672 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 1673 { 1674 gfn_t gfn = gpa >> PAGE_SHIFT; 1675 int seg; 1676 int offset = offset_in_page(gpa); 1677 int ret; 1678 1679 while ((seg = next_segment(len, offset)) != 0) { 1680 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 1681 if (ret < 0) 1682 return ret; 1683 offset = 0; 1684 len -= seg; 1685 data += seg; 1686 ++gfn; 1687 } 1688 return 0; 1689 } 1690 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 1691 1692 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1693 void *data, int offset, unsigned long len) 1694 { 1695 int r; 1696 unsigned long addr; 1697 1698 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1699 if (kvm_is_error_hva(addr)) 1700 return -EFAULT; 1701 pagefault_disable(); 1702 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1703 pagefault_enable(); 1704 if (r) 1705 return -EFAULT; 1706 return 0; 1707 } 1708 1709 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1710 unsigned long len) 1711 { 1712 gfn_t gfn = gpa >> PAGE_SHIFT; 1713 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1714 int offset = offset_in_page(gpa); 1715 1716 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1717 } 1718 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic); 1719 1720 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 1721 void *data, unsigned long len) 1722 { 1723 gfn_t gfn = gpa >> PAGE_SHIFT; 1724 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1725 int offset = offset_in_page(gpa); 1726 1727 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1728 } 1729 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 1730 1731 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn, 1732 const void *data, int offset, int len) 1733 { 1734 int r; 1735 unsigned long addr; 1736 1737 addr = gfn_to_hva_memslot(memslot, gfn); 1738 if (kvm_is_error_hva(addr)) 1739 return -EFAULT; 1740 r = __copy_to_user((void __user *)addr + offset, data, len); 1741 if (r) 1742 return -EFAULT; 1743 mark_page_dirty_in_slot(memslot, gfn); 1744 return 0; 1745 } 1746 1747 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 1748 const void *data, int offset, int len) 1749 { 1750 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1751 1752 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1753 } 1754 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1755 1756 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 1757 const void *data, int offset, int len) 1758 { 1759 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1760 1761 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1762 } 1763 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 1764 1765 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1766 unsigned long len) 1767 { 1768 gfn_t gfn = gpa >> PAGE_SHIFT; 1769 int seg; 1770 int offset = offset_in_page(gpa); 1771 int ret; 1772 1773 while ((seg = next_segment(len, offset)) != 0) { 1774 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1775 if (ret < 0) 1776 return ret; 1777 offset = 0; 1778 len -= seg; 1779 data += seg; 1780 ++gfn; 1781 } 1782 return 0; 1783 } 1784 EXPORT_SYMBOL_GPL(kvm_write_guest); 1785 1786 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1787 unsigned long len) 1788 { 1789 gfn_t gfn = gpa >> PAGE_SHIFT; 1790 int seg; 1791 int offset = offset_in_page(gpa); 1792 int ret; 1793 1794 while ((seg = next_segment(len, offset)) != 0) { 1795 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 1796 if (ret < 0) 1797 return ret; 1798 offset = 0; 1799 len -= seg; 1800 data += seg; 1801 ++gfn; 1802 } 1803 return 0; 1804 } 1805 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 1806 1807 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1808 gpa_t gpa, unsigned long len) 1809 { 1810 struct kvm_memslots *slots = kvm_memslots(kvm); 1811 int offset = offset_in_page(gpa); 1812 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1813 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1814 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1815 gfn_t nr_pages_avail; 1816 1817 ghc->gpa = gpa; 1818 ghc->generation = slots->generation; 1819 ghc->len = len; 1820 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1821 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL); 1822 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) { 1823 ghc->hva += offset; 1824 } else { 1825 /* 1826 * If the requested region crosses two memslots, we still 1827 * verify that the entire region is valid here. 1828 */ 1829 while (start_gfn <= end_gfn) { 1830 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1831 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1832 &nr_pages_avail); 1833 if (kvm_is_error_hva(ghc->hva)) 1834 return -EFAULT; 1835 start_gfn += nr_pages_avail; 1836 } 1837 /* Use the slow path for cross page reads and writes. */ 1838 ghc->memslot = NULL; 1839 } 1840 return 0; 1841 } 1842 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1843 1844 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1845 void *data, unsigned long len) 1846 { 1847 struct kvm_memslots *slots = kvm_memslots(kvm); 1848 int r; 1849 1850 BUG_ON(len > ghc->len); 1851 1852 if (slots->generation != ghc->generation) 1853 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1854 1855 if (unlikely(!ghc->memslot)) 1856 return kvm_write_guest(kvm, ghc->gpa, data, len); 1857 1858 if (kvm_is_error_hva(ghc->hva)) 1859 return -EFAULT; 1860 1861 r = __copy_to_user((void __user *)ghc->hva, data, len); 1862 if (r) 1863 return -EFAULT; 1864 mark_page_dirty_in_slot(ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1865 1866 return 0; 1867 } 1868 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1869 1870 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1871 void *data, unsigned long len) 1872 { 1873 struct kvm_memslots *slots = kvm_memslots(kvm); 1874 int r; 1875 1876 BUG_ON(len > ghc->len); 1877 1878 if (slots->generation != ghc->generation) 1879 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1880 1881 if (unlikely(!ghc->memslot)) 1882 return kvm_read_guest(kvm, ghc->gpa, data, len); 1883 1884 if (kvm_is_error_hva(ghc->hva)) 1885 return -EFAULT; 1886 1887 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1888 if (r) 1889 return -EFAULT; 1890 1891 return 0; 1892 } 1893 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1894 1895 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1896 { 1897 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 1898 1899 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 1900 } 1901 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1902 1903 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1904 { 1905 gfn_t gfn = gpa >> PAGE_SHIFT; 1906 int seg; 1907 int offset = offset_in_page(gpa); 1908 int ret; 1909 1910 while ((seg = next_segment(len, offset)) != 0) { 1911 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1912 if (ret < 0) 1913 return ret; 1914 offset = 0; 1915 len -= seg; 1916 ++gfn; 1917 } 1918 return 0; 1919 } 1920 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1921 1922 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, 1923 gfn_t gfn) 1924 { 1925 if (memslot && memslot->dirty_bitmap) { 1926 unsigned long rel_gfn = gfn - memslot->base_gfn; 1927 1928 set_bit_le(rel_gfn, memslot->dirty_bitmap); 1929 } 1930 } 1931 1932 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 1933 { 1934 struct kvm_memory_slot *memslot; 1935 1936 memslot = gfn_to_memslot(kvm, gfn); 1937 mark_page_dirty_in_slot(memslot, gfn); 1938 } 1939 EXPORT_SYMBOL_GPL(mark_page_dirty); 1940 1941 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 1942 { 1943 struct kvm_memory_slot *memslot; 1944 1945 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1946 mark_page_dirty_in_slot(memslot, gfn); 1947 } 1948 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 1949 1950 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) 1951 { 1952 int old, val; 1953 1954 old = val = vcpu->halt_poll_ns; 1955 /* 10us base */ 1956 if (val == 0 && halt_poll_ns_grow) 1957 val = 10000; 1958 else 1959 val *= halt_poll_ns_grow; 1960 1961 vcpu->halt_poll_ns = val; 1962 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); 1963 } 1964 1965 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu) 1966 { 1967 int old, val; 1968 1969 old = val = vcpu->halt_poll_ns; 1970 if (halt_poll_ns_shrink == 0) 1971 val = 0; 1972 else 1973 val /= halt_poll_ns_shrink; 1974 1975 vcpu->halt_poll_ns = val; 1976 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); 1977 } 1978 1979 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 1980 { 1981 if (kvm_arch_vcpu_runnable(vcpu)) { 1982 kvm_make_request(KVM_REQ_UNHALT, vcpu); 1983 return -EINTR; 1984 } 1985 if (kvm_cpu_has_pending_timer(vcpu)) 1986 return -EINTR; 1987 if (signal_pending(current)) 1988 return -EINTR; 1989 1990 return 0; 1991 } 1992 1993 /* 1994 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 1995 */ 1996 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 1997 { 1998 ktime_t start, cur; 1999 DEFINE_WAIT(wait); 2000 bool waited = false; 2001 u64 block_ns; 2002 2003 start = cur = ktime_get(); 2004 if (vcpu->halt_poll_ns) { 2005 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); 2006 2007 ++vcpu->stat.halt_attempted_poll; 2008 do { 2009 /* 2010 * This sets KVM_REQ_UNHALT if an interrupt 2011 * arrives. 2012 */ 2013 if (kvm_vcpu_check_block(vcpu) < 0) { 2014 ++vcpu->stat.halt_successful_poll; 2015 goto out; 2016 } 2017 cur = ktime_get(); 2018 } while (single_task_running() && ktime_before(cur, stop)); 2019 } 2020 2021 for (;;) { 2022 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 2023 2024 if (kvm_vcpu_check_block(vcpu) < 0) 2025 break; 2026 2027 waited = true; 2028 schedule(); 2029 } 2030 2031 finish_wait(&vcpu->wq, &wait); 2032 cur = ktime_get(); 2033 2034 out: 2035 block_ns = ktime_to_ns(cur) - ktime_to_ns(start); 2036 2037 if (halt_poll_ns) { 2038 if (block_ns <= vcpu->halt_poll_ns) 2039 ; 2040 /* we had a long block, shrink polling */ 2041 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns) 2042 shrink_halt_poll_ns(vcpu); 2043 /* we had a short halt and our poll time is too small */ 2044 else if (vcpu->halt_poll_ns < halt_poll_ns && 2045 block_ns < halt_poll_ns) 2046 grow_halt_poll_ns(vcpu); 2047 } else 2048 vcpu->halt_poll_ns = 0; 2049 2050 trace_kvm_vcpu_wakeup(block_ns, waited); 2051 } 2052 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 2053 2054 #ifndef CONFIG_S390 2055 /* 2056 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 2057 */ 2058 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 2059 { 2060 int me; 2061 int cpu = vcpu->cpu; 2062 wait_queue_head_t *wqp; 2063 2064 wqp = kvm_arch_vcpu_wq(vcpu); 2065 if (waitqueue_active(wqp)) { 2066 wake_up_interruptible(wqp); 2067 ++vcpu->stat.halt_wakeup; 2068 } 2069 2070 me = get_cpu(); 2071 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 2072 if (kvm_arch_vcpu_should_kick(vcpu)) 2073 smp_send_reschedule(cpu); 2074 put_cpu(); 2075 } 2076 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 2077 #endif /* !CONFIG_S390 */ 2078 2079 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 2080 { 2081 struct pid *pid; 2082 struct task_struct *task = NULL; 2083 int ret = 0; 2084 2085 rcu_read_lock(); 2086 pid = rcu_dereference(target->pid); 2087 if (pid) 2088 task = get_pid_task(pid, PIDTYPE_PID); 2089 rcu_read_unlock(); 2090 if (!task) 2091 return ret; 2092 ret = yield_to(task, 1); 2093 put_task_struct(task); 2094 2095 return ret; 2096 } 2097 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 2098 2099 /* 2100 * Helper that checks whether a VCPU is eligible for directed yield. 2101 * Most eligible candidate to yield is decided by following heuristics: 2102 * 2103 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 2104 * (preempted lock holder), indicated by @in_spin_loop. 2105 * Set at the beiginning and cleared at the end of interception/PLE handler. 2106 * 2107 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 2108 * chance last time (mostly it has become eligible now since we have probably 2109 * yielded to lockholder in last iteration. This is done by toggling 2110 * @dy_eligible each time a VCPU checked for eligibility.) 2111 * 2112 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 2113 * to preempted lock-holder could result in wrong VCPU selection and CPU 2114 * burning. Giving priority for a potential lock-holder increases lock 2115 * progress. 2116 * 2117 * Since algorithm is based on heuristics, accessing another VCPU data without 2118 * locking does not harm. It may result in trying to yield to same VCPU, fail 2119 * and continue with next VCPU and so on. 2120 */ 2121 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 2122 { 2123 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2124 bool eligible; 2125 2126 eligible = !vcpu->spin_loop.in_spin_loop || 2127 vcpu->spin_loop.dy_eligible; 2128 2129 if (vcpu->spin_loop.in_spin_loop) 2130 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 2131 2132 return eligible; 2133 #else 2134 return true; 2135 #endif 2136 } 2137 2138 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 2139 { 2140 struct kvm *kvm = me->kvm; 2141 struct kvm_vcpu *vcpu; 2142 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 2143 int yielded = 0; 2144 int try = 3; 2145 int pass; 2146 int i; 2147 2148 kvm_vcpu_set_in_spin_loop(me, true); 2149 /* 2150 * We boost the priority of a VCPU that is runnable but not 2151 * currently running, because it got preempted by something 2152 * else and called schedule in __vcpu_run. Hopefully that 2153 * VCPU is holding the lock that we need and will release it. 2154 * We approximate round-robin by starting at the last boosted VCPU. 2155 */ 2156 for (pass = 0; pass < 2 && !yielded && try; pass++) { 2157 kvm_for_each_vcpu(i, vcpu, kvm) { 2158 if (!pass && i <= last_boosted_vcpu) { 2159 i = last_boosted_vcpu; 2160 continue; 2161 } else if (pass && i > last_boosted_vcpu) 2162 break; 2163 if (!ACCESS_ONCE(vcpu->preempted)) 2164 continue; 2165 if (vcpu == me) 2166 continue; 2167 if (waitqueue_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 2168 continue; 2169 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 2170 continue; 2171 2172 yielded = kvm_vcpu_yield_to(vcpu); 2173 if (yielded > 0) { 2174 kvm->last_boosted_vcpu = i; 2175 break; 2176 } else if (yielded < 0) { 2177 try--; 2178 if (!try) 2179 break; 2180 } 2181 } 2182 } 2183 kvm_vcpu_set_in_spin_loop(me, false); 2184 2185 /* Ensure vcpu is not eligible during next spinloop */ 2186 kvm_vcpu_set_dy_eligible(me, false); 2187 } 2188 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 2189 2190 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2191 { 2192 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 2193 struct page *page; 2194 2195 if (vmf->pgoff == 0) 2196 page = virt_to_page(vcpu->run); 2197 #ifdef CONFIG_X86 2198 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 2199 page = virt_to_page(vcpu->arch.pio_data); 2200 #endif 2201 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2202 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 2203 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 2204 #endif 2205 else 2206 return kvm_arch_vcpu_fault(vcpu, vmf); 2207 get_page(page); 2208 vmf->page = page; 2209 return 0; 2210 } 2211 2212 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 2213 .fault = kvm_vcpu_fault, 2214 }; 2215 2216 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 2217 { 2218 vma->vm_ops = &kvm_vcpu_vm_ops; 2219 return 0; 2220 } 2221 2222 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 2223 { 2224 struct kvm_vcpu *vcpu = filp->private_data; 2225 2226 kvm_put_kvm(vcpu->kvm); 2227 return 0; 2228 } 2229 2230 static struct file_operations kvm_vcpu_fops = { 2231 .release = kvm_vcpu_release, 2232 .unlocked_ioctl = kvm_vcpu_ioctl, 2233 #ifdef CONFIG_KVM_COMPAT 2234 .compat_ioctl = kvm_vcpu_compat_ioctl, 2235 #endif 2236 .mmap = kvm_vcpu_mmap, 2237 .llseek = noop_llseek, 2238 }; 2239 2240 /* 2241 * Allocates an inode for the vcpu. 2242 */ 2243 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 2244 { 2245 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 2246 } 2247 2248 /* 2249 * Creates some virtual cpus. Good luck creating more than one. 2250 */ 2251 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 2252 { 2253 int r; 2254 struct kvm_vcpu *vcpu, *v; 2255 2256 if (id >= KVM_MAX_VCPUS) 2257 return -EINVAL; 2258 2259 vcpu = kvm_arch_vcpu_create(kvm, id); 2260 if (IS_ERR(vcpu)) 2261 return PTR_ERR(vcpu); 2262 2263 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 2264 2265 r = kvm_arch_vcpu_setup(vcpu); 2266 if (r) 2267 goto vcpu_destroy; 2268 2269 mutex_lock(&kvm->lock); 2270 if (!kvm_vcpu_compatible(vcpu)) { 2271 r = -EINVAL; 2272 goto unlock_vcpu_destroy; 2273 } 2274 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 2275 r = -EINVAL; 2276 goto unlock_vcpu_destroy; 2277 } 2278 2279 kvm_for_each_vcpu(r, v, kvm) 2280 if (v->vcpu_id == id) { 2281 r = -EEXIST; 2282 goto unlock_vcpu_destroy; 2283 } 2284 2285 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 2286 2287 /* Now it's all set up, let userspace reach it */ 2288 kvm_get_kvm(kvm); 2289 r = create_vcpu_fd(vcpu); 2290 if (r < 0) { 2291 kvm_put_kvm(kvm); 2292 goto unlock_vcpu_destroy; 2293 } 2294 2295 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 2296 2297 /* 2298 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus 2299 * before kvm->online_vcpu's incremented value. 2300 */ 2301 smp_wmb(); 2302 atomic_inc(&kvm->online_vcpus); 2303 2304 mutex_unlock(&kvm->lock); 2305 kvm_arch_vcpu_postcreate(vcpu); 2306 return r; 2307 2308 unlock_vcpu_destroy: 2309 mutex_unlock(&kvm->lock); 2310 vcpu_destroy: 2311 kvm_arch_vcpu_destroy(vcpu); 2312 return r; 2313 } 2314 2315 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 2316 { 2317 if (sigset) { 2318 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2319 vcpu->sigset_active = 1; 2320 vcpu->sigset = *sigset; 2321 } else 2322 vcpu->sigset_active = 0; 2323 return 0; 2324 } 2325 2326 static long kvm_vcpu_ioctl(struct file *filp, 2327 unsigned int ioctl, unsigned long arg) 2328 { 2329 struct kvm_vcpu *vcpu = filp->private_data; 2330 void __user *argp = (void __user *)arg; 2331 int r; 2332 struct kvm_fpu *fpu = NULL; 2333 struct kvm_sregs *kvm_sregs = NULL; 2334 2335 if (vcpu->kvm->mm != current->mm) 2336 return -EIO; 2337 2338 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 2339 return -EINVAL; 2340 2341 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS) 2342 /* 2343 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 2344 * so vcpu_load() would break it. 2345 */ 2346 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT) 2347 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2348 #endif 2349 2350 2351 r = vcpu_load(vcpu); 2352 if (r) 2353 return r; 2354 switch (ioctl) { 2355 case KVM_RUN: 2356 r = -EINVAL; 2357 if (arg) 2358 goto out; 2359 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 2360 /* The thread running this VCPU changed. */ 2361 struct pid *oldpid = vcpu->pid; 2362 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 2363 2364 rcu_assign_pointer(vcpu->pid, newpid); 2365 if (oldpid) 2366 synchronize_rcu(); 2367 put_pid(oldpid); 2368 } 2369 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 2370 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 2371 break; 2372 case KVM_GET_REGS: { 2373 struct kvm_regs *kvm_regs; 2374 2375 r = -ENOMEM; 2376 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2377 if (!kvm_regs) 2378 goto out; 2379 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2380 if (r) 2381 goto out_free1; 2382 r = -EFAULT; 2383 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2384 goto out_free1; 2385 r = 0; 2386 out_free1: 2387 kfree(kvm_regs); 2388 break; 2389 } 2390 case KVM_SET_REGS: { 2391 struct kvm_regs *kvm_regs; 2392 2393 r = -ENOMEM; 2394 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2395 if (IS_ERR(kvm_regs)) { 2396 r = PTR_ERR(kvm_regs); 2397 goto out; 2398 } 2399 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2400 kfree(kvm_regs); 2401 break; 2402 } 2403 case KVM_GET_SREGS: { 2404 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2405 r = -ENOMEM; 2406 if (!kvm_sregs) 2407 goto out; 2408 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2409 if (r) 2410 goto out; 2411 r = -EFAULT; 2412 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2413 goto out; 2414 r = 0; 2415 break; 2416 } 2417 case KVM_SET_SREGS: { 2418 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2419 if (IS_ERR(kvm_sregs)) { 2420 r = PTR_ERR(kvm_sregs); 2421 kvm_sregs = NULL; 2422 goto out; 2423 } 2424 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2425 break; 2426 } 2427 case KVM_GET_MP_STATE: { 2428 struct kvm_mp_state mp_state; 2429 2430 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2431 if (r) 2432 goto out; 2433 r = -EFAULT; 2434 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 2435 goto out; 2436 r = 0; 2437 break; 2438 } 2439 case KVM_SET_MP_STATE: { 2440 struct kvm_mp_state mp_state; 2441 2442 r = -EFAULT; 2443 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 2444 goto out; 2445 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2446 break; 2447 } 2448 case KVM_TRANSLATE: { 2449 struct kvm_translation tr; 2450 2451 r = -EFAULT; 2452 if (copy_from_user(&tr, argp, sizeof(tr))) 2453 goto out; 2454 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2455 if (r) 2456 goto out; 2457 r = -EFAULT; 2458 if (copy_to_user(argp, &tr, sizeof(tr))) 2459 goto out; 2460 r = 0; 2461 break; 2462 } 2463 case KVM_SET_GUEST_DEBUG: { 2464 struct kvm_guest_debug dbg; 2465 2466 r = -EFAULT; 2467 if (copy_from_user(&dbg, argp, sizeof(dbg))) 2468 goto out; 2469 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2470 break; 2471 } 2472 case KVM_SET_SIGNAL_MASK: { 2473 struct kvm_signal_mask __user *sigmask_arg = argp; 2474 struct kvm_signal_mask kvm_sigmask; 2475 sigset_t sigset, *p; 2476 2477 p = NULL; 2478 if (argp) { 2479 r = -EFAULT; 2480 if (copy_from_user(&kvm_sigmask, argp, 2481 sizeof(kvm_sigmask))) 2482 goto out; 2483 r = -EINVAL; 2484 if (kvm_sigmask.len != sizeof(sigset)) 2485 goto out; 2486 r = -EFAULT; 2487 if (copy_from_user(&sigset, sigmask_arg->sigset, 2488 sizeof(sigset))) 2489 goto out; 2490 p = &sigset; 2491 } 2492 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2493 break; 2494 } 2495 case KVM_GET_FPU: { 2496 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2497 r = -ENOMEM; 2498 if (!fpu) 2499 goto out; 2500 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2501 if (r) 2502 goto out; 2503 r = -EFAULT; 2504 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2505 goto out; 2506 r = 0; 2507 break; 2508 } 2509 case KVM_SET_FPU: { 2510 fpu = memdup_user(argp, sizeof(*fpu)); 2511 if (IS_ERR(fpu)) { 2512 r = PTR_ERR(fpu); 2513 fpu = NULL; 2514 goto out; 2515 } 2516 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2517 break; 2518 } 2519 default: 2520 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2521 } 2522 out: 2523 vcpu_put(vcpu); 2524 kfree(fpu); 2525 kfree(kvm_sregs); 2526 return r; 2527 } 2528 2529 #ifdef CONFIG_KVM_COMPAT 2530 static long kvm_vcpu_compat_ioctl(struct file *filp, 2531 unsigned int ioctl, unsigned long arg) 2532 { 2533 struct kvm_vcpu *vcpu = filp->private_data; 2534 void __user *argp = compat_ptr(arg); 2535 int r; 2536 2537 if (vcpu->kvm->mm != current->mm) 2538 return -EIO; 2539 2540 switch (ioctl) { 2541 case KVM_SET_SIGNAL_MASK: { 2542 struct kvm_signal_mask __user *sigmask_arg = argp; 2543 struct kvm_signal_mask kvm_sigmask; 2544 compat_sigset_t csigset; 2545 sigset_t sigset; 2546 2547 if (argp) { 2548 r = -EFAULT; 2549 if (copy_from_user(&kvm_sigmask, argp, 2550 sizeof(kvm_sigmask))) 2551 goto out; 2552 r = -EINVAL; 2553 if (kvm_sigmask.len != sizeof(csigset)) 2554 goto out; 2555 r = -EFAULT; 2556 if (copy_from_user(&csigset, sigmask_arg->sigset, 2557 sizeof(csigset))) 2558 goto out; 2559 sigset_from_compat(&sigset, &csigset); 2560 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2561 } else 2562 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2563 break; 2564 } 2565 default: 2566 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2567 } 2568 2569 out: 2570 return r; 2571 } 2572 #endif 2573 2574 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2575 int (*accessor)(struct kvm_device *dev, 2576 struct kvm_device_attr *attr), 2577 unsigned long arg) 2578 { 2579 struct kvm_device_attr attr; 2580 2581 if (!accessor) 2582 return -EPERM; 2583 2584 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2585 return -EFAULT; 2586 2587 return accessor(dev, &attr); 2588 } 2589 2590 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2591 unsigned long arg) 2592 { 2593 struct kvm_device *dev = filp->private_data; 2594 2595 switch (ioctl) { 2596 case KVM_SET_DEVICE_ATTR: 2597 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2598 case KVM_GET_DEVICE_ATTR: 2599 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2600 case KVM_HAS_DEVICE_ATTR: 2601 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2602 default: 2603 if (dev->ops->ioctl) 2604 return dev->ops->ioctl(dev, ioctl, arg); 2605 2606 return -ENOTTY; 2607 } 2608 } 2609 2610 static int kvm_device_release(struct inode *inode, struct file *filp) 2611 { 2612 struct kvm_device *dev = filp->private_data; 2613 struct kvm *kvm = dev->kvm; 2614 2615 kvm_put_kvm(kvm); 2616 return 0; 2617 } 2618 2619 static const struct file_operations kvm_device_fops = { 2620 .unlocked_ioctl = kvm_device_ioctl, 2621 #ifdef CONFIG_KVM_COMPAT 2622 .compat_ioctl = kvm_device_ioctl, 2623 #endif 2624 .release = kvm_device_release, 2625 }; 2626 2627 struct kvm_device *kvm_device_from_filp(struct file *filp) 2628 { 2629 if (filp->f_op != &kvm_device_fops) 2630 return NULL; 2631 2632 return filp->private_data; 2633 } 2634 2635 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 2636 #ifdef CONFIG_KVM_MPIC 2637 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 2638 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 2639 #endif 2640 2641 #ifdef CONFIG_KVM_XICS 2642 [KVM_DEV_TYPE_XICS] = &kvm_xics_ops, 2643 #endif 2644 }; 2645 2646 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type) 2647 { 2648 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 2649 return -ENOSPC; 2650 2651 if (kvm_device_ops_table[type] != NULL) 2652 return -EEXIST; 2653 2654 kvm_device_ops_table[type] = ops; 2655 return 0; 2656 } 2657 2658 void kvm_unregister_device_ops(u32 type) 2659 { 2660 if (kvm_device_ops_table[type] != NULL) 2661 kvm_device_ops_table[type] = NULL; 2662 } 2663 2664 static int kvm_ioctl_create_device(struct kvm *kvm, 2665 struct kvm_create_device *cd) 2666 { 2667 struct kvm_device_ops *ops = NULL; 2668 struct kvm_device *dev; 2669 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2670 int ret; 2671 2672 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 2673 return -ENODEV; 2674 2675 ops = kvm_device_ops_table[cd->type]; 2676 if (ops == NULL) 2677 return -ENODEV; 2678 2679 if (test) 2680 return 0; 2681 2682 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2683 if (!dev) 2684 return -ENOMEM; 2685 2686 dev->ops = ops; 2687 dev->kvm = kvm; 2688 2689 ret = ops->create(dev, cd->type); 2690 if (ret < 0) { 2691 kfree(dev); 2692 return ret; 2693 } 2694 2695 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2696 if (ret < 0) { 2697 ops->destroy(dev); 2698 return ret; 2699 } 2700 2701 list_add(&dev->vm_node, &kvm->devices); 2702 kvm_get_kvm(kvm); 2703 cd->fd = ret; 2704 return 0; 2705 } 2706 2707 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 2708 { 2709 switch (arg) { 2710 case KVM_CAP_USER_MEMORY: 2711 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2712 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2713 case KVM_CAP_INTERNAL_ERROR_DATA: 2714 #ifdef CONFIG_HAVE_KVM_MSI 2715 case KVM_CAP_SIGNAL_MSI: 2716 #endif 2717 #ifdef CONFIG_HAVE_KVM_IRQFD 2718 case KVM_CAP_IRQFD: 2719 case KVM_CAP_IRQFD_RESAMPLE: 2720 #endif 2721 case KVM_CAP_CHECK_EXTENSION_VM: 2722 return 1; 2723 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2724 case KVM_CAP_IRQ_ROUTING: 2725 return KVM_MAX_IRQ_ROUTES; 2726 #endif 2727 #if KVM_ADDRESS_SPACE_NUM > 1 2728 case KVM_CAP_MULTI_ADDRESS_SPACE: 2729 return KVM_ADDRESS_SPACE_NUM; 2730 #endif 2731 default: 2732 break; 2733 } 2734 return kvm_vm_ioctl_check_extension(kvm, arg); 2735 } 2736 2737 static long kvm_vm_ioctl(struct file *filp, 2738 unsigned int ioctl, unsigned long arg) 2739 { 2740 struct kvm *kvm = filp->private_data; 2741 void __user *argp = (void __user *)arg; 2742 int r; 2743 2744 if (kvm->mm != current->mm) 2745 return -EIO; 2746 switch (ioctl) { 2747 case KVM_CREATE_VCPU: 2748 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2749 break; 2750 case KVM_SET_USER_MEMORY_REGION: { 2751 struct kvm_userspace_memory_region kvm_userspace_mem; 2752 2753 r = -EFAULT; 2754 if (copy_from_user(&kvm_userspace_mem, argp, 2755 sizeof(kvm_userspace_mem))) 2756 goto out; 2757 2758 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2759 break; 2760 } 2761 case KVM_GET_DIRTY_LOG: { 2762 struct kvm_dirty_log log; 2763 2764 r = -EFAULT; 2765 if (copy_from_user(&log, argp, sizeof(log))) 2766 goto out; 2767 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2768 break; 2769 } 2770 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2771 case KVM_REGISTER_COALESCED_MMIO: { 2772 struct kvm_coalesced_mmio_zone zone; 2773 2774 r = -EFAULT; 2775 if (copy_from_user(&zone, argp, sizeof(zone))) 2776 goto out; 2777 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2778 break; 2779 } 2780 case KVM_UNREGISTER_COALESCED_MMIO: { 2781 struct kvm_coalesced_mmio_zone zone; 2782 2783 r = -EFAULT; 2784 if (copy_from_user(&zone, argp, sizeof(zone))) 2785 goto out; 2786 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2787 break; 2788 } 2789 #endif 2790 case KVM_IRQFD: { 2791 struct kvm_irqfd data; 2792 2793 r = -EFAULT; 2794 if (copy_from_user(&data, argp, sizeof(data))) 2795 goto out; 2796 r = kvm_irqfd(kvm, &data); 2797 break; 2798 } 2799 case KVM_IOEVENTFD: { 2800 struct kvm_ioeventfd data; 2801 2802 r = -EFAULT; 2803 if (copy_from_user(&data, argp, sizeof(data))) 2804 goto out; 2805 r = kvm_ioeventfd(kvm, &data); 2806 break; 2807 } 2808 #ifdef CONFIG_HAVE_KVM_MSI 2809 case KVM_SIGNAL_MSI: { 2810 struct kvm_msi msi; 2811 2812 r = -EFAULT; 2813 if (copy_from_user(&msi, argp, sizeof(msi))) 2814 goto out; 2815 r = kvm_send_userspace_msi(kvm, &msi); 2816 break; 2817 } 2818 #endif 2819 #ifdef __KVM_HAVE_IRQ_LINE 2820 case KVM_IRQ_LINE_STATUS: 2821 case KVM_IRQ_LINE: { 2822 struct kvm_irq_level irq_event; 2823 2824 r = -EFAULT; 2825 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 2826 goto out; 2827 2828 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 2829 ioctl == KVM_IRQ_LINE_STATUS); 2830 if (r) 2831 goto out; 2832 2833 r = -EFAULT; 2834 if (ioctl == KVM_IRQ_LINE_STATUS) { 2835 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 2836 goto out; 2837 } 2838 2839 r = 0; 2840 break; 2841 } 2842 #endif 2843 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2844 case KVM_SET_GSI_ROUTING: { 2845 struct kvm_irq_routing routing; 2846 struct kvm_irq_routing __user *urouting; 2847 struct kvm_irq_routing_entry *entries; 2848 2849 r = -EFAULT; 2850 if (copy_from_user(&routing, argp, sizeof(routing))) 2851 goto out; 2852 r = -EINVAL; 2853 if (routing.nr >= KVM_MAX_IRQ_ROUTES) 2854 goto out; 2855 if (routing.flags) 2856 goto out; 2857 r = -ENOMEM; 2858 entries = vmalloc(routing.nr * sizeof(*entries)); 2859 if (!entries) 2860 goto out; 2861 r = -EFAULT; 2862 urouting = argp; 2863 if (copy_from_user(entries, urouting->entries, 2864 routing.nr * sizeof(*entries))) 2865 goto out_free_irq_routing; 2866 r = kvm_set_irq_routing(kvm, entries, routing.nr, 2867 routing.flags); 2868 out_free_irq_routing: 2869 vfree(entries); 2870 break; 2871 } 2872 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 2873 case KVM_CREATE_DEVICE: { 2874 struct kvm_create_device cd; 2875 2876 r = -EFAULT; 2877 if (copy_from_user(&cd, argp, sizeof(cd))) 2878 goto out; 2879 2880 r = kvm_ioctl_create_device(kvm, &cd); 2881 if (r) 2882 goto out; 2883 2884 r = -EFAULT; 2885 if (copy_to_user(argp, &cd, sizeof(cd))) 2886 goto out; 2887 2888 r = 0; 2889 break; 2890 } 2891 case KVM_CHECK_EXTENSION: 2892 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 2893 break; 2894 default: 2895 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2896 } 2897 out: 2898 return r; 2899 } 2900 2901 #ifdef CONFIG_KVM_COMPAT 2902 struct compat_kvm_dirty_log { 2903 __u32 slot; 2904 __u32 padding1; 2905 union { 2906 compat_uptr_t dirty_bitmap; /* one bit per page */ 2907 __u64 padding2; 2908 }; 2909 }; 2910 2911 static long kvm_vm_compat_ioctl(struct file *filp, 2912 unsigned int ioctl, unsigned long arg) 2913 { 2914 struct kvm *kvm = filp->private_data; 2915 int r; 2916 2917 if (kvm->mm != current->mm) 2918 return -EIO; 2919 switch (ioctl) { 2920 case KVM_GET_DIRTY_LOG: { 2921 struct compat_kvm_dirty_log compat_log; 2922 struct kvm_dirty_log log; 2923 2924 r = -EFAULT; 2925 if (copy_from_user(&compat_log, (void __user *)arg, 2926 sizeof(compat_log))) 2927 goto out; 2928 log.slot = compat_log.slot; 2929 log.padding1 = compat_log.padding1; 2930 log.padding2 = compat_log.padding2; 2931 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 2932 2933 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2934 break; 2935 } 2936 default: 2937 r = kvm_vm_ioctl(filp, ioctl, arg); 2938 } 2939 2940 out: 2941 return r; 2942 } 2943 #endif 2944 2945 static struct file_operations kvm_vm_fops = { 2946 .release = kvm_vm_release, 2947 .unlocked_ioctl = kvm_vm_ioctl, 2948 #ifdef CONFIG_KVM_COMPAT 2949 .compat_ioctl = kvm_vm_compat_ioctl, 2950 #endif 2951 .llseek = noop_llseek, 2952 }; 2953 2954 static int kvm_dev_ioctl_create_vm(unsigned long type) 2955 { 2956 int r; 2957 struct kvm *kvm; 2958 2959 kvm = kvm_create_vm(type); 2960 if (IS_ERR(kvm)) 2961 return PTR_ERR(kvm); 2962 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2963 r = kvm_coalesced_mmio_init(kvm); 2964 if (r < 0) { 2965 kvm_put_kvm(kvm); 2966 return r; 2967 } 2968 #endif 2969 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC); 2970 if (r < 0) 2971 kvm_put_kvm(kvm); 2972 2973 return r; 2974 } 2975 2976 static long kvm_dev_ioctl(struct file *filp, 2977 unsigned int ioctl, unsigned long arg) 2978 { 2979 long r = -EINVAL; 2980 2981 switch (ioctl) { 2982 case KVM_GET_API_VERSION: 2983 if (arg) 2984 goto out; 2985 r = KVM_API_VERSION; 2986 break; 2987 case KVM_CREATE_VM: 2988 r = kvm_dev_ioctl_create_vm(arg); 2989 break; 2990 case KVM_CHECK_EXTENSION: 2991 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 2992 break; 2993 case KVM_GET_VCPU_MMAP_SIZE: 2994 if (arg) 2995 goto out; 2996 r = PAGE_SIZE; /* struct kvm_run */ 2997 #ifdef CONFIG_X86 2998 r += PAGE_SIZE; /* pio data page */ 2999 #endif 3000 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 3001 r += PAGE_SIZE; /* coalesced mmio ring page */ 3002 #endif 3003 break; 3004 case KVM_TRACE_ENABLE: 3005 case KVM_TRACE_PAUSE: 3006 case KVM_TRACE_DISABLE: 3007 r = -EOPNOTSUPP; 3008 break; 3009 default: 3010 return kvm_arch_dev_ioctl(filp, ioctl, arg); 3011 } 3012 out: 3013 return r; 3014 } 3015 3016 static struct file_operations kvm_chardev_ops = { 3017 .unlocked_ioctl = kvm_dev_ioctl, 3018 .compat_ioctl = kvm_dev_ioctl, 3019 .llseek = noop_llseek, 3020 }; 3021 3022 static struct miscdevice kvm_dev = { 3023 KVM_MINOR, 3024 "kvm", 3025 &kvm_chardev_ops, 3026 }; 3027 3028 static void hardware_enable_nolock(void *junk) 3029 { 3030 int cpu = raw_smp_processor_id(); 3031 int r; 3032 3033 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3034 return; 3035 3036 cpumask_set_cpu(cpu, cpus_hardware_enabled); 3037 3038 r = kvm_arch_hardware_enable(); 3039 3040 if (r) { 3041 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3042 atomic_inc(&hardware_enable_failed); 3043 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu); 3044 } 3045 } 3046 3047 static void hardware_enable(void) 3048 { 3049 raw_spin_lock(&kvm_count_lock); 3050 if (kvm_usage_count) 3051 hardware_enable_nolock(NULL); 3052 raw_spin_unlock(&kvm_count_lock); 3053 } 3054 3055 static void hardware_disable_nolock(void *junk) 3056 { 3057 int cpu = raw_smp_processor_id(); 3058 3059 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3060 return; 3061 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3062 kvm_arch_hardware_disable(); 3063 } 3064 3065 static void hardware_disable(void) 3066 { 3067 raw_spin_lock(&kvm_count_lock); 3068 if (kvm_usage_count) 3069 hardware_disable_nolock(NULL); 3070 raw_spin_unlock(&kvm_count_lock); 3071 } 3072 3073 static void hardware_disable_all_nolock(void) 3074 { 3075 BUG_ON(!kvm_usage_count); 3076 3077 kvm_usage_count--; 3078 if (!kvm_usage_count) 3079 on_each_cpu(hardware_disable_nolock, NULL, 1); 3080 } 3081 3082 static void hardware_disable_all(void) 3083 { 3084 raw_spin_lock(&kvm_count_lock); 3085 hardware_disable_all_nolock(); 3086 raw_spin_unlock(&kvm_count_lock); 3087 } 3088 3089 static int hardware_enable_all(void) 3090 { 3091 int r = 0; 3092 3093 raw_spin_lock(&kvm_count_lock); 3094 3095 kvm_usage_count++; 3096 if (kvm_usage_count == 1) { 3097 atomic_set(&hardware_enable_failed, 0); 3098 on_each_cpu(hardware_enable_nolock, NULL, 1); 3099 3100 if (atomic_read(&hardware_enable_failed)) { 3101 hardware_disable_all_nolock(); 3102 r = -EBUSY; 3103 } 3104 } 3105 3106 raw_spin_unlock(&kvm_count_lock); 3107 3108 return r; 3109 } 3110 3111 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 3112 void *v) 3113 { 3114 val &= ~CPU_TASKS_FROZEN; 3115 switch (val) { 3116 case CPU_DYING: 3117 hardware_disable(); 3118 break; 3119 case CPU_STARTING: 3120 hardware_enable(); 3121 break; 3122 } 3123 return NOTIFY_OK; 3124 } 3125 3126 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 3127 void *v) 3128 { 3129 /* 3130 * Some (well, at least mine) BIOSes hang on reboot if 3131 * in vmx root mode. 3132 * 3133 * And Intel TXT required VMX off for all cpu when system shutdown. 3134 */ 3135 pr_info("kvm: exiting hardware virtualization\n"); 3136 kvm_rebooting = true; 3137 on_each_cpu(hardware_disable_nolock, NULL, 1); 3138 return NOTIFY_OK; 3139 } 3140 3141 static struct notifier_block kvm_reboot_notifier = { 3142 .notifier_call = kvm_reboot, 3143 .priority = 0, 3144 }; 3145 3146 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 3147 { 3148 int i; 3149 3150 for (i = 0; i < bus->dev_count; i++) { 3151 struct kvm_io_device *pos = bus->range[i].dev; 3152 3153 kvm_iodevice_destructor(pos); 3154 } 3155 kfree(bus); 3156 } 3157 3158 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 3159 const struct kvm_io_range *r2) 3160 { 3161 gpa_t addr1 = r1->addr; 3162 gpa_t addr2 = r2->addr; 3163 3164 if (addr1 < addr2) 3165 return -1; 3166 3167 /* If r2->len == 0, match the exact address. If r2->len != 0, 3168 * accept any overlapping write. Any order is acceptable for 3169 * overlapping ranges, because kvm_io_bus_get_first_dev ensures 3170 * we process all of them. 3171 */ 3172 if (r2->len) { 3173 addr1 += r1->len; 3174 addr2 += r2->len; 3175 } 3176 3177 if (addr1 > addr2) 3178 return 1; 3179 3180 return 0; 3181 } 3182 3183 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 3184 { 3185 return kvm_io_bus_cmp(p1, p2); 3186 } 3187 3188 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 3189 gpa_t addr, int len) 3190 { 3191 bus->range[bus->dev_count++] = (struct kvm_io_range) { 3192 .addr = addr, 3193 .len = len, 3194 .dev = dev, 3195 }; 3196 3197 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 3198 kvm_io_bus_sort_cmp, NULL); 3199 3200 return 0; 3201 } 3202 3203 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 3204 gpa_t addr, int len) 3205 { 3206 struct kvm_io_range *range, key; 3207 int off; 3208 3209 key = (struct kvm_io_range) { 3210 .addr = addr, 3211 .len = len, 3212 }; 3213 3214 range = bsearch(&key, bus->range, bus->dev_count, 3215 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 3216 if (range == NULL) 3217 return -ENOENT; 3218 3219 off = range - bus->range; 3220 3221 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 3222 off--; 3223 3224 return off; 3225 } 3226 3227 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3228 struct kvm_io_range *range, const void *val) 3229 { 3230 int idx; 3231 3232 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3233 if (idx < 0) 3234 return -EOPNOTSUPP; 3235 3236 while (idx < bus->dev_count && 3237 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3238 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 3239 range->len, val)) 3240 return idx; 3241 idx++; 3242 } 3243 3244 return -EOPNOTSUPP; 3245 } 3246 3247 /* kvm_io_bus_write - called under kvm->slots_lock */ 3248 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3249 int len, const void *val) 3250 { 3251 struct kvm_io_bus *bus; 3252 struct kvm_io_range range; 3253 int r; 3254 3255 range = (struct kvm_io_range) { 3256 .addr = addr, 3257 .len = len, 3258 }; 3259 3260 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3261 r = __kvm_io_bus_write(vcpu, bus, &range, val); 3262 return r < 0 ? r : 0; 3263 } 3264 3265 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 3266 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 3267 gpa_t addr, int len, const void *val, long cookie) 3268 { 3269 struct kvm_io_bus *bus; 3270 struct kvm_io_range range; 3271 3272 range = (struct kvm_io_range) { 3273 .addr = addr, 3274 .len = len, 3275 }; 3276 3277 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3278 3279 /* First try the device referenced by cookie. */ 3280 if ((cookie >= 0) && (cookie < bus->dev_count) && 3281 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 3282 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 3283 val)) 3284 return cookie; 3285 3286 /* 3287 * cookie contained garbage; fall back to search and return the 3288 * correct cookie value. 3289 */ 3290 return __kvm_io_bus_write(vcpu, bus, &range, val); 3291 } 3292 3293 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3294 struct kvm_io_range *range, void *val) 3295 { 3296 int idx; 3297 3298 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3299 if (idx < 0) 3300 return -EOPNOTSUPP; 3301 3302 while (idx < bus->dev_count && 3303 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3304 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 3305 range->len, val)) 3306 return idx; 3307 idx++; 3308 } 3309 3310 return -EOPNOTSUPP; 3311 } 3312 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3313 3314 /* kvm_io_bus_read - called under kvm->slots_lock */ 3315 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3316 int len, void *val) 3317 { 3318 struct kvm_io_bus *bus; 3319 struct kvm_io_range range; 3320 int r; 3321 3322 range = (struct kvm_io_range) { 3323 .addr = addr, 3324 .len = len, 3325 }; 3326 3327 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3328 r = __kvm_io_bus_read(vcpu, bus, &range, val); 3329 return r < 0 ? r : 0; 3330 } 3331 3332 3333 /* Caller must hold slots_lock. */ 3334 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3335 int len, struct kvm_io_device *dev) 3336 { 3337 struct kvm_io_bus *new_bus, *bus; 3338 3339 bus = kvm->buses[bus_idx]; 3340 /* exclude ioeventfd which is limited by maximum fd */ 3341 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 3342 return -ENOSPC; 3343 3344 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) * 3345 sizeof(struct kvm_io_range)), GFP_KERNEL); 3346 if (!new_bus) 3347 return -ENOMEM; 3348 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 3349 sizeof(struct kvm_io_range))); 3350 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 3351 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3352 synchronize_srcu_expedited(&kvm->srcu); 3353 kfree(bus); 3354 3355 return 0; 3356 } 3357 3358 /* Caller must hold slots_lock. */ 3359 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3360 struct kvm_io_device *dev) 3361 { 3362 int i, r; 3363 struct kvm_io_bus *new_bus, *bus; 3364 3365 bus = kvm->buses[bus_idx]; 3366 r = -ENOENT; 3367 for (i = 0; i < bus->dev_count; i++) 3368 if (bus->range[i].dev == dev) { 3369 r = 0; 3370 break; 3371 } 3372 3373 if (r) 3374 return r; 3375 3376 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) * 3377 sizeof(struct kvm_io_range)), GFP_KERNEL); 3378 if (!new_bus) 3379 return -ENOMEM; 3380 3381 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3382 new_bus->dev_count--; 3383 memcpy(new_bus->range + i, bus->range + i + 1, 3384 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3385 3386 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3387 synchronize_srcu_expedited(&kvm->srcu); 3388 kfree(bus); 3389 return r; 3390 } 3391 3392 static struct notifier_block kvm_cpu_notifier = { 3393 .notifier_call = kvm_cpu_hotplug, 3394 }; 3395 3396 static int vm_stat_get(void *_offset, u64 *val) 3397 { 3398 unsigned offset = (long)_offset; 3399 struct kvm *kvm; 3400 3401 *val = 0; 3402 spin_lock(&kvm_lock); 3403 list_for_each_entry(kvm, &vm_list, vm_list) 3404 *val += *(u32 *)((void *)kvm + offset); 3405 spin_unlock(&kvm_lock); 3406 return 0; 3407 } 3408 3409 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 3410 3411 static int vcpu_stat_get(void *_offset, u64 *val) 3412 { 3413 unsigned offset = (long)_offset; 3414 struct kvm *kvm; 3415 struct kvm_vcpu *vcpu; 3416 int i; 3417 3418 *val = 0; 3419 spin_lock(&kvm_lock); 3420 list_for_each_entry(kvm, &vm_list, vm_list) 3421 kvm_for_each_vcpu(i, vcpu, kvm) 3422 *val += *(u32 *)((void *)vcpu + offset); 3423 3424 spin_unlock(&kvm_lock); 3425 return 0; 3426 } 3427 3428 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 3429 3430 static const struct file_operations *stat_fops[] = { 3431 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3432 [KVM_STAT_VM] = &vm_stat_fops, 3433 }; 3434 3435 static int kvm_init_debug(void) 3436 { 3437 int r = -EEXIST; 3438 struct kvm_stats_debugfs_item *p; 3439 3440 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3441 if (kvm_debugfs_dir == NULL) 3442 goto out; 3443 3444 for (p = debugfs_entries; p->name; ++p) { 3445 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 3446 (void *)(long)p->offset, 3447 stat_fops[p->kind]); 3448 if (p->dentry == NULL) 3449 goto out_dir; 3450 } 3451 3452 return 0; 3453 3454 out_dir: 3455 debugfs_remove_recursive(kvm_debugfs_dir); 3456 out: 3457 return r; 3458 } 3459 3460 static void kvm_exit_debug(void) 3461 { 3462 struct kvm_stats_debugfs_item *p; 3463 3464 for (p = debugfs_entries; p->name; ++p) 3465 debugfs_remove(p->dentry); 3466 debugfs_remove(kvm_debugfs_dir); 3467 } 3468 3469 static int kvm_suspend(void) 3470 { 3471 if (kvm_usage_count) 3472 hardware_disable_nolock(NULL); 3473 return 0; 3474 } 3475 3476 static void kvm_resume(void) 3477 { 3478 if (kvm_usage_count) { 3479 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3480 hardware_enable_nolock(NULL); 3481 } 3482 } 3483 3484 static struct syscore_ops kvm_syscore_ops = { 3485 .suspend = kvm_suspend, 3486 .resume = kvm_resume, 3487 }; 3488 3489 static inline 3490 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3491 { 3492 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3493 } 3494 3495 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3496 { 3497 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3498 3499 if (vcpu->preempted) 3500 vcpu->preempted = false; 3501 3502 kvm_arch_sched_in(vcpu, cpu); 3503 3504 kvm_arch_vcpu_load(vcpu, cpu); 3505 } 3506 3507 static void kvm_sched_out(struct preempt_notifier *pn, 3508 struct task_struct *next) 3509 { 3510 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3511 3512 if (current->state == TASK_RUNNING) 3513 vcpu->preempted = true; 3514 kvm_arch_vcpu_put(vcpu); 3515 } 3516 3517 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3518 struct module *module) 3519 { 3520 int r; 3521 int cpu; 3522 3523 r = kvm_arch_init(opaque); 3524 if (r) 3525 goto out_fail; 3526 3527 /* 3528 * kvm_arch_init makes sure there's at most one caller 3529 * for architectures that support multiple implementations, 3530 * like intel and amd on x86. 3531 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3532 * conflicts in case kvm is already setup for another implementation. 3533 */ 3534 r = kvm_irqfd_init(); 3535 if (r) 3536 goto out_irqfd; 3537 3538 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 3539 r = -ENOMEM; 3540 goto out_free_0; 3541 } 3542 3543 r = kvm_arch_hardware_setup(); 3544 if (r < 0) 3545 goto out_free_0a; 3546 3547 for_each_online_cpu(cpu) { 3548 smp_call_function_single(cpu, 3549 kvm_arch_check_processor_compat, 3550 &r, 1); 3551 if (r < 0) 3552 goto out_free_1; 3553 } 3554 3555 r = register_cpu_notifier(&kvm_cpu_notifier); 3556 if (r) 3557 goto out_free_2; 3558 register_reboot_notifier(&kvm_reboot_notifier); 3559 3560 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 3561 if (!vcpu_align) 3562 vcpu_align = __alignof__(struct kvm_vcpu); 3563 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 3564 0, NULL); 3565 if (!kvm_vcpu_cache) { 3566 r = -ENOMEM; 3567 goto out_free_3; 3568 } 3569 3570 r = kvm_async_pf_init(); 3571 if (r) 3572 goto out_free; 3573 3574 kvm_chardev_ops.owner = module; 3575 kvm_vm_fops.owner = module; 3576 kvm_vcpu_fops.owner = module; 3577 3578 r = misc_register(&kvm_dev); 3579 if (r) { 3580 pr_err("kvm: misc device register failed\n"); 3581 goto out_unreg; 3582 } 3583 3584 register_syscore_ops(&kvm_syscore_ops); 3585 3586 kvm_preempt_ops.sched_in = kvm_sched_in; 3587 kvm_preempt_ops.sched_out = kvm_sched_out; 3588 3589 r = kvm_init_debug(); 3590 if (r) { 3591 pr_err("kvm: create debugfs files failed\n"); 3592 goto out_undebugfs; 3593 } 3594 3595 r = kvm_vfio_ops_init(); 3596 WARN_ON(r); 3597 3598 return 0; 3599 3600 out_undebugfs: 3601 unregister_syscore_ops(&kvm_syscore_ops); 3602 misc_deregister(&kvm_dev); 3603 out_unreg: 3604 kvm_async_pf_deinit(); 3605 out_free: 3606 kmem_cache_destroy(kvm_vcpu_cache); 3607 out_free_3: 3608 unregister_reboot_notifier(&kvm_reboot_notifier); 3609 unregister_cpu_notifier(&kvm_cpu_notifier); 3610 out_free_2: 3611 out_free_1: 3612 kvm_arch_hardware_unsetup(); 3613 out_free_0a: 3614 free_cpumask_var(cpus_hardware_enabled); 3615 out_free_0: 3616 kvm_irqfd_exit(); 3617 out_irqfd: 3618 kvm_arch_exit(); 3619 out_fail: 3620 return r; 3621 } 3622 EXPORT_SYMBOL_GPL(kvm_init); 3623 3624 void kvm_exit(void) 3625 { 3626 kvm_exit_debug(); 3627 misc_deregister(&kvm_dev); 3628 kmem_cache_destroy(kvm_vcpu_cache); 3629 kvm_async_pf_deinit(); 3630 unregister_syscore_ops(&kvm_syscore_ops); 3631 unregister_reboot_notifier(&kvm_reboot_notifier); 3632 unregister_cpu_notifier(&kvm_cpu_notifier); 3633 on_each_cpu(hardware_disable_nolock, NULL, 1); 3634 kvm_arch_hardware_unsetup(); 3635 kvm_arch_exit(); 3636 kvm_irqfd_exit(); 3637 free_cpumask_var(cpus_hardware_enabled); 3638 kvm_vfio_ops_exit(); 3639 } 3640 EXPORT_SYMBOL_GPL(kvm_exit); 3641