1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include "iodev.h" 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 #include <linux/sort.h> 51 #include <linux/bsearch.h> 52 53 #include <asm/processor.h> 54 #include <asm/io.h> 55 #include <asm/uaccess.h> 56 #include <asm/pgtable.h> 57 58 #include "coalesced_mmio.h" 59 #include "async_pf.h" 60 61 #define CREATE_TRACE_POINTS 62 #include <trace/events/kvm.h> 63 64 MODULE_AUTHOR("Qumranet"); 65 MODULE_LICENSE("GPL"); 66 67 /* 68 * Ordering of locks: 69 * 70 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 71 */ 72 73 DEFINE_RAW_SPINLOCK(kvm_lock); 74 LIST_HEAD(vm_list); 75 76 static cpumask_var_t cpus_hardware_enabled; 77 static int kvm_usage_count = 0; 78 static atomic_t hardware_enable_failed; 79 80 struct kmem_cache *kvm_vcpu_cache; 81 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 82 83 static __read_mostly struct preempt_ops kvm_preempt_ops; 84 85 struct dentry *kvm_debugfs_dir; 86 87 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 88 unsigned long arg); 89 #ifdef CONFIG_COMPAT 90 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 91 unsigned long arg); 92 #endif 93 static int hardware_enable_all(void); 94 static void hardware_disable_all(void); 95 96 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 97 98 bool kvm_rebooting; 99 EXPORT_SYMBOL_GPL(kvm_rebooting); 100 101 static bool largepages_enabled = true; 102 103 static struct page *hwpoison_page; 104 static pfn_t hwpoison_pfn; 105 106 struct page *fault_page; 107 pfn_t fault_pfn; 108 109 inline int kvm_is_mmio_pfn(pfn_t pfn) 110 { 111 if (pfn_valid(pfn)) { 112 int reserved; 113 struct page *tail = pfn_to_page(pfn); 114 struct page *head = compound_trans_head(tail); 115 reserved = PageReserved(head); 116 if (head != tail) { 117 /* 118 * "head" is not a dangling pointer 119 * (compound_trans_head takes care of that) 120 * but the hugepage may have been splitted 121 * from under us (and we may not hold a 122 * reference count on the head page so it can 123 * be reused before we run PageReferenced), so 124 * we've to check PageTail before returning 125 * what we just read. 126 */ 127 smp_rmb(); 128 if (PageTail(tail)) 129 return reserved; 130 } 131 return PageReserved(tail); 132 } 133 134 return true; 135 } 136 137 /* 138 * Switches to specified vcpu, until a matching vcpu_put() 139 */ 140 void vcpu_load(struct kvm_vcpu *vcpu) 141 { 142 int cpu; 143 144 mutex_lock(&vcpu->mutex); 145 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 146 /* The thread running this VCPU changed. */ 147 struct pid *oldpid = vcpu->pid; 148 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 149 rcu_assign_pointer(vcpu->pid, newpid); 150 synchronize_rcu(); 151 put_pid(oldpid); 152 } 153 cpu = get_cpu(); 154 preempt_notifier_register(&vcpu->preempt_notifier); 155 kvm_arch_vcpu_load(vcpu, cpu); 156 put_cpu(); 157 } 158 159 void vcpu_put(struct kvm_vcpu *vcpu) 160 { 161 preempt_disable(); 162 kvm_arch_vcpu_put(vcpu); 163 preempt_notifier_unregister(&vcpu->preempt_notifier); 164 preempt_enable(); 165 mutex_unlock(&vcpu->mutex); 166 } 167 168 static void ack_flush(void *_completed) 169 { 170 } 171 172 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req) 173 { 174 int i, cpu, me; 175 cpumask_var_t cpus; 176 bool called = true; 177 struct kvm_vcpu *vcpu; 178 179 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 180 181 me = get_cpu(); 182 kvm_for_each_vcpu(i, vcpu, kvm) { 183 kvm_make_request(req, vcpu); 184 cpu = vcpu->cpu; 185 186 /* Set ->requests bit before we read ->mode */ 187 smp_mb(); 188 189 if (cpus != NULL && cpu != -1 && cpu != me && 190 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 191 cpumask_set_cpu(cpu, cpus); 192 } 193 if (unlikely(cpus == NULL)) 194 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 195 else if (!cpumask_empty(cpus)) 196 smp_call_function_many(cpus, ack_flush, NULL, 1); 197 else 198 called = false; 199 put_cpu(); 200 free_cpumask_var(cpus); 201 return called; 202 } 203 204 void kvm_flush_remote_tlbs(struct kvm *kvm) 205 { 206 long dirty_count = kvm->tlbs_dirty; 207 208 smp_mb(); 209 if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 210 ++kvm->stat.remote_tlb_flush; 211 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 212 } 213 214 void kvm_reload_remote_mmus(struct kvm *kvm) 215 { 216 make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 217 } 218 219 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 220 { 221 struct page *page; 222 int r; 223 224 mutex_init(&vcpu->mutex); 225 vcpu->cpu = -1; 226 vcpu->kvm = kvm; 227 vcpu->vcpu_id = id; 228 vcpu->pid = NULL; 229 init_waitqueue_head(&vcpu->wq); 230 kvm_async_pf_vcpu_init(vcpu); 231 232 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 233 if (!page) { 234 r = -ENOMEM; 235 goto fail; 236 } 237 vcpu->run = page_address(page); 238 239 r = kvm_arch_vcpu_init(vcpu); 240 if (r < 0) 241 goto fail_free_run; 242 return 0; 243 244 fail_free_run: 245 free_page((unsigned long)vcpu->run); 246 fail: 247 return r; 248 } 249 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 250 251 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 252 { 253 put_pid(vcpu->pid); 254 kvm_arch_vcpu_uninit(vcpu); 255 free_page((unsigned long)vcpu->run); 256 } 257 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 258 259 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 260 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 261 { 262 return container_of(mn, struct kvm, mmu_notifier); 263 } 264 265 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 266 struct mm_struct *mm, 267 unsigned long address) 268 { 269 struct kvm *kvm = mmu_notifier_to_kvm(mn); 270 int need_tlb_flush, idx; 271 272 /* 273 * When ->invalidate_page runs, the linux pte has been zapped 274 * already but the page is still allocated until 275 * ->invalidate_page returns. So if we increase the sequence 276 * here the kvm page fault will notice if the spte can't be 277 * established because the page is going to be freed. If 278 * instead the kvm page fault establishes the spte before 279 * ->invalidate_page runs, kvm_unmap_hva will release it 280 * before returning. 281 * 282 * The sequence increase only need to be seen at spin_unlock 283 * time, and not at spin_lock time. 284 * 285 * Increasing the sequence after the spin_unlock would be 286 * unsafe because the kvm page fault could then establish the 287 * pte after kvm_unmap_hva returned, without noticing the page 288 * is going to be freed. 289 */ 290 idx = srcu_read_lock(&kvm->srcu); 291 spin_lock(&kvm->mmu_lock); 292 293 kvm->mmu_notifier_seq++; 294 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 295 /* we've to flush the tlb before the pages can be freed */ 296 if (need_tlb_flush) 297 kvm_flush_remote_tlbs(kvm); 298 299 spin_unlock(&kvm->mmu_lock); 300 srcu_read_unlock(&kvm->srcu, idx); 301 } 302 303 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 304 struct mm_struct *mm, 305 unsigned long address, 306 pte_t pte) 307 { 308 struct kvm *kvm = mmu_notifier_to_kvm(mn); 309 int idx; 310 311 idx = srcu_read_lock(&kvm->srcu); 312 spin_lock(&kvm->mmu_lock); 313 kvm->mmu_notifier_seq++; 314 kvm_set_spte_hva(kvm, address, pte); 315 spin_unlock(&kvm->mmu_lock); 316 srcu_read_unlock(&kvm->srcu, idx); 317 } 318 319 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 320 struct mm_struct *mm, 321 unsigned long start, 322 unsigned long end) 323 { 324 struct kvm *kvm = mmu_notifier_to_kvm(mn); 325 int need_tlb_flush = 0, idx; 326 327 idx = srcu_read_lock(&kvm->srcu); 328 spin_lock(&kvm->mmu_lock); 329 /* 330 * The count increase must become visible at unlock time as no 331 * spte can be established without taking the mmu_lock and 332 * count is also read inside the mmu_lock critical section. 333 */ 334 kvm->mmu_notifier_count++; 335 for (; start < end; start += PAGE_SIZE) 336 need_tlb_flush |= kvm_unmap_hva(kvm, start); 337 need_tlb_flush |= kvm->tlbs_dirty; 338 /* we've to flush the tlb before the pages can be freed */ 339 if (need_tlb_flush) 340 kvm_flush_remote_tlbs(kvm); 341 342 spin_unlock(&kvm->mmu_lock); 343 srcu_read_unlock(&kvm->srcu, idx); 344 } 345 346 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 347 struct mm_struct *mm, 348 unsigned long start, 349 unsigned long end) 350 { 351 struct kvm *kvm = mmu_notifier_to_kvm(mn); 352 353 spin_lock(&kvm->mmu_lock); 354 /* 355 * This sequence increase will notify the kvm page fault that 356 * the page that is going to be mapped in the spte could have 357 * been freed. 358 */ 359 kvm->mmu_notifier_seq++; 360 smp_wmb(); 361 /* 362 * The above sequence increase must be visible before the 363 * below count decrease, which is ensured by the smp_wmb above 364 * in conjunction with the smp_rmb in mmu_notifier_retry(). 365 */ 366 kvm->mmu_notifier_count--; 367 spin_unlock(&kvm->mmu_lock); 368 369 BUG_ON(kvm->mmu_notifier_count < 0); 370 } 371 372 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 373 struct mm_struct *mm, 374 unsigned long address) 375 { 376 struct kvm *kvm = mmu_notifier_to_kvm(mn); 377 int young, idx; 378 379 idx = srcu_read_lock(&kvm->srcu); 380 spin_lock(&kvm->mmu_lock); 381 382 young = kvm_age_hva(kvm, address); 383 if (young) 384 kvm_flush_remote_tlbs(kvm); 385 386 spin_unlock(&kvm->mmu_lock); 387 srcu_read_unlock(&kvm->srcu, idx); 388 389 return young; 390 } 391 392 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 393 struct mm_struct *mm, 394 unsigned long address) 395 { 396 struct kvm *kvm = mmu_notifier_to_kvm(mn); 397 int young, idx; 398 399 idx = srcu_read_lock(&kvm->srcu); 400 spin_lock(&kvm->mmu_lock); 401 young = kvm_test_age_hva(kvm, address); 402 spin_unlock(&kvm->mmu_lock); 403 srcu_read_unlock(&kvm->srcu, idx); 404 405 return young; 406 } 407 408 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 409 struct mm_struct *mm) 410 { 411 struct kvm *kvm = mmu_notifier_to_kvm(mn); 412 int idx; 413 414 idx = srcu_read_lock(&kvm->srcu); 415 kvm_arch_flush_shadow(kvm); 416 srcu_read_unlock(&kvm->srcu, idx); 417 } 418 419 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 420 .invalidate_page = kvm_mmu_notifier_invalidate_page, 421 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 422 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 423 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 424 .test_young = kvm_mmu_notifier_test_young, 425 .change_pte = kvm_mmu_notifier_change_pte, 426 .release = kvm_mmu_notifier_release, 427 }; 428 429 static int kvm_init_mmu_notifier(struct kvm *kvm) 430 { 431 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 432 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 433 } 434 435 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 436 437 static int kvm_init_mmu_notifier(struct kvm *kvm) 438 { 439 return 0; 440 } 441 442 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 443 444 static void kvm_init_memslots_id(struct kvm *kvm) 445 { 446 int i; 447 struct kvm_memslots *slots = kvm->memslots; 448 449 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 450 slots->id_to_index[i] = slots->memslots[i].id = i; 451 } 452 453 static struct kvm *kvm_create_vm(unsigned long type) 454 { 455 int r, i; 456 struct kvm *kvm = kvm_arch_alloc_vm(); 457 458 if (!kvm) 459 return ERR_PTR(-ENOMEM); 460 461 r = kvm_arch_init_vm(kvm, type); 462 if (r) 463 goto out_err_nodisable; 464 465 r = hardware_enable_all(); 466 if (r) 467 goto out_err_nodisable; 468 469 #ifdef CONFIG_HAVE_KVM_IRQCHIP 470 INIT_HLIST_HEAD(&kvm->mask_notifier_list); 471 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 472 #endif 473 474 r = -ENOMEM; 475 kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); 476 if (!kvm->memslots) 477 goto out_err_nosrcu; 478 kvm_init_memslots_id(kvm); 479 if (init_srcu_struct(&kvm->srcu)) 480 goto out_err_nosrcu; 481 for (i = 0; i < KVM_NR_BUSES; i++) { 482 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 483 GFP_KERNEL); 484 if (!kvm->buses[i]) 485 goto out_err; 486 } 487 488 spin_lock_init(&kvm->mmu_lock); 489 kvm->mm = current->mm; 490 atomic_inc(&kvm->mm->mm_count); 491 kvm_eventfd_init(kvm); 492 mutex_init(&kvm->lock); 493 mutex_init(&kvm->irq_lock); 494 mutex_init(&kvm->slots_lock); 495 atomic_set(&kvm->users_count, 1); 496 497 r = kvm_init_mmu_notifier(kvm); 498 if (r) 499 goto out_err; 500 501 raw_spin_lock(&kvm_lock); 502 list_add(&kvm->vm_list, &vm_list); 503 raw_spin_unlock(&kvm_lock); 504 505 return kvm; 506 507 out_err: 508 cleanup_srcu_struct(&kvm->srcu); 509 out_err_nosrcu: 510 hardware_disable_all(); 511 out_err_nodisable: 512 for (i = 0; i < KVM_NR_BUSES; i++) 513 kfree(kvm->buses[i]); 514 kfree(kvm->memslots); 515 kvm_arch_free_vm(kvm); 516 return ERR_PTR(r); 517 } 518 519 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 520 { 521 if (!memslot->dirty_bitmap) 522 return; 523 524 if (2 * kvm_dirty_bitmap_bytes(memslot) > PAGE_SIZE) 525 vfree(memslot->dirty_bitmap_head); 526 else 527 kfree(memslot->dirty_bitmap_head); 528 529 memslot->dirty_bitmap = NULL; 530 memslot->dirty_bitmap_head = NULL; 531 } 532 533 /* 534 * Free any memory in @free but not in @dont. 535 */ 536 static void kvm_free_physmem_slot(struct kvm_memory_slot *free, 537 struct kvm_memory_slot *dont) 538 { 539 if (!dont || free->rmap != dont->rmap) 540 vfree(free->rmap); 541 542 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 543 kvm_destroy_dirty_bitmap(free); 544 545 kvm_arch_free_memslot(free, dont); 546 547 free->npages = 0; 548 free->rmap = NULL; 549 } 550 551 void kvm_free_physmem(struct kvm *kvm) 552 { 553 struct kvm_memslots *slots = kvm->memslots; 554 struct kvm_memory_slot *memslot; 555 556 kvm_for_each_memslot(memslot, slots) 557 kvm_free_physmem_slot(memslot, NULL); 558 559 kfree(kvm->memslots); 560 } 561 562 static void kvm_destroy_vm(struct kvm *kvm) 563 { 564 int i; 565 struct mm_struct *mm = kvm->mm; 566 567 kvm_arch_sync_events(kvm); 568 raw_spin_lock(&kvm_lock); 569 list_del(&kvm->vm_list); 570 raw_spin_unlock(&kvm_lock); 571 kvm_free_irq_routing(kvm); 572 for (i = 0; i < KVM_NR_BUSES; i++) 573 kvm_io_bus_destroy(kvm->buses[i]); 574 kvm_coalesced_mmio_free(kvm); 575 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 576 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 577 #else 578 kvm_arch_flush_shadow(kvm); 579 #endif 580 kvm_arch_destroy_vm(kvm); 581 kvm_free_physmem(kvm); 582 cleanup_srcu_struct(&kvm->srcu); 583 kvm_arch_free_vm(kvm); 584 hardware_disable_all(); 585 mmdrop(mm); 586 } 587 588 void kvm_get_kvm(struct kvm *kvm) 589 { 590 atomic_inc(&kvm->users_count); 591 } 592 EXPORT_SYMBOL_GPL(kvm_get_kvm); 593 594 void kvm_put_kvm(struct kvm *kvm) 595 { 596 if (atomic_dec_and_test(&kvm->users_count)) 597 kvm_destroy_vm(kvm); 598 } 599 EXPORT_SYMBOL_GPL(kvm_put_kvm); 600 601 602 static int kvm_vm_release(struct inode *inode, struct file *filp) 603 { 604 struct kvm *kvm = filp->private_data; 605 606 kvm_irqfd_release(kvm); 607 608 kvm_put_kvm(kvm); 609 return 0; 610 } 611 612 /* 613 * Allocation size is twice as large as the actual dirty bitmap size. 614 * This makes it possible to do double buffering: see x86's 615 * kvm_vm_ioctl_get_dirty_log(). 616 */ 617 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 618 { 619 #ifndef CONFIG_S390 620 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 621 622 if (dirty_bytes > PAGE_SIZE) 623 memslot->dirty_bitmap = vzalloc(dirty_bytes); 624 else 625 memslot->dirty_bitmap = kzalloc(dirty_bytes, GFP_KERNEL); 626 627 if (!memslot->dirty_bitmap) 628 return -ENOMEM; 629 630 memslot->dirty_bitmap_head = memslot->dirty_bitmap; 631 memslot->nr_dirty_pages = 0; 632 #endif /* !CONFIG_S390 */ 633 return 0; 634 } 635 636 static int cmp_memslot(const void *slot1, const void *slot2) 637 { 638 struct kvm_memory_slot *s1, *s2; 639 640 s1 = (struct kvm_memory_slot *)slot1; 641 s2 = (struct kvm_memory_slot *)slot2; 642 643 if (s1->npages < s2->npages) 644 return 1; 645 if (s1->npages > s2->npages) 646 return -1; 647 648 return 0; 649 } 650 651 /* 652 * Sort the memslots base on its size, so the larger slots 653 * will get better fit. 654 */ 655 static void sort_memslots(struct kvm_memslots *slots) 656 { 657 int i; 658 659 sort(slots->memslots, KVM_MEM_SLOTS_NUM, 660 sizeof(struct kvm_memory_slot), cmp_memslot, NULL); 661 662 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 663 slots->id_to_index[slots->memslots[i].id] = i; 664 } 665 666 void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new) 667 { 668 if (new) { 669 int id = new->id; 670 struct kvm_memory_slot *old = id_to_memslot(slots, id); 671 unsigned long npages = old->npages; 672 673 *old = *new; 674 if (new->npages != npages) 675 sort_memslots(slots); 676 } 677 678 slots->generation++; 679 } 680 681 /* 682 * Allocate some memory and give it an address in the guest physical address 683 * space. 684 * 685 * Discontiguous memory is allowed, mostly for framebuffers. 686 * 687 * Must be called holding mmap_sem for write. 688 */ 689 int __kvm_set_memory_region(struct kvm *kvm, 690 struct kvm_userspace_memory_region *mem, 691 int user_alloc) 692 { 693 int r; 694 gfn_t base_gfn; 695 unsigned long npages; 696 unsigned long i; 697 struct kvm_memory_slot *memslot; 698 struct kvm_memory_slot old, new; 699 struct kvm_memslots *slots, *old_memslots; 700 701 r = -EINVAL; 702 /* General sanity checks */ 703 if (mem->memory_size & (PAGE_SIZE - 1)) 704 goto out; 705 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 706 goto out; 707 /* We can read the guest memory with __xxx_user() later on. */ 708 if (user_alloc && 709 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 710 !access_ok(VERIFY_WRITE, 711 (void __user *)(unsigned long)mem->userspace_addr, 712 mem->memory_size))) 713 goto out; 714 if (mem->slot >= KVM_MEM_SLOTS_NUM) 715 goto out; 716 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 717 goto out; 718 719 memslot = id_to_memslot(kvm->memslots, mem->slot); 720 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 721 npages = mem->memory_size >> PAGE_SHIFT; 722 723 r = -EINVAL; 724 if (npages > KVM_MEM_MAX_NR_PAGES) 725 goto out; 726 727 if (!npages) 728 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; 729 730 new = old = *memslot; 731 732 new.id = mem->slot; 733 new.base_gfn = base_gfn; 734 new.npages = npages; 735 new.flags = mem->flags; 736 737 /* Disallow changing a memory slot's size. */ 738 r = -EINVAL; 739 if (npages && old.npages && npages != old.npages) 740 goto out_free; 741 742 /* Check for overlaps */ 743 r = -EEXIST; 744 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { 745 struct kvm_memory_slot *s = &kvm->memslots->memslots[i]; 746 747 if (s == memslot || !s->npages) 748 continue; 749 if (!((base_gfn + npages <= s->base_gfn) || 750 (base_gfn >= s->base_gfn + s->npages))) 751 goto out_free; 752 } 753 754 /* Free page dirty bitmap if unneeded */ 755 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 756 new.dirty_bitmap = NULL; 757 758 r = -ENOMEM; 759 760 /* Allocate if a slot is being created */ 761 if (npages && !old.npages) { 762 new.user_alloc = user_alloc; 763 new.userspace_addr = mem->userspace_addr; 764 #ifndef CONFIG_S390 765 new.rmap = vzalloc(npages * sizeof(*new.rmap)); 766 if (!new.rmap) 767 goto out_free; 768 #endif /* not defined CONFIG_S390 */ 769 if (kvm_arch_create_memslot(&new, npages)) 770 goto out_free; 771 } 772 773 /* Allocate page dirty bitmap if needed */ 774 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 775 if (kvm_create_dirty_bitmap(&new) < 0) 776 goto out_free; 777 /* destroy any largepage mappings for dirty tracking */ 778 } 779 780 if (!npages) { 781 struct kvm_memory_slot *slot; 782 783 r = -ENOMEM; 784 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots), 785 GFP_KERNEL); 786 if (!slots) 787 goto out_free; 788 slot = id_to_memslot(slots, mem->slot); 789 slot->flags |= KVM_MEMSLOT_INVALID; 790 791 update_memslots(slots, NULL); 792 793 old_memslots = kvm->memslots; 794 rcu_assign_pointer(kvm->memslots, slots); 795 synchronize_srcu_expedited(&kvm->srcu); 796 /* From this point no new shadow pages pointing to a deleted 797 * memslot will be created. 798 * 799 * validation of sp->gfn happens in: 800 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 801 * - kvm_is_visible_gfn (mmu_check_roots) 802 */ 803 kvm_arch_flush_shadow(kvm); 804 kfree(old_memslots); 805 } 806 807 r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc); 808 if (r) 809 goto out_free; 810 811 /* map the pages in iommu page table */ 812 if (npages) { 813 r = kvm_iommu_map_pages(kvm, &new); 814 if (r) 815 goto out_free; 816 } 817 818 r = -ENOMEM; 819 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots), 820 GFP_KERNEL); 821 if (!slots) 822 goto out_free; 823 824 /* actual memory is freed via old in kvm_free_physmem_slot below */ 825 if (!npages) { 826 new.rmap = NULL; 827 new.dirty_bitmap = NULL; 828 memset(&new.arch, 0, sizeof(new.arch)); 829 } 830 831 update_memslots(slots, &new); 832 old_memslots = kvm->memslots; 833 rcu_assign_pointer(kvm->memslots, slots); 834 synchronize_srcu_expedited(&kvm->srcu); 835 836 kvm_arch_commit_memory_region(kvm, mem, old, user_alloc); 837 838 /* 839 * If the new memory slot is created, we need to clear all 840 * mmio sptes. 841 */ 842 if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT) 843 kvm_arch_flush_shadow(kvm); 844 845 kvm_free_physmem_slot(&old, &new); 846 kfree(old_memslots); 847 848 return 0; 849 850 out_free: 851 kvm_free_physmem_slot(&new, &old); 852 out: 853 return r; 854 855 } 856 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 857 858 int kvm_set_memory_region(struct kvm *kvm, 859 struct kvm_userspace_memory_region *mem, 860 int user_alloc) 861 { 862 int r; 863 864 mutex_lock(&kvm->slots_lock); 865 r = __kvm_set_memory_region(kvm, mem, user_alloc); 866 mutex_unlock(&kvm->slots_lock); 867 return r; 868 } 869 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 870 871 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 872 struct 873 kvm_userspace_memory_region *mem, 874 int user_alloc) 875 { 876 if (mem->slot >= KVM_MEMORY_SLOTS) 877 return -EINVAL; 878 return kvm_set_memory_region(kvm, mem, user_alloc); 879 } 880 881 int kvm_get_dirty_log(struct kvm *kvm, 882 struct kvm_dirty_log *log, int *is_dirty) 883 { 884 struct kvm_memory_slot *memslot; 885 int r, i; 886 unsigned long n; 887 unsigned long any = 0; 888 889 r = -EINVAL; 890 if (log->slot >= KVM_MEMORY_SLOTS) 891 goto out; 892 893 memslot = id_to_memslot(kvm->memslots, log->slot); 894 r = -ENOENT; 895 if (!memslot->dirty_bitmap) 896 goto out; 897 898 n = kvm_dirty_bitmap_bytes(memslot); 899 900 for (i = 0; !any && i < n/sizeof(long); ++i) 901 any = memslot->dirty_bitmap[i]; 902 903 r = -EFAULT; 904 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 905 goto out; 906 907 if (any) 908 *is_dirty = 1; 909 910 r = 0; 911 out: 912 return r; 913 } 914 915 bool kvm_largepages_enabled(void) 916 { 917 return largepages_enabled; 918 } 919 920 void kvm_disable_largepages(void) 921 { 922 largepages_enabled = false; 923 } 924 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 925 926 int is_error_page(struct page *page) 927 { 928 return page == bad_page || page == hwpoison_page || page == fault_page; 929 } 930 EXPORT_SYMBOL_GPL(is_error_page); 931 932 int is_error_pfn(pfn_t pfn) 933 { 934 return pfn == bad_pfn || pfn == hwpoison_pfn || pfn == fault_pfn; 935 } 936 EXPORT_SYMBOL_GPL(is_error_pfn); 937 938 int is_hwpoison_pfn(pfn_t pfn) 939 { 940 return pfn == hwpoison_pfn; 941 } 942 EXPORT_SYMBOL_GPL(is_hwpoison_pfn); 943 944 int is_fault_pfn(pfn_t pfn) 945 { 946 return pfn == fault_pfn; 947 } 948 EXPORT_SYMBOL_GPL(is_fault_pfn); 949 950 int is_noslot_pfn(pfn_t pfn) 951 { 952 return pfn == bad_pfn; 953 } 954 EXPORT_SYMBOL_GPL(is_noslot_pfn); 955 956 int is_invalid_pfn(pfn_t pfn) 957 { 958 return pfn == hwpoison_pfn || pfn == fault_pfn; 959 } 960 EXPORT_SYMBOL_GPL(is_invalid_pfn); 961 962 static inline unsigned long bad_hva(void) 963 { 964 return PAGE_OFFSET; 965 } 966 967 int kvm_is_error_hva(unsigned long addr) 968 { 969 return addr == bad_hva(); 970 } 971 EXPORT_SYMBOL_GPL(kvm_is_error_hva); 972 973 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 974 { 975 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 976 } 977 EXPORT_SYMBOL_GPL(gfn_to_memslot); 978 979 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 980 { 981 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 982 983 if (!memslot || memslot->id >= KVM_MEMORY_SLOTS || 984 memslot->flags & KVM_MEMSLOT_INVALID) 985 return 0; 986 987 return 1; 988 } 989 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 990 991 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 992 { 993 struct vm_area_struct *vma; 994 unsigned long addr, size; 995 996 size = PAGE_SIZE; 997 998 addr = gfn_to_hva(kvm, gfn); 999 if (kvm_is_error_hva(addr)) 1000 return PAGE_SIZE; 1001 1002 down_read(¤t->mm->mmap_sem); 1003 vma = find_vma(current->mm, addr); 1004 if (!vma) 1005 goto out; 1006 1007 size = vma_kernel_pagesize(vma); 1008 1009 out: 1010 up_read(¤t->mm->mmap_sem); 1011 1012 return size; 1013 } 1014 1015 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1016 gfn_t *nr_pages) 1017 { 1018 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1019 return bad_hva(); 1020 1021 if (nr_pages) 1022 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1023 1024 return gfn_to_hva_memslot(slot, gfn); 1025 } 1026 1027 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1028 { 1029 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1030 } 1031 EXPORT_SYMBOL_GPL(gfn_to_hva); 1032 1033 static pfn_t get_fault_pfn(void) 1034 { 1035 get_page(fault_page); 1036 return fault_pfn; 1037 } 1038 1039 int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm, 1040 unsigned long start, int write, struct page **page) 1041 { 1042 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1043 1044 if (write) 1045 flags |= FOLL_WRITE; 1046 1047 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL); 1048 } 1049 1050 static inline int check_user_page_hwpoison(unsigned long addr) 1051 { 1052 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1053 1054 rc = __get_user_pages(current, current->mm, addr, 1, 1055 flags, NULL, NULL, NULL); 1056 return rc == -EHWPOISON; 1057 } 1058 1059 static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr, bool atomic, 1060 bool *async, bool write_fault, bool *writable) 1061 { 1062 struct page *page[1]; 1063 int npages = 0; 1064 pfn_t pfn; 1065 1066 /* we can do it either atomically or asynchronously, not both */ 1067 BUG_ON(atomic && async); 1068 1069 BUG_ON(!write_fault && !writable); 1070 1071 if (writable) 1072 *writable = true; 1073 1074 if (atomic || async) 1075 npages = __get_user_pages_fast(addr, 1, 1, page); 1076 1077 if (unlikely(npages != 1) && !atomic) { 1078 might_sleep(); 1079 1080 if (writable) 1081 *writable = write_fault; 1082 1083 if (async) { 1084 down_read(¤t->mm->mmap_sem); 1085 npages = get_user_page_nowait(current, current->mm, 1086 addr, write_fault, page); 1087 up_read(¤t->mm->mmap_sem); 1088 } else 1089 npages = get_user_pages_fast(addr, 1, write_fault, 1090 page); 1091 1092 /* map read fault as writable if possible */ 1093 if (unlikely(!write_fault) && npages == 1) { 1094 struct page *wpage[1]; 1095 1096 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1097 if (npages == 1) { 1098 *writable = true; 1099 put_page(page[0]); 1100 page[0] = wpage[0]; 1101 } 1102 npages = 1; 1103 } 1104 } 1105 1106 if (unlikely(npages != 1)) { 1107 struct vm_area_struct *vma; 1108 1109 if (atomic) 1110 return get_fault_pfn(); 1111 1112 down_read(¤t->mm->mmap_sem); 1113 if (npages == -EHWPOISON || 1114 (!async && check_user_page_hwpoison(addr))) { 1115 up_read(¤t->mm->mmap_sem); 1116 get_page(hwpoison_page); 1117 return page_to_pfn(hwpoison_page); 1118 } 1119 1120 vma = find_vma_intersection(current->mm, addr, addr+1); 1121 1122 if (vma == NULL) 1123 pfn = get_fault_pfn(); 1124 else if ((vma->vm_flags & VM_PFNMAP)) { 1125 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1126 vma->vm_pgoff; 1127 BUG_ON(!kvm_is_mmio_pfn(pfn)); 1128 } else { 1129 if (async && (vma->vm_flags & VM_WRITE)) 1130 *async = true; 1131 pfn = get_fault_pfn(); 1132 } 1133 up_read(¤t->mm->mmap_sem); 1134 } else 1135 pfn = page_to_pfn(page[0]); 1136 1137 return pfn; 1138 } 1139 1140 pfn_t hva_to_pfn_atomic(struct kvm *kvm, unsigned long addr) 1141 { 1142 return hva_to_pfn(kvm, addr, true, NULL, true, NULL); 1143 } 1144 EXPORT_SYMBOL_GPL(hva_to_pfn_atomic); 1145 1146 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async, 1147 bool write_fault, bool *writable) 1148 { 1149 unsigned long addr; 1150 1151 if (async) 1152 *async = false; 1153 1154 addr = gfn_to_hva(kvm, gfn); 1155 if (kvm_is_error_hva(addr)) { 1156 get_page(bad_page); 1157 return page_to_pfn(bad_page); 1158 } 1159 1160 return hva_to_pfn(kvm, addr, atomic, async, write_fault, writable); 1161 } 1162 1163 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1164 { 1165 return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL); 1166 } 1167 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1168 1169 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async, 1170 bool write_fault, bool *writable) 1171 { 1172 return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable); 1173 } 1174 EXPORT_SYMBOL_GPL(gfn_to_pfn_async); 1175 1176 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1177 { 1178 return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL); 1179 } 1180 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1181 1182 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1183 bool *writable) 1184 { 1185 return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable); 1186 } 1187 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1188 1189 pfn_t gfn_to_pfn_memslot(struct kvm *kvm, 1190 struct kvm_memory_slot *slot, gfn_t gfn) 1191 { 1192 unsigned long addr = gfn_to_hva_memslot(slot, gfn); 1193 return hva_to_pfn(kvm, addr, false, NULL, true, NULL); 1194 } 1195 1196 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages, 1197 int nr_pages) 1198 { 1199 unsigned long addr; 1200 gfn_t entry; 1201 1202 addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry); 1203 if (kvm_is_error_hva(addr)) 1204 return -1; 1205 1206 if (entry < nr_pages) 1207 return 0; 1208 1209 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1210 } 1211 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1212 1213 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1214 { 1215 pfn_t pfn; 1216 1217 pfn = gfn_to_pfn(kvm, gfn); 1218 if (!kvm_is_mmio_pfn(pfn)) 1219 return pfn_to_page(pfn); 1220 1221 WARN_ON(kvm_is_mmio_pfn(pfn)); 1222 1223 get_page(bad_page); 1224 return bad_page; 1225 } 1226 1227 EXPORT_SYMBOL_GPL(gfn_to_page); 1228 1229 void kvm_release_page_clean(struct page *page) 1230 { 1231 kvm_release_pfn_clean(page_to_pfn(page)); 1232 } 1233 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1234 1235 void kvm_release_pfn_clean(pfn_t pfn) 1236 { 1237 if (!kvm_is_mmio_pfn(pfn)) 1238 put_page(pfn_to_page(pfn)); 1239 } 1240 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1241 1242 void kvm_release_page_dirty(struct page *page) 1243 { 1244 kvm_release_pfn_dirty(page_to_pfn(page)); 1245 } 1246 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1247 1248 void kvm_release_pfn_dirty(pfn_t pfn) 1249 { 1250 kvm_set_pfn_dirty(pfn); 1251 kvm_release_pfn_clean(pfn); 1252 } 1253 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); 1254 1255 void kvm_set_page_dirty(struct page *page) 1256 { 1257 kvm_set_pfn_dirty(page_to_pfn(page)); 1258 } 1259 EXPORT_SYMBOL_GPL(kvm_set_page_dirty); 1260 1261 void kvm_set_pfn_dirty(pfn_t pfn) 1262 { 1263 if (!kvm_is_mmio_pfn(pfn)) { 1264 struct page *page = pfn_to_page(pfn); 1265 if (!PageReserved(page)) 1266 SetPageDirty(page); 1267 } 1268 } 1269 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1270 1271 void kvm_set_pfn_accessed(pfn_t pfn) 1272 { 1273 if (!kvm_is_mmio_pfn(pfn)) 1274 mark_page_accessed(pfn_to_page(pfn)); 1275 } 1276 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1277 1278 void kvm_get_pfn(pfn_t pfn) 1279 { 1280 if (!kvm_is_mmio_pfn(pfn)) 1281 get_page(pfn_to_page(pfn)); 1282 } 1283 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1284 1285 static int next_segment(unsigned long len, int offset) 1286 { 1287 if (len > PAGE_SIZE - offset) 1288 return PAGE_SIZE - offset; 1289 else 1290 return len; 1291 } 1292 1293 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1294 int len) 1295 { 1296 int r; 1297 unsigned long addr; 1298 1299 addr = gfn_to_hva(kvm, gfn); 1300 if (kvm_is_error_hva(addr)) 1301 return -EFAULT; 1302 r = __copy_from_user(data, (void __user *)addr + offset, len); 1303 if (r) 1304 return -EFAULT; 1305 return 0; 1306 } 1307 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1308 1309 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1310 { 1311 gfn_t gfn = gpa >> PAGE_SHIFT; 1312 int seg; 1313 int offset = offset_in_page(gpa); 1314 int ret; 1315 1316 while ((seg = next_segment(len, offset)) != 0) { 1317 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1318 if (ret < 0) 1319 return ret; 1320 offset = 0; 1321 len -= seg; 1322 data += seg; 1323 ++gfn; 1324 } 1325 return 0; 1326 } 1327 EXPORT_SYMBOL_GPL(kvm_read_guest); 1328 1329 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1330 unsigned long len) 1331 { 1332 int r; 1333 unsigned long addr; 1334 gfn_t gfn = gpa >> PAGE_SHIFT; 1335 int offset = offset_in_page(gpa); 1336 1337 addr = gfn_to_hva(kvm, gfn); 1338 if (kvm_is_error_hva(addr)) 1339 return -EFAULT; 1340 pagefault_disable(); 1341 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1342 pagefault_enable(); 1343 if (r) 1344 return -EFAULT; 1345 return 0; 1346 } 1347 EXPORT_SYMBOL(kvm_read_guest_atomic); 1348 1349 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 1350 int offset, int len) 1351 { 1352 int r; 1353 unsigned long addr; 1354 1355 addr = gfn_to_hva(kvm, gfn); 1356 if (kvm_is_error_hva(addr)) 1357 return -EFAULT; 1358 r = __copy_to_user((void __user *)addr + offset, data, len); 1359 if (r) 1360 return -EFAULT; 1361 mark_page_dirty(kvm, gfn); 1362 return 0; 1363 } 1364 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1365 1366 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1367 unsigned long len) 1368 { 1369 gfn_t gfn = gpa >> PAGE_SHIFT; 1370 int seg; 1371 int offset = offset_in_page(gpa); 1372 int ret; 1373 1374 while ((seg = next_segment(len, offset)) != 0) { 1375 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1376 if (ret < 0) 1377 return ret; 1378 offset = 0; 1379 len -= seg; 1380 data += seg; 1381 ++gfn; 1382 } 1383 return 0; 1384 } 1385 1386 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1387 gpa_t gpa) 1388 { 1389 struct kvm_memslots *slots = kvm_memslots(kvm); 1390 int offset = offset_in_page(gpa); 1391 gfn_t gfn = gpa >> PAGE_SHIFT; 1392 1393 ghc->gpa = gpa; 1394 ghc->generation = slots->generation; 1395 ghc->memslot = gfn_to_memslot(kvm, gfn); 1396 ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL); 1397 if (!kvm_is_error_hva(ghc->hva)) 1398 ghc->hva += offset; 1399 else 1400 return -EFAULT; 1401 1402 return 0; 1403 } 1404 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1405 1406 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1407 void *data, unsigned long len) 1408 { 1409 struct kvm_memslots *slots = kvm_memslots(kvm); 1410 int r; 1411 1412 if (slots->generation != ghc->generation) 1413 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa); 1414 1415 if (kvm_is_error_hva(ghc->hva)) 1416 return -EFAULT; 1417 1418 r = __copy_to_user((void __user *)ghc->hva, data, len); 1419 if (r) 1420 return -EFAULT; 1421 mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1422 1423 return 0; 1424 } 1425 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1426 1427 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1428 void *data, unsigned long len) 1429 { 1430 struct kvm_memslots *slots = kvm_memslots(kvm); 1431 int r; 1432 1433 if (slots->generation != ghc->generation) 1434 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa); 1435 1436 if (kvm_is_error_hva(ghc->hva)) 1437 return -EFAULT; 1438 1439 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1440 if (r) 1441 return -EFAULT; 1442 1443 return 0; 1444 } 1445 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1446 1447 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1448 { 1449 return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page, 1450 offset, len); 1451 } 1452 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1453 1454 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1455 { 1456 gfn_t gfn = gpa >> PAGE_SHIFT; 1457 int seg; 1458 int offset = offset_in_page(gpa); 1459 int ret; 1460 1461 while ((seg = next_segment(len, offset)) != 0) { 1462 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1463 if (ret < 0) 1464 return ret; 1465 offset = 0; 1466 len -= seg; 1467 ++gfn; 1468 } 1469 return 0; 1470 } 1471 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1472 1473 void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot, 1474 gfn_t gfn) 1475 { 1476 if (memslot && memslot->dirty_bitmap) { 1477 unsigned long rel_gfn = gfn - memslot->base_gfn; 1478 1479 if (!test_and_set_bit_le(rel_gfn, memslot->dirty_bitmap)) 1480 memslot->nr_dirty_pages++; 1481 } 1482 } 1483 1484 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 1485 { 1486 struct kvm_memory_slot *memslot; 1487 1488 memslot = gfn_to_memslot(kvm, gfn); 1489 mark_page_dirty_in_slot(kvm, memslot, gfn); 1490 } 1491 1492 /* 1493 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 1494 */ 1495 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 1496 { 1497 DEFINE_WAIT(wait); 1498 1499 for (;;) { 1500 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 1501 1502 if (kvm_arch_vcpu_runnable(vcpu)) { 1503 kvm_make_request(KVM_REQ_UNHALT, vcpu); 1504 break; 1505 } 1506 if (kvm_cpu_has_pending_timer(vcpu)) 1507 break; 1508 if (signal_pending(current)) 1509 break; 1510 1511 schedule(); 1512 } 1513 1514 finish_wait(&vcpu->wq, &wait); 1515 } 1516 1517 void kvm_resched(struct kvm_vcpu *vcpu) 1518 { 1519 if (!need_resched()) 1520 return; 1521 cond_resched(); 1522 } 1523 EXPORT_SYMBOL_GPL(kvm_resched); 1524 1525 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 1526 { 1527 struct kvm *kvm = me->kvm; 1528 struct kvm_vcpu *vcpu; 1529 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 1530 int yielded = 0; 1531 int pass; 1532 int i; 1533 1534 /* 1535 * We boost the priority of a VCPU that is runnable but not 1536 * currently running, because it got preempted by something 1537 * else and called schedule in __vcpu_run. Hopefully that 1538 * VCPU is holding the lock that we need and will release it. 1539 * We approximate round-robin by starting at the last boosted VCPU. 1540 */ 1541 for (pass = 0; pass < 2 && !yielded; pass++) { 1542 kvm_for_each_vcpu(i, vcpu, kvm) { 1543 struct task_struct *task = NULL; 1544 struct pid *pid; 1545 if (!pass && i < last_boosted_vcpu) { 1546 i = last_boosted_vcpu; 1547 continue; 1548 } else if (pass && i > last_boosted_vcpu) 1549 break; 1550 if (vcpu == me) 1551 continue; 1552 if (waitqueue_active(&vcpu->wq)) 1553 continue; 1554 rcu_read_lock(); 1555 pid = rcu_dereference(vcpu->pid); 1556 if (pid) 1557 task = get_pid_task(vcpu->pid, PIDTYPE_PID); 1558 rcu_read_unlock(); 1559 if (!task) 1560 continue; 1561 if (task->flags & PF_VCPU) { 1562 put_task_struct(task); 1563 continue; 1564 } 1565 if (yield_to(task, 1)) { 1566 put_task_struct(task); 1567 kvm->last_boosted_vcpu = i; 1568 yielded = 1; 1569 break; 1570 } 1571 put_task_struct(task); 1572 } 1573 } 1574 } 1575 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 1576 1577 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1578 { 1579 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 1580 struct page *page; 1581 1582 if (vmf->pgoff == 0) 1583 page = virt_to_page(vcpu->run); 1584 #ifdef CONFIG_X86 1585 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 1586 page = virt_to_page(vcpu->arch.pio_data); 1587 #endif 1588 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 1589 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 1590 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 1591 #endif 1592 else 1593 return kvm_arch_vcpu_fault(vcpu, vmf); 1594 get_page(page); 1595 vmf->page = page; 1596 return 0; 1597 } 1598 1599 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 1600 .fault = kvm_vcpu_fault, 1601 }; 1602 1603 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 1604 { 1605 vma->vm_ops = &kvm_vcpu_vm_ops; 1606 return 0; 1607 } 1608 1609 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 1610 { 1611 struct kvm_vcpu *vcpu = filp->private_data; 1612 1613 kvm_put_kvm(vcpu->kvm); 1614 return 0; 1615 } 1616 1617 static struct file_operations kvm_vcpu_fops = { 1618 .release = kvm_vcpu_release, 1619 .unlocked_ioctl = kvm_vcpu_ioctl, 1620 #ifdef CONFIG_COMPAT 1621 .compat_ioctl = kvm_vcpu_compat_ioctl, 1622 #endif 1623 .mmap = kvm_vcpu_mmap, 1624 .llseek = noop_llseek, 1625 }; 1626 1627 /* 1628 * Allocates an inode for the vcpu. 1629 */ 1630 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 1631 { 1632 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR); 1633 } 1634 1635 /* 1636 * Creates some virtual cpus. Good luck creating more than one. 1637 */ 1638 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 1639 { 1640 int r; 1641 struct kvm_vcpu *vcpu, *v; 1642 1643 vcpu = kvm_arch_vcpu_create(kvm, id); 1644 if (IS_ERR(vcpu)) 1645 return PTR_ERR(vcpu); 1646 1647 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 1648 1649 r = kvm_arch_vcpu_setup(vcpu); 1650 if (r) 1651 goto vcpu_destroy; 1652 1653 mutex_lock(&kvm->lock); 1654 if (!kvm_vcpu_compatible(vcpu)) { 1655 r = -EINVAL; 1656 goto unlock_vcpu_destroy; 1657 } 1658 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 1659 r = -EINVAL; 1660 goto unlock_vcpu_destroy; 1661 } 1662 1663 kvm_for_each_vcpu(r, v, kvm) 1664 if (v->vcpu_id == id) { 1665 r = -EEXIST; 1666 goto unlock_vcpu_destroy; 1667 } 1668 1669 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 1670 1671 /* Now it's all set up, let userspace reach it */ 1672 kvm_get_kvm(kvm); 1673 r = create_vcpu_fd(vcpu); 1674 if (r < 0) { 1675 kvm_put_kvm(kvm); 1676 goto unlock_vcpu_destroy; 1677 } 1678 1679 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 1680 smp_wmb(); 1681 atomic_inc(&kvm->online_vcpus); 1682 1683 mutex_unlock(&kvm->lock); 1684 return r; 1685 1686 unlock_vcpu_destroy: 1687 mutex_unlock(&kvm->lock); 1688 vcpu_destroy: 1689 kvm_arch_vcpu_destroy(vcpu); 1690 return r; 1691 } 1692 1693 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 1694 { 1695 if (sigset) { 1696 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 1697 vcpu->sigset_active = 1; 1698 vcpu->sigset = *sigset; 1699 } else 1700 vcpu->sigset_active = 0; 1701 return 0; 1702 } 1703 1704 static long kvm_vcpu_ioctl(struct file *filp, 1705 unsigned int ioctl, unsigned long arg) 1706 { 1707 struct kvm_vcpu *vcpu = filp->private_data; 1708 void __user *argp = (void __user *)arg; 1709 int r; 1710 struct kvm_fpu *fpu = NULL; 1711 struct kvm_sregs *kvm_sregs = NULL; 1712 1713 if (vcpu->kvm->mm != current->mm) 1714 return -EIO; 1715 1716 #if defined(CONFIG_S390) || defined(CONFIG_PPC) 1717 /* 1718 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 1719 * so vcpu_load() would break it. 1720 */ 1721 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT) 1722 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 1723 #endif 1724 1725 1726 vcpu_load(vcpu); 1727 switch (ioctl) { 1728 case KVM_RUN: 1729 r = -EINVAL; 1730 if (arg) 1731 goto out; 1732 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 1733 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 1734 break; 1735 case KVM_GET_REGS: { 1736 struct kvm_regs *kvm_regs; 1737 1738 r = -ENOMEM; 1739 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 1740 if (!kvm_regs) 1741 goto out; 1742 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 1743 if (r) 1744 goto out_free1; 1745 r = -EFAULT; 1746 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 1747 goto out_free1; 1748 r = 0; 1749 out_free1: 1750 kfree(kvm_regs); 1751 break; 1752 } 1753 case KVM_SET_REGS: { 1754 struct kvm_regs *kvm_regs; 1755 1756 r = -ENOMEM; 1757 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 1758 if (IS_ERR(kvm_regs)) { 1759 r = PTR_ERR(kvm_regs); 1760 goto out; 1761 } 1762 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 1763 if (r) 1764 goto out_free2; 1765 r = 0; 1766 out_free2: 1767 kfree(kvm_regs); 1768 break; 1769 } 1770 case KVM_GET_SREGS: { 1771 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 1772 r = -ENOMEM; 1773 if (!kvm_sregs) 1774 goto out; 1775 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 1776 if (r) 1777 goto out; 1778 r = -EFAULT; 1779 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 1780 goto out; 1781 r = 0; 1782 break; 1783 } 1784 case KVM_SET_SREGS: { 1785 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 1786 if (IS_ERR(kvm_sregs)) { 1787 r = PTR_ERR(kvm_sregs); 1788 goto out; 1789 } 1790 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 1791 if (r) 1792 goto out; 1793 r = 0; 1794 break; 1795 } 1796 case KVM_GET_MP_STATE: { 1797 struct kvm_mp_state mp_state; 1798 1799 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 1800 if (r) 1801 goto out; 1802 r = -EFAULT; 1803 if (copy_to_user(argp, &mp_state, sizeof mp_state)) 1804 goto out; 1805 r = 0; 1806 break; 1807 } 1808 case KVM_SET_MP_STATE: { 1809 struct kvm_mp_state mp_state; 1810 1811 r = -EFAULT; 1812 if (copy_from_user(&mp_state, argp, sizeof mp_state)) 1813 goto out; 1814 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 1815 if (r) 1816 goto out; 1817 r = 0; 1818 break; 1819 } 1820 case KVM_TRANSLATE: { 1821 struct kvm_translation tr; 1822 1823 r = -EFAULT; 1824 if (copy_from_user(&tr, argp, sizeof tr)) 1825 goto out; 1826 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 1827 if (r) 1828 goto out; 1829 r = -EFAULT; 1830 if (copy_to_user(argp, &tr, sizeof tr)) 1831 goto out; 1832 r = 0; 1833 break; 1834 } 1835 case KVM_SET_GUEST_DEBUG: { 1836 struct kvm_guest_debug dbg; 1837 1838 r = -EFAULT; 1839 if (copy_from_user(&dbg, argp, sizeof dbg)) 1840 goto out; 1841 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 1842 if (r) 1843 goto out; 1844 r = 0; 1845 break; 1846 } 1847 case KVM_SET_SIGNAL_MASK: { 1848 struct kvm_signal_mask __user *sigmask_arg = argp; 1849 struct kvm_signal_mask kvm_sigmask; 1850 sigset_t sigset, *p; 1851 1852 p = NULL; 1853 if (argp) { 1854 r = -EFAULT; 1855 if (copy_from_user(&kvm_sigmask, argp, 1856 sizeof kvm_sigmask)) 1857 goto out; 1858 r = -EINVAL; 1859 if (kvm_sigmask.len != sizeof sigset) 1860 goto out; 1861 r = -EFAULT; 1862 if (copy_from_user(&sigset, sigmask_arg->sigset, 1863 sizeof sigset)) 1864 goto out; 1865 p = &sigset; 1866 } 1867 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 1868 break; 1869 } 1870 case KVM_GET_FPU: { 1871 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 1872 r = -ENOMEM; 1873 if (!fpu) 1874 goto out; 1875 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 1876 if (r) 1877 goto out; 1878 r = -EFAULT; 1879 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 1880 goto out; 1881 r = 0; 1882 break; 1883 } 1884 case KVM_SET_FPU: { 1885 fpu = memdup_user(argp, sizeof(*fpu)); 1886 if (IS_ERR(fpu)) { 1887 r = PTR_ERR(fpu); 1888 goto out; 1889 } 1890 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 1891 if (r) 1892 goto out; 1893 r = 0; 1894 break; 1895 } 1896 default: 1897 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 1898 } 1899 out: 1900 vcpu_put(vcpu); 1901 kfree(fpu); 1902 kfree(kvm_sregs); 1903 return r; 1904 } 1905 1906 #ifdef CONFIG_COMPAT 1907 static long kvm_vcpu_compat_ioctl(struct file *filp, 1908 unsigned int ioctl, unsigned long arg) 1909 { 1910 struct kvm_vcpu *vcpu = filp->private_data; 1911 void __user *argp = compat_ptr(arg); 1912 int r; 1913 1914 if (vcpu->kvm->mm != current->mm) 1915 return -EIO; 1916 1917 switch (ioctl) { 1918 case KVM_SET_SIGNAL_MASK: { 1919 struct kvm_signal_mask __user *sigmask_arg = argp; 1920 struct kvm_signal_mask kvm_sigmask; 1921 compat_sigset_t csigset; 1922 sigset_t sigset; 1923 1924 if (argp) { 1925 r = -EFAULT; 1926 if (copy_from_user(&kvm_sigmask, argp, 1927 sizeof kvm_sigmask)) 1928 goto out; 1929 r = -EINVAL; 1930 if (kvm_sigmask.len != sizeof csigset) 1931 goto out; 1932 r = -EFAULT; 1933 if (copy_from_user(&csigset, sigmask_arg->sigset, 1934 sizeof csigset)) 1935 goto out; 1936 } 1937 sigset_from_compat(&sigset, &csigset); 1938 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 1939 break; 1940 } 1941 default: 1942 r = kvm_vcpu_ioctl(filp, ioctl, arg); 1943 } 1944 1945 out: 1946 return r; 1947 } 1948 #endif 1949 1950 static long kvm_vm_ioctl(struct file *filp, 1951 unsigned int ioctl, unsigned long arg) 1952 { 1953 struct kvm *kvm = filp->private_data; 1954 void __user *argp = (void __user *)arg; 1955 int r; 1956 1957 if (kvm->mm != current->mm) 1958 return -EIO; 1959 switch (ioctl) { 1960 case KVM_CREATE_VCPU: 1961 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 1962 if (r < 0) 1963 goto out; 1964 break; 1965 case KVM_SET_USER_MEMORY_REGION: { 1966 struct kvm_userspace_memory_region kvm_userspace_mem; 1967 1968 r = -EFAULT; 1969 if (copy_from_user(&kvm_userspace_mem, argp, 1970 sizeof kvm_userspace_mem)) 1971 goto out; 1972 1973 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1); 1974 if (r) 1975 goto out; 1976 break; 1977 } 1978 case KVM_GET_DIRTY_LOG: { 1979 struct kvm_dirty_log log; 1980 1981 r = -EFAULT; 1982 if (copy_from_user(&log, argp, sizeof log)) 1983 goto out; 1984 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 1985 if (r) 1986 goto out; 1987 break; 1988 } 1989 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 1990 case KVM_REGISTER_COALESCED_MMIO: { 1991 struct kvm_coalesced_mmio_zone zone; 1992 r = -EFAULT; 1993 if (copy_from_user(&zone, argp, sizeof zone)) 1994 goto out; 1995 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 1996 if (r) 1997 goto out; 1998 r = 0; 1999 break; 2000 } 2001 case KVM_UNREGISTER_COALESCED_MMIO: { 2002 struct kvm_coalesced_mmio_zone zone; 2003 r = -EFAULT; 2004 if (copy_from_user(&zone, argp, sizeof zone)) 2005 goto out; 2006 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2007 if (r) 2008 goto out; 2009 r = 0; 2010 break; 2011 } 2012 #endif 2013 case KVM_IRQFD: { 2014 struct kvm_irqfd data; 2015 2016 r = -EFAULT; 2017 if (copy_from_user(&data, argp, sizeof data)) 2018 goto out; 2019 r = kvm_irqfd(kvm, data.fd, data.gsi, data.flags); 2020 break; 2021 } 2022 case KVM_IOEVENTFD: { 2023 struct kvm_ioeventfd data; 2024 2025 r = -EFAULT; 2026 if (copy_from_user(&data, argp, sizeof data)) 2027 goto out; 2028 r = kvm_ioeventfd(kvm, &data); 2029 break; 2030 } 2031 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2032 case KVM_SET_BOOT_CPU_ID: 2033 r = 0; 2034 mutex_lock(&kvm->lock); 2035 if (atomic_read(&kvm->online_vcpus) != 0) 2036 r = -EBUSY; 2037 else 2038 kvm->bsp_vcpu_id = arg; 2039 mutex_unlock(&kvm->lock); 2040 break; 2041 #endif 2042 default: 2043 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2044 if (r == -ENOTTY) 2045 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg); 2046 } 2047 out: 2048 return r; 2049 } 2050 2051 #ifdef CONFIG_COMPAT 2052 struct compat_kvm_dirty_log { 2053 __u32 slot; 2054 __u32 padding1; 2055 union { 2056 compat_uptr_t dirty_bitmap; /* one bit per page */ 2057 __u64 padding2; 2058 }; 2059 }; 2060 2061 static long kvm_vm_compat_ioctl(struct file *filp, 2062 unsigned int ioctl, unsigned long arg) 2063 { 2064 struct kvm *kvm = filp->private_data; 2065 int r; 2066 2067 if (kvm->mm != current->mm) 2068 return -EIO; 2069 switch (ioctl) { 2070 case KVM_GET_DIRTY_LOG: { 2071 struct compat_kvm_dirty_log compat_log; 2072 struct kvm_dirty_log log; 2073 2074 r = -EFAULT; 2075 if (copy_from_user(&compat_log, (void __user *)arg, 2076 sizeof(compat_log))) 2077 goto out; 2078 log.slot = compat_log.slot; 2079 log.padding1 = compat_log.padding1; 2080 log.padding2 = compat_log.padding2; 2081 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 2082 2083 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2084 if (r) 2085 goto out; 2086 break; 2087 } 2088 default: 2089 r = kvm_vm_ioctl(filp, ioctl, arg); 2090 } 2091 2092 out: 2093 return r; 2094 } 2095 #endif 2096 2097 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2098 { 2099 struct page *page[1]; 2100 unsigned long addr; 2101 int npages; 2102 gfn_t gfn = vmf->pgoff; 2103 struct kvm *kvm = vma->vm_file->private_data; 2104 2105 addr = gfn_to_hva(kvm, gfn); 2106 if (kvm_is_error_hva(addr)) 2107 return VM_FAULT_SIGBUS; 2108 2109 npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page, 2110 NULL); 2111 if (unlikely(npages != 1)) 2112 return VM_FAULT_SIGBUS; 2113 2114 vmf->page = page[0]; 2115 return 0; 2116 } 2117 2118 static const struct vm_operations_struct kvm_vm_vm_ops = { 2119 .fault = kvm_vm_fault, 2120 }; 2121 2122 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma) 2123 { 2124 vma->vm_ops = &kvm_vm_vm_ops; 2125 return 0; 2126 } 2127 2128 static struct file_operations kvm_vm_fops = { 2129 .release = kvm_vm_release, 2130 .unlocked_ioctl = kvm_vm_ioctl, 2131 #ifdef CONFIG_COMPAT 2132 .compat_ioctl = kvm_vm_compat_ioctl, 2133 #endif 2134 .mmap = kvm_vm_mmap, 2135 .llseek = noop_llseek, 2136 }; 2137 2138 static int kvm_dev_ioctl_create_vm(unsigned long type) 2139 { 2140 int r; 2141 struct kvm *kvm; 2142 2143 kvm = kvm_create_vm(type); 2144 if (IS_ERR(kvm)) 2145 return PTR_ERR(kvm); 2146 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2147 r = kvm_coalesced_mmio_init(kvm); 2148 if (r < 0) { 2149 kvm_put_kvm(kvm); 2150 return r; 2151 } 2152 #endif 2153 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); 2154 if (r < 0) 2155 kvm_put_kvm(kvm); 2156 2157 return r; 2158 } 2159 2160 static long kvm_dev_ioctl_check_extension_generic(long arg) 2161 { 2162 switch (arg) { 2163 case KVM_CAP_USER_MEMORY: 2164 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2165 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2166 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 2167 case KVM_CAP_SET_BOOT_CPU_ID: 2168 #endif 2169 case KVM_CAP_INTERNAL_ERROR_DATA: 2170 return 1; 2171 #ifdef CONFIG_HAVE_KVM_IRQCHIP 2172 case KVM_CAP_IRQ_ROUTING: 2173 return KVM_MAX_IRQ_ROUTES; 2174 #endif 2175 default: 2176 break; 2177 } 2178 return kvm_dev_ioctl_check_extension(arg); 2179 } 2180 2181 static long kvm_dev_ioctl(struct file *filp, 2182 unsigned int ioctl, unsigned long arg) 2183 { 2184 long r = -EINVAL; 2185 2186 switch (ioctl) { 2187 case KVM_GET_API_VERSION: 2188 r = -EINVAL; 2189 if (arg) 2190 goto out; 2191 r = KVM_API_VERSION; 2192 break; 2193 case KVM_CREATE_VM: 2194 r = kvm_dev_ioctl_create_vm(arg); 2195 break; 2196 case KVM_CHECK_EXTENSION: 2197 r = kvm_dev_ioctl_check_extension_generic(arg); 2198 break; 2199 case KVM_GET_VCPU_MMAP_SIZE: 2200 r = -EINVAL; 2201 if (arg) 2202 goto out; 2203 r = PAGE_SIZE; /* struct kvm_run */ 2204 #ifdef CONFIG_X86 2205 r += PAGE_SIZE; /* pio data page */ 2206 #endif 2207 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2208 r += PAGE_SIZE; /* coalesced mmio ring page */ 2209 #endif 2210 break; 2211 case KVM_TRACE_ENABLE: 2212 case KVM_TRACE_PAUSE: 2213 case KVM_TRACE_DISABLE: 2214 r = -EOPNOTSUPP; 2215 break; 2216 default: 2217 return kvm_arch_dev_ioctl(filp, ioctl, arg); 2218 } 2219 out: 2220 return r; 2221 } 2222 2223 static struct file_operations kvm_chardev_ops = { 2224 .unlocked_ioctl = kvm_dev_ioctl, 2225 .compat_ioctl = kvm_dev_ioctl, 2226 .llseek = noop_llseek, 2227 }; 2228 2229 static struct miscdevice kvm_dev = { 2230 KVM_MINOR, 2231 "kvm", 2232 &kvm_chardev_ops, 2233 }; 2234 2235 static void hardware_enable_nolock(void *junk) 2236 { 2237 int cpu = raw_smp_processor_id(); 2238 int r; 2239 2240 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2241 return; 2242 2243 cpumask_set_cpu(cpu, cpus_hardware_enabled); 2244 2245 r = kvm_arch_hardware_enable(NULL); 2246 2247 if (r) { 2248 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2249 atomic_inc(&hardware_enable_failed); 2250 printk(KERN_INFO "kvm: enabling virtualization on " 2251 "CPU%d failed\n", cpu); 2252 } 2253 } 2254 2255 static void hardware_enable(void *junk) 2256 { 2257 raw_spin_lock(&kvm_lock); 2258 hardware_enable_nolock(junk); 2259 raw_spin_unlock(&kvm_lock); 2260 } 2261 2262 static void hardware_disable_nolock(void *junk) 2263 { 2264 int cpu = raw_smp_processor_id(); 2265 2266 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2267 return; 2268 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2269 kvm_arch_hardware_disable(NULL); 2270 } 2271 2272 static void hardware_disable(void *junk) 2273 { 2274 raw_spin_lock(&kvm_lock); 2275 hardware_disable_nolock(junk); 2276 raw_spin_unlock(&kvm_lock); 2277 } 2278 2279 static void hardware_disable_all_nolock(void) 2280 { 2281 BUG_ON(!kvm_usage_count); 2282 2283 kvm_usage_count--; 2284 if (!kvm_usage_count) 2285 on_each_cpu(hardware_disable_nolock, NULL, 1); 2286 } 2287 2288 static void hardware_disable_all(void) 2289 { 2290 raw_spin_lock(&kvm_lock); 2291 hardware_disable_all_nolock(); 2292 raw_spin_unlock(&kvm_lock); 2293 } 2294 2295 static int hardware_enable_all(void) 2296 { 2297 int r = 0; 2298 2299 raw_spin_lock(&kvm_lock); 2300 2301 kvm_usage_count++; 2302 if (kvm_usage_count == 1) { 2303 atomic_set(&hardware_enable_failed, 0); 2304 on_each_cpu(hardware_enable_nolock, NULL, 1); 2305 2306 if (atomic_read(&hardware_enable_failed)) { 2307 hardware_disable_all_nolock(); 2308 r = -EBUSY; 2309 } 2310 } 2311 2312 raw_spin_unlock(&kvm_lock); 2313 2314 return r; 2315 } 2316 2317 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 2318 void *v) 2319 { 2320 int cpu = (long)v; 2321 2322 if (!kvm_usage_count) 2323 return NOTIFY_OK; 2324 2325 val &= ~CPU_TASKS_FROZEN; 2326 switch (val) { 2327 case CPU_DYING: 2328 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", 2329 cpu); 2330 hardware_disable(NULL); 2331 break; 2332 case CPU_STARTING: 2333 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", 2334 cpu); 2335 hardware_enable(NULL); 2336 break; 2337 } 2338 return NOTIFY_OK; 2339 } 2340 2341 2342 asmlinkage void kvm_spurious_fault(void) 2343 { 2344 /* Fault while not rebooting. We want the trace. */ 2345 BUG(); 2346 } 2347 EXPORT_SYMBOL_GPL(kvm_spurious_fault); 2348 2349 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 2350 void *v) 2351 { 2352 /* 2353 * Some (well, at least mine) BIOSes hang on reboot if 2354 * in vmx root mode. 2355 * 2356 * And Intel TXT required VMX off for all cpu when system shutdown. 2357 */ 2358 printk(KERN_INFO "kvm: exiting hardware virtualization\n"); 2359 kvm_rebooting = true; 2360 on_each_cpu(hardware_disable_nolock, NULL, 1); 2361 return NOTIFY_OK; 2362 } 2363 2364 static struct notifier_block kvm_reboot_notifier = { 2365 .notifier_call = kvm_reboot, 2366 .priority = 0, 2367 }; 2368 2369 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 2370 { 2371 int i; 2372 2373 for (i = 0; i < bus->dev_count; i++) { 2374 struct kvm_io_device *pos = bus->range[i].dev; 2375 2376 kvm_iodevice_destructor(pos); 2377 } 2378 kfree(bus); 2379 } 2380 2381 int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 2382 { 2383 const struct kvm_io_range *r1 = p1; 2384 const struct kvm_io_range *r2 = p2; 2385 2386 if (r1->addr < r2->addr) 2387 return -1; 2388 if (r1->addr + r1->len > r2->addr + r2->len) 2389 return 1; 2390 return 0; 2391 } 2392 2393 int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 2394 gpa_t addr, int len) 2395 { 2396 if (bus->dev_count == NR_IOBUS_DEVS) 2397 return -ENOSPC; 2398 2399 bus->range[bus->dev_count++] = (struct kvm_io_range) { 2400 .addr = addr, 2401 .len = len, 2402 .dev = dev, 2403 }; 2404 2405 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 2406 kvm_io_bus_sort_cmp, NULL); 2407 2408 return 0; 2409 } 2410 2411 int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 2412 gpa_t addr, int len) 2413 { 2414 struct kvm_io_range *range, key; 2415 int off; 2416 2417 key = (struct kvm_io_range) { 2418 .addr = addr, 2419 .len = len, 2420 }; 2421 2422 range = bsearch(&key, bus->range, bus->dev_count, 2423 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 2424 if (range == NULL) 2425 return -ENOENT; 2426 2427 off = range - bus->range; 2428 2429 while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0) 2430 off--; 2431 2432 return off; 2433 } 2434 2435 /* kvm_io_bus_write - called under kvm->slots_lock */ 2436 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2437 int len, const void *val) 2438 { 2439 int idx; 2440 struct kvm_io_bus *bus; 2441 struct kvm_io_range range; 2442 2443 range = (struct kvm_io_range) { 2444 .addr = addr, 2445 .len = len, 2446 }; 2447 2448 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2449 idx = kvm_io_bus_get_first_dev(bus, addr, len); 2450 if (idx < 0) 2451 return -EOPNOTSUPP; 2452 2453 while (idx < bus->dev_count && 2454 kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) { 2455 if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val)) 2456 return 0; 2457 idx++; 2458 } 2459 2460 return -EOPNOTSUPP; 2461 } 2462 2463 /* kvm_io_bus_read - called under kvm->slots_lock */ 2464 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2465 int len, void *val) 2466 { 2467 int idx; 2468 struct kvm_io_bus *bus; 2469 struct kvm_io_range range; 2470 2471 range = (struct kvm_io_range) { 2472 .addr = addr, 2473 .len = len, 2474 }; 2475 2476 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); 2477 idx = kvm_io_bus_get_first_dev(bus, addr, len); 2478 if (idx < 0) 2479 return -EOPNOTSUPP; 2480 2481 while (idx < bus->dev_count && 2482 kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) { 2483 if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val)) 2484 return 0; 2485 idx++; 2486 } 2487 2488 return -EOPNOTSUPP; 2489 } 2490 2491 /* Caller must hold slots_lock. */ 2492 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 2493 int len, struct kvm_io_device *dev) 2494 { 2495 struct kvm_io_bus *new_bus, *bus; 2496 2497 bus = kvm->buses[bus_idx]; 2498 if (bus->dev_count > NR_IOBUS_DEVS-1) 2499 return -ENOSPC; 2500 2501 new_bus = kmemdup(bus, sizeof(struct kvm_io_bus), GFP_KERNEL); 2502 if (!new_bus) 2503 return -ENOMEM; 2504 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 2505 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 2506 synchronize_srcu_expedited(&kvm->srcu); 2507 kfree(bus); 2508 2509 return 0; 2510 } 2511 2512 /* Caller must hold slots_lock. */ 2513 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 2514 struct kvm_io_device *dev) 2515 { 2516 int i, r; 2517 struct kvm_io_bus *new_bus, *bus; 2518 2519 bus = kvm->buses[bus_idx]; 2520 2521 new_bus = kmemdup(bus, sizeof(*bus), GFP_KERNEL); 2522 if (!new_bus) 2523 return -ENOMEM; 2524 2525 r = -ENOENT; 2526 for (i = 0; i < new_bus->dev_count; i++) 2527 if (new_bus->range[i].dev == dev) { 2528 r = 0; 2529 new_bus->dev_count--; 2530 new_bus->range[i] = new_bus->range[new_bus->dev_count]; 2531 sort(new_bus->range, new_bus->dev_count, 2532 sizeof(struct kvm_io_range), 2533 kvm_io_bus_sort_cmp, NULL); 2534 break; 2535 } 2536 2537 if (r) { 2538 kfree(new_bus); 2539 return r; 2540 } 2541 2542 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 2543 synchronize_srcu_expedited(&kvm->srcu); 2544 kfree(bus); 2545 return r; 2546 } 2547 2548 static struct notifier_block kvm_cpu_notifier = { 2549 .notifier_call = kvm_cpu_hotplug, 2550 }; 2551 2552 static int vm_stat_get(void *_offset, u64 *val) 2553 { 2554 unsigned offset = (long)_offset; 2555 struct kvm *kvm; 2556 2557 *val = 0; 2558 raw_spin_lock(&kvm_lock); 2559 list_for_each_entry(kvm, &vm_list, vm_list) 2560 *val += *(u32 *)((void *)kvm + offset); 2561 raw_spin_unlock(&kvm_lock); 2562 return 0; 2563 } 2564 2565 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 2566 2567 static int vcpu_stat_get(void *_offset, u64 *val) 2568 { 2569 unsigned offset = (long)_offset; 2570 struct kvm *kvm; 2571 struct kvm_vcpu *vcpu; 2572 int i; 2573 2574 *val = 0; 2575 raw_spin_lock(&kvm_lock); 2576 list_for_each_entry(kvm, &vm_list, vm_list) 2577 kvm_for_each_vcpu(i, vcpu, kvm) 2578 *val += *(u32 *)((void *)vcpu + offset); 2579 2580 raw_spin_unlock(&kvm_lock); 2581 return 0; 2582 } 2583 2584 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 2585 2586 static const struct file_operations *stat_fops[] = { 2587 [KVM_STAT_VCPU] = &vcpu_stat_fops, 2588 [KVM_STAT_VM] = &vm_stat_fops, 2589 }; 2590 2591 static int kvm_init_debug(void) 2592 { 2593 int r = -EFAULT; 2594 struct kvm_stats_debugfs_item *p; 2595 2596 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 2597 if (kvm_debugfs_dir == NULL) 2598 goto out; 2599 2600 for (p = debugfs_entries; p->name; ++p) { 2601 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 2602 (void *)(long)p->offset, 2603 stat_fops[p->kind]); 2604 if (p->dentry == NULL) 2605 goto out_dir; 2606 } 2607 2608 return 0; 2609 2610 out_dir: 2611 debugfs_remove_recursive(kvm_debugfs_dir); 2612 out: 2613 return r; 2614 } 2615 2616 static void kvm_exit_debug(void) 2617 { 2618 struct kvm_stats_debugfs_item *p; 2619 2620 for (p = debugfs_entries; p->name; ++p) 2621 debugfs_remove(p->dentry); 2622 debugfs_remove(kvm_debugfs_dir); 2623 } 2624 2625 static int kvm_suspend(void) 2626 { 2627 if (kvm_usage_count) 2628 hardware_disable_nolock(NULL); 2629 return 0; 2630 } 2631 2632 static void kvm_resume(void) 2633 { 2634 if (kvm_usage_count) { 2635 WARN_ON(raw_spin_is_locked(&kvm_lock)); 2636 hardware_enable_nolock(NULL); 2637 } 2638 } 2639 2640 static struct syscore_ops kvm_syscore_ops = { 2641 .suspend = kvm_suspend, 2642 .resume = kvm_resume, 2643 }; 2644 2645 struct page *bad_page; 2646 pfn_t bad_pfn; 2647 2648 static inline 2649 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 2650 { 2651 return container_of(pn, struct kvm_vcpu, preempt_notifier); 2652 } 2653 2654 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 2655 { 2656 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 2657 2658 kvm_arch_vcpu_load(vcpu, cpu); 2659 } 2660 2661 static void kvm_sched_out(struct preempt_notifier *pn, 2662 struct task_struct *next) 2663 { 2664 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 2665 2666 kvm_arch_vcpu_put(vcpu); 2667 } 2668 2669 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 2670 struct module *module) 2671 { 2672 int r; 2673 int cpu; 2674 2675 r = kvm_arch_init(opaque); 2676 if (r) 2677 goto out_fail; 2678 2679 bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); 2680 2681 if (bad_page == NULL) { 2682 r = -ENOMEM; 2683 goto out; 2684 } 2685 2686 bad_pfn = page_to_pfn(bad_page); 2687 2688 hwpoison_page = alloc_page(GFP_KERNEL | __GFP_ZERO); 2689 2690 if (hwpoison_page == NULL) { 2691 r = -ENOMEM; 2692 goto out_free_0; 2693 } 2694 2695 hwpoison_pfn = page_to_pfn(hwpoison_page); 2696 2697 fault_page = alloc_page(GFP_KERNEL | __GFP_ZERO); 2698 2699 if (fault_page == NULL) { 2700 r = -ENOMEM; 2701 goto out_free_0; 2702 } 2703 2704 fault_pfn = page_to_pfn(fault_page); 2705 2706 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 2707 r = -ENOMEM; 2708 goto out_free_0; 2709 } 2710 2711 r = kvm_arch_hardware_setup(); 2712 if (r < 0) 2713 goto out_free_0a; 2714 2715 for_each_online_cpu(cpu) { 2716 smp_call_function_single(cpu, 2717 kvm_arch_check_processor_compat, 2718 &r, 1); 2719 if (r < 0) 2720 goto out_free_1; 2721 } 2722 2723 r = register_cpu_notifier(&kvm_cpu_notifier); 2724 if (r) 2725 goto out_free_2; 2726 register_reboot_notifier(&kvm_reboot_notifier); 2727 2728 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 2729 if (!vcpu_align) 2730 vcpu_align = __alignof__(struct kvm_vcpu); 2731 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 2732 0, NULL); 2733 if (!kvm_vcpu_cache) { 2734 r = -ENOMEM; 2735 goto out_free_3; 2736 } 2737 2738 r = kvm_async_pf_init(); 2739 if (r) 2740 goto out_free; 2741 2742 kvm_chardev_ops.owner = module; 2743 kvm_vm_fops.owner = module; 2744 kvm_vcpu_fops.owner = module; 2745 2746 r = misc_register(&kvm_dev); 2747 if (r) { 2748 printk(KERN_ERR "kvm: misc device register failed\n"); 2749 goto out_unreg; 2750 } 2751 2752 register_syscore_ops(&kvm_syscore_ops); 2753 2754 kvm_preempt_ops.sched_in = kvm_sched_in; 2755 kvm_preempt_ops.sched_out = kvm_sched_out; 2756 2757 r = kvm_init_debug(); 2758 if (r) { 2759 printk(KERN_ERR "kvm: create debugfs files failed\n"); 2760 goto out_undebugfs; 2761 } 2762 2763 return 0; 2764 2765 out_undebugfs: 2766 unregister_syscore_ops(&kvm_syscore_ops); 2767 out_unreg: 2768 kvm_async_pf_deinit(); 2769 out_free: 2770 kmem_cache_destroy(kvm_vcpu_cache); 2771 out_free_3: 2772 unregister_reboot_notifier(&kvm_reboot_notifier); 2773 unregister_cpu_notifier(&kvm_cpu_notifier); 2774 out_free_2: 2775 out_free_1: 2776 kvm_arch_hardware_unsetup(); 2777 out_free_0a: 2778 free_cpumask_var(cpus_hardware_enabled); 2779 out_free_0: 2780 if (fault_page) 2781 __free_page(fault_page); 2782 if (hwpoison_page) 2783 __free_page(hwpoison_page); 2784 __free_page(bad_page); 2785 out: 2786 kvm_arch_exit(); 2787 out_fail: 2788 return r; 2789 } 2790 EXPORT_SYMBOL_GPL(kvm_init); 2791 2792 void kvm_exit(void) 2793 { 2794 kvm_exit_debug(); 2795 misc_deregister(&kvm_dev); 2796 kmem_cache_destroy(kvm_vcpu_cache); 2797 kvm_async_pf_deinit(); 2798 unregister_syscore_ops(&kvm_syscore_ops); 2799 unregister_reboot_notifier(&kvm_reboot_notifier); 2800 unregister_cpu_notifier(&kvm_cpu_notifier); 2801 on_each_cpu(hardware_disable_nolock, NULL, 1); 2802 kvm_arch_hardware_unsetup(); 2803 kvm_arch_exit(); 2804 free_cpumask_var(cpus_hardware_enabled); 2805 __free_page(hwpoison_page); 2806 __free_page(bad_page); 2807 } 2808 EXPORT_SYMBOL_GPL(kvm_exit); 2809