1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include <kvm/iodev.h> 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 #include <linux/sort.h> 51 #include <linux/bsearch.h> 52 53 #include <asm/processor.h> 54 #include <asm/io.h> 55 #include <asm/ioctl.h> 56 #include <asm/uaccess.h> 57 #include <asm/pgtable.h> 58 59 #include "coalesced_mmio.h" 60 #include "async_pf.h" 61 #include "vfio.h" 62 63 #define CREATE_TRACE_POINTS 64 #include <trace/events/kvm.h> 65 66 MODULE_AUTHOR("Qumranet"); 67 MODULE_LICENSE("GPL"); 68 69 static unsigned int halt_poll_ns; 70 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR); 71 72 /* 73 * Ordering of locks: 74 * 75 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 76 */ 77 78 DEFINE_SPINLOCK(kvm_lock); 79 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 80 LIST_HEAD(vm_list); 81 82 static cpumask_var_t cpus_hardware_enabled; 83 static int kvm_usage_count; 84 static atomic_t hardware_enable_failed; 85 86 struct kmem_cache *kvm_vcpu_cache; 87 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 88 89 static __read_mostly struct preempt_ops kvm_preempt_ops; 90 91 struct dentry *kvm_debugfs_dir; 92 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 93 94 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 95 unsigned long arg); 96 #ifdef CONFIG_KVM_COMPAT 97 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 98 unsigned long arg); 99 #endif 100 static int hardware_enable_all(void); 101 static void hardware_disable_all(void); 102 103 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 104 105 static void kvm_release_pfn_dirty(pfn_t pfn); 106 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn); 107 108 __visible bool kvm_rebooting; 109 EXPORT_SYMBOL_GPL(kvm_rebooting); 110 111 static bool largepages_enabled = true; 112 113 bool kvm_is_reserved_pfn(pfn_t pfn) 114 { 115 if (pfn_valid(pfn)) 116 return PageReserved(pfn_to_page(pfn)); 117 118 return true; 119 } 120 121 /* 122 * Switches to specified vcpu, until a matching vcpu_put() 123 */ 124 int vcpu_load(struct kvm_vcpu *vcpu) 125 { 126 int cpu; 127 128 if (mutex_lock_killable(&vcpu->mutex)) 129 return -EINTR; 130 cpu = get_cpu(); 131 preempt_notifier_register(&vcpu->preempt_notifier); 132 kvm_arch_vcpu_load(vcpu, cpu); 133 put_cpu(); 134 return 0; 135 } 136 137 void vcpu_put(struct kvm_vcpu *vcpu) 138 { 139 preempt_disable(); 140 kvm_arch_vcpu_put(vcpu); 141 preempt_notifier_unregister(&vcpu->preempt_notifier); 142 preempt_enable(); 143 mutex_unlock(&vcpu->mutex); 144 } 145 146 static void ack_flush(void *_completed) 147 { 148 } 149 150 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 151 { 152 int i, cpu, me; 153 cpumask_var_t cpus; 154 bool called = true; 155 struct kvm_vcpu *vcpu; 156 157 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 158 159 me = get_cpu(); 160 kvm_for_each_vcpu(i, vcpu, kvm) { 161 kvm_make_request(req, vcpu); 162 cpu = vcpu->cpu; 163 164 /* Set ->requests bit before we read ->mode */ 165 smp_mb(); 166 167 if (cpus != NULL && cpu != -1 && cpu != me && 168 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 169 cpumask_set_cpu(cpu, cpus); 170 } 171 if (unlikely(cpus == NULL)) 172 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 173 else if (!cpumask_empty(cpus)) 174 smp_call_function_many(cpus, ack_flush, NULL, 1); 175 else 176 called = false; 177 put_cpu(); 178 free_cpumask_var(cpus); 179 return called; 180 } 181 182 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 183 void kvm_flush_remote_tlbs(struct kvm *kvm) 184 { 185 long dirty_count = kvm->tlbs_dirty; 186 187 smp_mb(); 188 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 189 ++kvm->stat.remote_tlb_flush; 190 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 191 } 192 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 193 #endif 194 195 void kvm_reload_remote_mmus(struct kvm *kvm) 196 { 197 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 198 } 199 200 void kvm_make_mclock_inprogress_request(struct kvm *kvm) 201 { 202 kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS); 203 } 204 205 void kvm_make_scan_ioapic_request(struct kvm *kvm) 206 { 207 kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC); 208 } 209 210 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 211 { 212 struct page *page; 213 int r; 214 215 mutex_init(&vcpu->mutex); 216 vcpu->cpu = -1; 217 vcpu->kvm = kvm; 218 vcpu->vcpu_id = id; 219 vcpu->pid = NULL; 220 init_waitqueue_head(&vcpu->wq); 221 kvm_async_pf_vcpu_init(vcpu); 222 223 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 224 if (!page) { 225 r = -ENOMEM; 226 goto fail; 227 } 228 vcpu->run = page_address(page); 229 230 kvm_vcpu_set_in_spin_loop(vcpu, false); 231 kvm_vcpu_set_dy_eligible(vcpu, false); 232 vcpu->preempted = false; 233 234 r = kvm_arch_vcpu_init(vcpu); 235 if (r < 0) 236 goto fail_free_run; 237 return 0; 238 239 fail_free_run: 240 free_page((unsigned long)vcpu->run); 241 fail: 242 return r; 243 } 244 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 245 246 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 247 { 248 put_pid(vcpu->pid); 249 kvm_arch_vcpu_uninit(vcpu); 250 free_page((unsigned long)vcpu->run); 251 } 252 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 253 254 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 255 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 256 { 257 return container_of(mn, struct kvm, mmu_notifier); 258 } 259 260 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 261 struct mm_struct *mm, 262 unsigned long address) 263 { 264 struct kvm *kvm = mmu_notifier_to_kvm(mn); 265 int need_tlb_flush, idx; 266 267 /* 268 * When ->invalidate_page runs, the linux pte has been zapped 269 * already but the page is still allocated until 270 * ->invalidate_page returns. So if we increase the sequence 271 * here the kvm page fault will notice if the spte can't be 272 * established because the page is going to be freed. If 273 * instead the kvm page fault establishes the spte before 274 * ->invalidate_page runs, kvm_unmap_hva will release it 275 * before returning. 276 * 277 * The sequence increase only need to be seen at spin_unlock 278 * time, and not at spin_lock time. 279 * 280 * Increasing the sequence after the spin_unlock would be 281 * unsafe because the kvm page fault could then establish the 282 * pte after kvm_unmap_hva returned, without noticing the page 283 * is going to be freed. 284 */ 285 idx = srcu_read_lock(&kvm->srcu); 286 spin_lock(&kvm->mmu_lock); 287 288 kvm->mmu_notifier_seq++; 289 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 290 /* we've to flush the tlb before the pages can be freed */ 291 if (need_tlb_flush) 292 kvm_flush_remote_tlbs(kvm); 293 294 spin_unlock(&kvm->mmu_lock); 295 296 kvm_arch_mmu_notifier_invalidate_page(kvm, address); 297 298 srcu_read_unlock(&kvm->srcu, idx); 299 } 300 301 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 302 struct mm_struct *mm, 303 unsigned long address, 304 pte_t pte) 305 { 306 struct kvm *kvm = mmu_notifier_to_kvm(mn); 307 int idx; 308 309 idx = srcu_read_lock(&kvm->srcu); 310 spin_lock(&kvm->mmu_lock); 311 kvm->mmu_notifier_seq++; 312 kvm_set_spte_hva(kvm, address, pte); 313 spin_unlock(&kvm->mmu_lock); 314 srcu_read_unlock(&kvm->srcu, idx); 315 } 316 317 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 318 struct mm_struct *mm, 319 unsigned long start, 320 unsigned long end) 321 { 322 struct kvm *kvm = mmu_notifier_to_kvm(mn); 323 int need_tlb_flush = 0, idx; 324 325 idx = srcu_read_lock(&kvm->srcu); 326 spin_lock(&kvm->mmu_lock); 327 /* 328 * The count increase must become visible at unlock time as no 329 * spte can be established without taking the mmu_lock and 330 * count is also read inside the mmu_lock critical section. 331 */ 332 kvm->mmu_notifier_count++; 333 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 334 need_tlb_flush |= kvm->tlbs_dirty; 335 /* we've to flush the tlb before the pages can be freed */ 336 if (need_tlb_flush) 337 kvm_flush_remote_tlbs(kvm); 338 339 spin_unlock(&kvm->mmu_lock); 340 srcu_read_unlock(&kvm->srcu, idx); 341 } 342 343 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 344 struct mm_struct *mm, 345 unsigned long start, 346 unsigned long end) 347 { 348 struct kvm *kvm = mmu_notifier_to_kvm(mn); 349 350 spin_lock(&kvm->mmu_lock); 351 /* 352 * This sequence increase will notify the kvm page fault that 353 * the page that is going to be mapped in the spte could have 354 * been freed. 355 */ 356 kvm->mmu_notifier_seq++; 357 smp_wmb(); 358 /* 359 * The above sequence increase must be visible before the 360 * below count decrease, which is ensured by the smp_wmb above 361 * in conjunction with the smp_rmb in mmu_notifier_retry(). 362 */ 363 kvm->mmu_notifier_count--; 364 spin_unlock(&kvm->mmu_lock); 365 366 BUG_ON(kvm->mmu_notifier_count < 0); 367 } 368 369 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 370 struct mm_struct *mm, 371 unsigned long start, 372 unsigned long end) 373 { 374 struct kvm *kvm = mmu_notifier_to_kvm(mn); 375 int young, idx; 376 377 idx = srcu_read_lock(&kvm->srcu); 378 spin_lock(&kvm->mmu_lock); 379 380 young = kvm_age_hva(kvm, start, end); 381 if (young) 382 kvm_flush_remote_tlbs(kvm); 383 384 spin_unlock(&kvm->mmu_lock); 385 srcu_read_unlock(&kvm->srcu, idx); 386 387 return young; 388 } 389 390 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 391 struct mm_struct *mm, 392 unsigned long address) 393 { 394 struct kvm *kvm = mmu_notifier_to_kvm(mn); 395 int young, idx; 396 397 idx = srcu_read_lock(&kvm->srcu); 398 spin_lock(&kvm->mmu_lock); 399 young = kvm_test_age_hva(kvm, address); 400 spin_unlock(&kvm->mmu_lock); 401 srcu_read_unlock(&kvm->srcu, idx); 402 403 return young; 404 } 405 406 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 407 struct mm_struct *mm) 408 { 409 struct kvm *kvm = mmu_notifier_to_kvm(mn); 410 int idx; 411 412 idx = srcu_read_lock(&kvm->srcu); 413 kvm_arch_flush_shadow_all(kvm); 414 srcu_read_unlock(&kvm->srcu, idx); 415 } 416 417 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 418 .invalidate_page = kvm_mmu_notifier_invalidate_page, 419 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 420 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 421 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 422 .test_young = kvm_mmu_notifier_test_young, 423 .change_pte = kvm_mmu_notifier_change_pte, 424 .release = kvm_mmu_notifier_release, 425 }; 426 427 static int kvm_init_mmu_notifier(struct kvm *kvm) 428 { 429 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 430 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 431 } 432 433 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 434 435 static int kvm_init_mmu_notifier(struct kvm *kvm) 436 { 437 return 0; 438 } 439 440 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 441 442 static struct kvm_memslots *kvm_alloc_memslots(void) 443 { 444 int i; 445 struct kvm_memslots *slots; 446 447 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 448 if (!slots) 449 return NULL; 450 451 /* 452 * Init kvm generation close to the maximum to easily test the 453 * code of handling generation number wrap-around. 454 */ 455 slots->generation = -150; 456 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 457 slots->id_to_index[i] = slots->memslots[i].id = i; 458 459 return slots; 460 } 461 462 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 463 { 464 if (!memslot->dirty_bitmap) 465 return; 466 467 kvfree(memslot->dirty_bitmap); 468 memslot->dirty_bitmap = NULL; 469 } 470 471 /* 472 * Free any memory in @free but not in @dont. 473 */ 474 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 475 struct kvm_memory_slot *dont) 476 { 477 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 478 kvm_destroy_dirty_bitmap(free); 479 480 kvm_arch_free_memslot(kvm, free, dont); 481 482 free->npages = 0; 483 } 484 485 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 486 { 487 struct kvm_memory_slot *memslot; 488 489 if (!slots) 490 return; 491 492 kvm_for_each_memslot(memslot, slots) 493 kvm_free_memslot(kvm, memslot, NULL); 494 495 kvfree(slots); 496 } 497 498 static struct kvm *kvm_create_vm(unsigned long type) 499 { 500 int r, i; 501 struct kvm *kvm = kvm_arch_alloc_vm(); 502 503 if (!kvm) 504 return ERR_PTR(-ENOMEM); 505 506 r = kvm_arch_init_vm(kvm, type); 507 if (r) 508 goto out_err_no_disable; 509 510 r = hardware_enable_all(); 511 if (r) 512 goto out_err_no_disable; 513 514 #ifdef CONFIG_HAVE_KVM_IRQFD 515 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 516 #endif 517 518 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 519 520 r = -ENOMEM; 521 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 522 kvm->memslots[i] = kvm_alloc_memslots(); 523 if (!kvm->memslots[i]) 524 goto out_err_no_srcu; 525 } 526 527 if (init_srcu_struct(&kvm->srcu)) 528 goto out_err_no_srcu; 529 if (init_srcu_struct(&kvm->irq_srcu)) 530 goto out_err_no_irq_srcu; 531 for (i = 0; i < KVM_NR_BUSES; i++) { 532 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 533 GFP_KERNEL); 534 if (!kvm->buses[i]) 535 goto out_err; 536 } 537 538 spin_lock_init(&kvm->mmu_lock); 539 kvm->mm = current->mm; 540 atomic_inc(&kvm->mm->mm_count); 541 kvm_eventfd_init(kvm); 542 mutex_init(&kvm->lock); 543 mutex_init(&kvm->irq_lock); 544 mutex_init(&kvm->slots_lock); 545 atomic_set(&kvm->users_count, 1); 546 INIT_LIST_HEAD(&kvm->devices); 547 548 r = kvm_init_mmu_notifier(kvm); 549 if (r) 550 goto out_err; 551 552 spin_lock(&kvm_lock); 553 list_add(&kvm->vm_list, &vm_list); 554 spin_unlock(&kvm_lock); 555 556 preempt_notifier_inc(); 557 558 return kvm; 559 560 out_err: 561 cleanup_srcu_struct(&kvm->irq_srcu); 562 out_err_no_irq_srcu: 563 cleanup_srcu_struct(&kvm->srcu); 564 out_err_no_srcu: 565 hardware_disable_all(); 566 out_err_no_disable: 567 for (i = 0; i < KVM_NR_BUSES; i++) 568 kfree(kvm->buses[i]); 569 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 570 kvm_free_memslots(kvm, kvm->memslots[i]); 571 kvm_arch_free_vm(kvm); 572 return ERR_PTR(r); 573 } 574 575 /* 576 * Avoid using vmalloc for a small buffer. 577 * Should not be used when the size is statically known. 578 */ 579 void *kvm_kvzalloc(unsigned long size) 580 { 581 if (size > PAGE_SIZE) 582 return vzalloc(size); 583 else 584 return kzalloc(size, GFP_KERNEL); 585 } 586 587 static void kvm_destroy_devices(struct kvm *kvm) 588 { 589 struct list_head *node, *tmp; 590 591 list_for_each_safe(node, tmp, &kvm->devices) { 592 struct kvm_device *dev = 593 list_entry(node, struct kvm_device, vm_node); 594 595 list_del(node); 596 dev->ops->destroy(dev); 597 } 598 } 599 600 static void kvm_destroy_vm(struct kvm *kvm) 601 { 602 int i; 603 struct mm_struct *mm = kvm->mm; 604 605 kvm_arch_sync_events(kvm); 606 spin_lock(&kvm_lock); 607 list_del(&kvm->vm_list); 608 spin_unlock(&kvm_lock); 609 kvm_free_irq_routing(kvm); 610 for (i = 0; i < KVM_NR_BUSES; i++) 611 kvm_io_bus_destroy(kvm->buses[i]); 612 kvm_coalesced_mmio_free(kvm); 613 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 614 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 615 #else 616 kvm_arch_flush_shadow_all(kvm); 617 #endif 618 kvm_arch_destroy_vm(kvm); 619 kvm_destroy_devices(kvm); 620 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 621 kvm_free_memslots(kvm, kvm->memslots[i]); 622 cleanup_srcu_struct(&kvm->irq_srcu); 623 cleanup_srcu_struct(&kvm->srcu); 624 kvm_arch_free_vm(kvm); 625 preempt_notifier_dec(); 626 hardware_disable_all(); 627 mmdrop(mm); 628 } 629 630 void kvm_get_kvm(struct kvm *kvm) 631 { 632 atomic_inc(&kvm->users_count); 633 } 634 EXPORT_SYMBOL_GPL(kvm_get_kvm); 635 636 void kvm_put_kvm(struct kvm *kvm) 637 { 638 if (atomic_dec_and_test(&kvm->users_count)) 639 kvm_destroy_vm(kvm); 640 } 641 EXPORT_SYMBOL_GPL(kvm_put_kvm); 642 643 644 static int kvm_vm_release(struct inode *inode, struct file *filp) 645 { 646 struct kvm *kvm = filp->private_data; 647 648 kvm_irqfd_release(kvm); 649 650 kvm_put_kvm(kvm); 651 return 0; 652 } 653 654 /* 655 * Allocation size is twice as large as the actual dirty bitmap size. 656 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 657 */ 658 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 659 { 660 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 661 662 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes); 663 if (!memslot->dirty_bitmap) 664 return -ENOMEM; 665 666 return 0; 667 } 668 669 /* 670 * Insert memslot and re-sort memslots based on their GFN, 671 * so binary search could be used to lookup GFN. 672 * Sorting algorithm takes advantage of having initially 673 * sorted array and known changed memslot position. 674 */ 675 static void update_memslots(struct kvm_memslots *slots, 676 struct kvm_memory_slot *new) 677 { 678 int id = new->id; 679 int i = slots->id_to_index[id]; 680 struct kvm_memory_slot *mslots = slots->memslots; 681 682 WARN_ON(mslots[i].id != id); 683 if (!new->npages) { 684 WARN_ON(!mslots[i].npages); 685 if (mslots[i].npages) 686 slots->used_slots--; 687 } else { 688 if (!mslots[i].npages) 689 slots->used_slots++; 690 } 691 692 while (i < KVM_MEM_SLOTS_NUM - 1 && 693 new->base_gfn <= mslots[i + 1].base_gfn) { 694 if (!mslots[i + 1].npages) 695 break; 696 mslots[i] = mslots[i + 1]; 697 slots->id_to_index[mslots[i].id] = i; 698 i++; 699 } 700 701 /* 702 * The ">=" is needed when creating a slot with base_gfn == 0, 703 * so that it moves before all those with base_gfn == npages == 0. 704 * 705 * On the other hand, if new->npages is zero, the above loop has 706 * already left i pointing to the beginning of the empty part of 707 * mslots, and the ">=" would move the hole backwards in this 708 * case---which is wrong. So skip the loop when deleting a slot. 709 */ 710 if (new->npages) { 711 while (i > 0 && 712 new->base_gfn >= mslots[i - 1].base_gfn) { 713 mslots[i] = mslots[i - 1]; 714 slots->id_to_index[mslots[i].id] = i; 715 i--; 716 } 717 } else 718 WARN_ON_ONCE(i != slots->used_slots); 719 720 mslots[i] = *new; 721 slots->id_to_index[mslots[i].id] = i; 722 } 723 724 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 725 { 726 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 727 728 #ifdef __KVM_HAVE_READONLY_MEM 729 valid_flags |= KVM_MEM_READONLY; 730 #endif 731 732 if (mem->flags & ~valid_flags) 733 return -EINVAL; 734 735 return 0; 736 } 737 738 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 739 int as_id, struct kvm_memslots *slots) 740 { 741 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); 742 743 /* 744 * Set the low bit in the generation, which disables SPTE caching 745 * until the end of synchronize_srcu_expedited. 746 */ 747 WARN_ON(old_memslots->generation & 1); 748 slots->generation = old_memslots->generation + 1; 749 750 rcu_assign_pointer(kvm->memslots[as_id], slots); 751 synchronize_srcu_expedited(&kvm->srcu); 752 753 /* 754 * Increment the new memslot generation a second time. This prevents 755 * vm exits that race with memslot updates from caching a memslot 756 * generation that will (potentially) be valid forever. 757 */ 758 slots->generation++; 759 760 kvm_arch_memslots_updated(kvm, slots); 761 762 return old_memslots; 763 } 764 765 /* 766 * Allocate some memory and give it an address in the guest physical address 767 * space. 768 * 769 * Discontiguous memory is allowed, mostly for framebuffers. 770 * 771 * Must be called holding kvm->slots_lock for write. 772 */ 773 int __kvm_set_memory_region(struct kvm *kvm, 774 const struct kvm_userspace_memory_region *mem) 775 { 776 int r; 777 gfn_t base_gfn; 778 unsigned long npages; 779 struct kvm_memory_slot *slot; 780 struct kvm_memory_slot old, new; 781 struct kvm_memslots *slots = NULL, *old_memslots; 782 int as_id, id; 783 enum kvm_mr_change change; 784 785 r = check_memory_region_flags(mem); 786 if (r) 787 goto out; 788 789 r = -EINVAL; 790 as_id = mem->slot >> 16; 791 id = (u16)mem->slot; 792 793 /* General sanity checks */ 794 if (mem->memory_size & (PAGE_SIZE - 1)) 795 goto out; 796 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 797 goto out; 798 /* We can read the guest memory with __xxx_user() later on. */ 799 if ((id < KVM_USER_MEM_SLOTS) && 800 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 801 !access_ok(VERIFY_WRITE, 802 (void __user *)(unsigned long)mem->userspace_addr, 803 mem->memory_size))) 804 goto out; 805 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 806 goto out; 807 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 808 goto out; 809 810 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); 811 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 812 npages = mem->memory_size >> PAGE_SHIFT; 813 814 if (npages > KVM_MEM_MAX_NR_PAGES) 815 goto out; 816 817 new = old = *slot; 818 819 new.id = id; 820 new.base_gfn = base_gfn; 821 new.npages = npages; 822 new.flags = mem->flags; 823 824 if (npages) { 825 if (!old.npages) 826 change = KVM_MR_CREATE; 827 else { /* Modify an existing slot. */ 828 if ((mem->userspace_addr != old.userspace_addr) || 829 (npages != old.npages) || 830 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 831 goto out; 832 833 if (base_gfn != old.base_gfn) 834 change = KVM_MR_MOVE; 835 else if (new.flags != old.flags) 836 change = KVM_MR_FLAGS_ONLY; 837 else { /* Nothing to change. */ 838 r = 0; 839 goto out; 840 } 841 } 842 } else { 843 if (!old.npages) 844 goto out; 845 846 change = KVM_MR_DELETE; 847 new.base_gfn = 0; 848 new.flags = 0; 849 } 850 851 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 852 /* Check for overlaps */ 853 r = -EEXIST; 854 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { 855 if ((slot->id >= KVM_USER_MEM_SLOTS) || 856 (slot->id == id)) 857 continue; 858 if (!((base_gfn + npages <= slot->base_gfn) || 859 (base_gfn >= slot->base_gfn + slot->npages))) 860 goto out; 861 } 862 } 863 864 /* Free page dirty bitmap if unneeded */ 865 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 866 new.dirty_bitmap = NULL; 867 868 r = -ENOMEM; 869 if (change == KVM_MR_CREATE) { 870 new.userspace_addr = mem->userspace_addr; 871 872 if (kvm_arch_create_memslot(kvm, &new, npages)) 873 goto out_free; 874 } 875 876 /* Allocate page dirty bitmap if needed */ 877 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 878 if (kvm_create_dirty_bitmap(&new) < 0) 879 goto out_free; 880 } 881 882 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 883 if (!slots) 884 goto out_free; 885 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); 886 887 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 888 slot = id_to_memslot(slots, id); 889 slot->flags |= KVM_MEMSLOT_INVALID; 890 891 old_memslots = install_new_memslots(kvm, as_id, slots); 892 893 /* slot was deleted or moved, clear iommu mapping */ 894 kvm_iommu_unmap_pages(kvm, &old); 895 /* From this point no new shadow pages pointing to a deleted, 896 * or moved, memslot will be created. 897 * 898 * validation of sp->gfn happens in: 899 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 900 * - kvm_is_visible_gfn (mmu_check_roots) 901 */ 902 kvm_arch_flush_shadow_memslot(kvm, slot); 903 904 /* 905 * We can re-use the old_memslots from above, the only difference 906 * from the currently installed memslots is the invalid flag. This 907 * will get overwritten by update_memslots anyway. 908 */ 909 slots = old_memslots; 910 } 911 912 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 913 if (r) 914 goto out_slots; 915 916 /* actual memory is freed via old in kvm_free_memslot below */ 917 if (change == KVM_MR_DELETE) { 918 new.dirty_bitmap = NULL; 919 memset(&new.arch, 0, sizeof(new.arch)); 920 } 921 922 update_memslots(slots, &new); 923 old_memslots = install_new_memslots(kvm, as_id, slots); 924 925 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); 926 927 kvm_free_memslot(kvm, &old, &new); 928 kvfree(old_memslots); 929 930 /* 931 * IOMMU mapping: New slots need to be mapped. Old slots need to be 932 * un-mapped and re-mapped if their base changes. Since base change 933 * unmapping is handled above with slot deletion, mapping alone is 934 * needed here. Anything else the iommu might care about for existing 935 * slots (size changes, userspace addr changes and read-only flag 936 * changes) is disallowed above, so any other attribute changes getting 937 * here can be skipped. 938 */ 939 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 940 r = kvm_iommu_map_pages(kvm, &new); 941 return r; 942 } 943 944 return 0; 945 946 out_slots: 947 kvfree(slots); 948 out_free: 949 kvm_free_memslot(kvm, &new, &old); 950 out: 951 return r; 952 } 953 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 954 955 int kvm_set_memory_region(struct kvm *kvm, 956 const struct kvm_userspace_memory_region *mem) 957 { 958 int r; 959 960 mutex_lock(&kvm->slots_lock); 961 r = __kvm_set_memory_region(kvm, mem); 962 mutex_unlock(&kvm->slots_lock); 963 return r; 964 } 965 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 966 967 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 968 struct kvm_userspace_memory_region *mem) 969 { 970 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 971 return -EINVAL; 972 973 return kvm_set_memory_region(kvm, mem); 974 } 975 976 int kvm_get_dirty_log(struct kvm *kvm, 977 struct kvm_dirty_log *log, int *is_dirty) 978 { 979 struct kvm_memslots *slots; 980 struct kvm_memory_slot *memslot; 981 int r, i, as_id, id; 982 unsigned long n; 983 unsigned long any = 0; 984 985 r = -EINVAL; 986 as_id = log->slot >> 16; 987 id = (u16)log->slot; 988 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 989 goto out; 990 991 slots = __kvm_memslots(kvm, as_id); 992 memslot = id_to_memslot(slots, id); 993 r = -ENOENT; 994 if (!memslot->dirty_bitmap) 995 goto out; 996 997 n = kvm_dirty_bitmap_bytes(memslot); 998 999 for (i = 0; !any && i < n/sizeof(long); ++i) 1000 any = memslot->dirty_bitmap[i]; 1001 1002 r = -EFAULT; 1003 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 1004 goto out; 1005 1006 if (any) 1007 *is_dirty = 1; 1008 1009 r = 0; 1010 out: 1011 return r; 1012 } 1013 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1014 1015 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1016 /** 1017 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages 1018 * are dirty write protect them for next write. 1019 * @kvm: pointer to kvm instance 1020 * @log: slot id and address to which we copy the log 1021 * @is_dirty: flag set if any page is dirty 1022 * 1023 * We need to keep it in mind that VCPU threads can write to the bitmap 1024 * concurrently. So, to avoid losing track of dirty pages we keep the 1025 * following order: 1026 * 1027 * 1. Take a snapshot of the bit and clear it if needed. 1028 * 2. Write protect the corresponding page. 1029 * 3. Copy the snapshot to the userspace. 1030 * 4. Upon return caller flushes TLB's if needed. 1031 * 1032 * Between 2 and 4, the guest may write to the page using the remaining TLB 1033 * entry. This is not a problem because the page is reported dirty using 1034 * the snapshot taken before and step 4 ensures that writes done after 1035 * exiting to userspace will be logged for the next call. 1036 * 1037 */ 1038 int kvm_get_dirty_log_protect(struct kvm *kvm, 1039 struct kvm_dirty_log *log, bool *is_dirty) 1040 { 1041 struct kvm_memslots *slots; 1042 struct kvm_memory_slot *memslot; 1043 int r, i, as_id, id; 1044 unsigned long n; 1045 unsigned long *dirty_bitmap; 1046 unsigned long *dirty_bitmap_buffer; 1047 1048 r = -EINVAL; 1049 as_id = log->slot >> 16; 1050 id = (u16)log->slot; 1051 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1052 goto out; 1053 1054 slots = __kvm_memslots(kvm, as_id); 1055 memslot = id_to_memslot(slots, id); 1056 1057 dirty_bitmap = memslot->dirty_bitmap; 1058 r = -ENOENT; 1059 if (!dirty_bitmap) 1060 goto out; 1061 1062 n = kvm_dirty_bitmap_bytes(memslot); 1063 1064 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long); 1065 memset(dirty_bitmap_buffer, 0, n); 1066 1067 spin_lock(&kvm->mmu_lock); 1068 *is_dirty = false; 1069 for (i = 0; i < n / sizeof(long); i++) { 1070 unsigned long mask; 1071 gfn_t offset; 1072 1073 if (!dirty_bitmap[i]) 1074 continue; 1075 1076 *is_dirty = true; 1077 1078 mask = xchg(&dirty_bitmap[i], 0); 1079 dirty_bitmap_buffer[i] = mask; 1080 1081 if (mask) { 1082 offset = i * BITS_PER_LONG; 1083 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1084 offset, mask); 1085 } 1086 } 1087 1088 spin_unlock(&kvm->mmu_lock); 1089 1090 r = -EFAULT; 1091 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 1092 goto out; 1093 1094 r = 0; 1095 out: 1096 return r; 1097 } 1098 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect); 1099 #endif 1100 1101 bool kvm_largepages_enabled(void) 1102 { 1103 return largepages_enabled; 1104 } 1105 1106 void kvm_disable_largepages(void) 1107 { 1108 largepages_enabled = false; 1109 } 1110 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1111 1112 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1113 { 1114 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1115 } 1116 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1117 1118 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 1119 { 1120 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); 1121 } 1122 1123 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1124 { 1125 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1126 1127 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1128 memslot->flags & KVM_MEMSLOT_INVALID) 1129 return 0; 1130 1131 return 1; 1132 } 1133 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1134 1135 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1136 { 1137 struct vm_area_struct *vma; 1138 unsigned long addr, size; 1139 1140 size = PAGE_SIZE; 1141 1142 addr = gfn_to_hva(kvm, gfn); 1143 if (kvm_is_error_hva(addr)) 1144 return PAGE_SIZE; 1145 1146 down_read(¤t->mm->mmap_sem); 1147 vma = find_vma(current->mm, addr); 1148 if (!vma) 1149 goto out; 1150 1151 size = vma_kernel_pagesize(vma); 1152 1153 out: 1154 up_read(¤t->mm->mmap_sem); 1155 1156 return size; 1157 } 1158 1159 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1160 { 1161 return slot->flags & KVM_MEM_READONLY; 1162 } 1163 1164 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1165 gfn_t *nr_pages, bool write) 1166 { 1167 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1168 return KVM_HVA_ERR_BAD; 1169 1170 if (memslot_is_readonly(slot) && write) 1171 return KVM_HVA_ERR_RO_BAD; 1172 1173 if (nr_pages) 1174 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1175 1176 return __gfn_to_hva_memslot(slot, gfn); 1177 } 1178 1179 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1180 gfn_t *nr_pages) 1181 { 1182 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1183 } 1184 1185 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1186 gfn_t gfn) 1187 { 1188 return gfn_to_hva_many(slot, gfn, NULL); 1189 } 1190 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1191 1192 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1193 { 1194 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1195 } 1196 EXPORT_SYMBOL_GPL(gfn_to_hva); 1197 1198 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 1199 { 1200 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 1201 } 1202 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 1203 1204 /* 1205 * If writable is set to false, the hva returned by this function is only 1206 * allowed to be read. 1207 */ 1208 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1209 gfn_t gfn, bool *writable) 1210 { 1211 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1212 1213 if (!kvm_is_error_hva(hva) && writable) 1214 *writable = !memslot_is_readonly(slot); 1215 1216 return hva; 1217 } 1218 1219 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1220 { 1221 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1222 1223 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1224 } 1225 1226 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 1227 { 1228 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1229 1230 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1231 } 1232 1233 static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm, 1234 unsigned long start, int write, struct page **page) 1235 { 1236 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1237 1238 if (write) 1239 flags |= FOLL_WRITE; 1240 1241 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL); 1242 } 1243 1244 static inline int check_user_page_hwpoison(unsigned long addr) 1245 { 1246 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1247 1248 rc = __get_user_pages(current, current->mm, addr, 1, 1249 flags, NULL, NULL, NULL); 1250 return rc == -EHWPOISON; 1251 } 1252 1253 /* 1254 * The atomic path to get the writable pfn which will be stored in @pfn, 1255 * true indicates success, otherwise false is returned. 1256 */ 1257 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1258 bool write_fault, bool *writable, pfn_t *pfn) 1259 { 1260 struct page *page[1]; 1261 int npages; 1262 1263 if (!(async || atomic)) 1264 return false; 1265 1266 /* 1267 * Fast pin a writable pfn only if it is a write fault request 1268 * or the caller allows to map a writable pfn for a read fault 1269 * request. 1270 */ 1271 if (!(write_fault || writable)) 1272 return false; 1273 1274 npages = __get_user_pages_fast(addr, 1, 1, page); 1275 if (npages == 1) { 1276 *pfn = page_to_pfn(page[0]); 1277 1278 if (writable) 1279 *writable = true; 1280 return true; 1281 } 1282 1283 return false; 1284 } 1285 1286 /* 1287 * The slow path to get the pfn of the specified host virtual address, 1288 * 1 indicates success, -errno is returned if error is detected. 1289 */ 1290 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1291 bool *writable, pfn_t *pfn) 1292 { 1293 struct page *page[1]; 1294 int npages = 0; 1295 1296 might_sleep(); 1297 1298 if (writable) 1299 *writable = write_fault; 1300 1301 if (async) { 1302 down_read(¤t->mm->mmap_sem); 1303 npages = get_user_page_nowait(current, current->mm, 1304 addr, write_fault, page); 1305 up_read(¤t->mm->mmap_sem); 1306 } else 1307 npages = __get_user_pages_unlocked(current, current->mm, addr, 1, 1308 write_fault, 0, page, 1309 FOLL_TOUCH|FOLL_HWPOISON); 1310 if (npages != 1) 1311 return npages; 1312 1313 /* map read fault as writable if possible */ 1314 if (unlikely(!write_fault) && writable) { 1315 struct page *wpage[1]; 1316 1317 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1318 if (npages == 1) { 1319 *writable = true; 1320 put_page(page[0]); 1321 page[0] = wpage[0]; 1322 } 1323 1324 npages = 1; 1325 } 1326 *pfn = page_to_pfn(page[0]); 1327 return npages; 1328 } 1329 1330 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1331 { 1332 if (unlikely(!(vma->vm_flags & VM_READ))) 1333 return false; 1334 1335 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1336 return false; 1337 1338 return true; 1339 } 1340 1341 /* 1342 * Pin guest page in memory and return its pfn. 1343 * @addr: host virtual address which maps memory to the guest 1344 * @atomic: whether this function can sleep 1345 * @async: whether this function need to wait IO complete if the 1346 * host page is not in the memory 1347 * @write_fault: whether we should get a writable host page 1348 * @writable: whether it allows to map a writable host page for !@write_fault 1349 * 1350 * The function will map a writable host page for these two cases: 1351 * 1): @write_fault = true 1352 * 2): @write_fault = false && @writable, @writable will tell the caller 1353 * whether the mapping is writable. 1354 */ 1355 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1356 bool write_fault, bool *writable) 1357 { 1358 struct vm_area_struct *vma; 1359 pfn_t pfn = 0; 1360 int npages; 1361 1362 /* we can do it either atomically or asynchronously, not both */ 1363 BUG_ON(atomic && async); 1364 1365 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1366 return pfn; 1367 1368 if (atomic) 1369 return KVM_PFN_ERR_FAULT; 1370 1371 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1372 if (npages == 1) 1373 return pfn; 1374 1375 down_read(¤t->mm->mmap_sem); 1376 if (npages == -EHWPOISON || 1377 (!async && check_user_page_hwpoison(addr))) { 1378 pfn = KVM_PFN_ERR_HWPOISON; 1379 goto exit; 1380 } 1381 1382 vma = find_vma_intersection(current->mm, addr, addr + 1); 1383 1384 if (vma == NULL) 1385 pfn = KVM_PFN_ERR_FAULT; 1386 else if ((vma->vm_flags & VM_PFNMAP)) { 1387 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1388 vma->vm_pgoff; 1389 BUG_ON(!kvm_is_reserved_pfn(pfn)); 1390 } else { 1391 if (async && vma_is_valid(vma, write_fault)) 1392 *async = true; 1393 pfn = KVM_PFN_ERR_FAULT; 1394 } 1395 exit: 1396 up_read(¤t->mm->mmap_sem); 1397 return pfn; 1398 } 1399 1400 pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic, 1401 bool *async, bool write_fault, bool *writable) 1402 { 1403 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1404 1405 if (addr == KVM_HVA_ERR_RO_BAD) 1406 return KVM_PFN_ERR_RO_FAULT; 1407 1408 if (kvm_is_error_hva(addr)) 1409 return KVM_PFN_NOSLOT; 1410 1411 /* Do not map writable pfn in the readonly memslot. */ 1412 if (writable && memslot_is_readonly(slot)) { 1413 *writable = false; 1414 writable = NULL; 1415 } 1416 1417 return hva_to_pfn(addr, atomic, async, write_fault, 1418 writable); 1419 } 1420 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 1421 1422 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1423 bool *writable) 1424 { 1425 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, 1426 write_fault, writable); 1427 } 1428 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1429 1430 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1431 { 1432 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1433 } 1434 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 1435 1436 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1437 { 1438 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1439 } 1440 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1441 1442 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1443 { 1444 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); 1445 } 1446 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1447 1448 pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 1449 { 1450 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1451 } 1452 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 1453 1454 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1455 { 1456 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 1457 } 1458 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1459 1460 pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 1461 { 1462 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1463 } 1464 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 1465 1466 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1467 struct page **pages, int nr_pages) 1468 { 1469 unsigned long addr; 1470 gfn_t entry; 1471 1472 addr = gfn_to_hva_many(slot, gfn, &entry); 1473 if (kvm_is_error_hva(addr)) 1474 return -1; 1475 1476 if (entry < nr_pages) 1477 return 0; 1478 1479 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1480 } 1481 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1482 1483 static struct page *kvm_pfn_to_page(pfn_t pfn) 1484 { 1485 if (is_error_noslot_pfn(pfn)) 1486 return KVM_ERR_PTR_BAD_PAGE; 1487 1488 if (kvm_is_reserved_pfn(pfn)) { 1489 WARN_ON(1); 1490 return KVM_ERR_PTR_BAD_PAGE; 1491 } 1492 1493 return pfn_to_page(pfn); 1494 } 1495 1496 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1497 { 1498 pfn_t pfn; 1499 1500 pfn = gfn_to_pfn(kvm, gfn); 1501 1502 return kvm_pfn_to_page(pfn); 1503 } 1504 EXPORT_SYMBOL_GPL(gfn_to_page); 1505 1506 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) 1507 { 1508 pfn_t pfn; 1509 1510 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); 1511 1512 return kvm_pfn_to_page(pfn); 1513 } 1514 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); 1515 1516 void kvm_release_page_clean(struct page *page) 1517 { 1518 WARN_ON(is_error_page(page)); 1519 1520 kvm_release_pfn_clean(page_to_pfn(page)); 1521 } 1522 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1523 1524 void kvm_release_pfn_clean(pfn_t pfn) 1525 { 1526 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) 1527 put_page(pfn_to_page(pfn)); 1528 } 1529 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1530 1531 void kvm_release_page_dirty(struct page *page) 1532 { 1533 WARN_ON(is_error_page(page)); 1534 1535 kvm_release_pfn_dirty(page_to_pfn(page)); 1536 } 1537 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1538 1539 static void kvm_release_pfn_dirty(pfn_t pfn) 1540 { 1541 kvm_set_pfn_dirty(pfn); 1542 kvm_release_pfn_clean(pfn); 1543 } 1544 1545 void kvm_set_pfn_dirty(pfn_t pfn) 1546 { 1547 if (!kvm_is_reserved_pfn(pfn)) { 1548 struct page *page = pfn_to_page(pfn); 1549 1550 if (!PageReserved(page)) 1551 SetPageDirty(page); 1552 } 1553 } 1554 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1555 1556 void kvm_set_pfn_accessed(pfn_t pfn) 1557 { 1558 if (!kvm_is_reserved_pfn(pfn)) 1559 mark_page_accessed(pfn_to_page(pfn)); 1560 } 1561 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1562 1563 void kvm_get_pfn(pfn_t pfn) 1564 { 1565 if (!kvm_is_reserved_pfn(pfn)) 1566 get_page(pfn_to_page(pfn)); 1567 } 1568 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1569 1570 static int next_segment(unsigned long len, int offset) 1571 { 1572 if (len > PAGE_SIZE - offset) 1573 return PAGE_SIZE - offset; 1574 else 1575 return len; 1576 } 1577 1578 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 1579 void *data, int offset, int len) 1580 { 1581 int r; 1582 unsigned long addr; 1583 1584 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1585 if (kvm_is_error_hva(addr)) 1586 return -EFAULT; 1587 r = __copy_from_user(data, (void __user *)addr + offset, len); 1588 if (r) 1589 return -EFAULT; 1590 return 0; 1591 } 1592 1593 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1594 int len) 1595 { 1596 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1597 1598 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1599 } 1600 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1601 1602 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 1603 int offset, int len) 1604 { 1605 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1606 1607 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1608 } 1609 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 1610 1611 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1612 { 1613 gfn_t gfn = gpa >> PAGE_SHIFT; 1614 int seg; 1615 int offset = offset_in_page(gpa); 1616 int ret; 1617 1618 while ((seg = next_segment(len, offset)) != 0) { 1619 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1620 if (ret < 0) 1621 return ret; 1622 offset = 0; 1623 len -= seg; 1624 data += seg; 1625 ++gfn; 1626 } 1627 return 0; 1628 } 1629 EXPORT_SYMBOL_GPL(kvm_read_guest); 1630 1631 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 1632 { 1633 gfn_t gfn = gpa >> PAGE_SHIFT; 1634 int seg; 1635 int offset = offset_in_page(gpa); 1636 int ret; 1637 1638 while ((seg = next_segment(len, offset)) != 0) { 1639 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 1640 if (ret < 0) 1641 return ret; 1642 offset = 0; 1643 len -= seg; 1644 data += seg; 1645 ++gfn; 1646 } 1647 return 0; 1648 } 1649 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 1650 1651 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1652 void *data, int offset, unsigned long len) 1653 { 1654 int r; 1655 unsigned long addr; 1656 1657 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1658 if (kvm_is_error_hva(addr)) 1659 return -EFAULT; 1660 pagefault_disable(); 1661 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1662 pagefault_enable(); 1663 if (r) 1664 return -EFAULT; 1665 return 0; 1666 } 1667 1668 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1669 unsigned long len) 1670 { 1671 gfn_t gfn = gpa >> PAGE_SHIFT; 1672 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1673 int offset = offset_in_page(gpa); 1674 1675 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1676 } 1677 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic); 1678 1679 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 1680 void *data, unsigned long len) 1681 { 1682 gfn_t gfn = gpa >> PAGE_SHIFT; 1683 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1684 int offset = offset_in_page(gpa); 1685 1686 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1687 } 1688 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 1689 1690 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn, 1691 const void *data, int offset, int len) 1692 { 1693 int r; 1694 unsigned long addr; 1695 1696 addr = gfn_to_hva_memslot(memslot, gfn); 1697 if (kvm_is_error_hva(addr)) 1698 return -EFAULT; 1699 r = __copy_to_user((void __user *)addr + offset, data, len); 1700 if (r) 1701 return -EFAULT; 1702 mark_page_dirty_in_slot(memslot, gfn); 1703 return 0; 1704 } 1705 1706 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 1707 const void *data, int offset, int len) 1708 { 1709 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1710 1711 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1712 } 1713 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1714 1715 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 1716 const void *data, int offset, int len) 1717 { 1718 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1719 1720 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1721 } 1722 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 1723 1724 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1725 unsigned long len) 1726 { 1727 gfn_t gfn = gpa >> PAGE_SHIFT; 1728 int seg; 1729 int offset = offset_in_page(gpa); 1730 int ret; 1731 1732 while ((seg = next_segment(len, offset)) != 0) { 1733 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1734 if (ret < 0) 1735 return ret; 1736 offset = 0; 1737 len -= seg; 1738 data += seg; 1739 ++gfn; 1740 } 1741 return 0; 1742 } 1743 EXPORT_SYMBOL_GPL(kvm_write_guest); 1744 1745 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1746 unsigned long len) 1747 { 1748 gfn_t gfn = gpa >> PAGE_SHIFT; 1749 int seg; 1750 int offset = offset_in_page(gpa); 1751 int ret; 1752 1753 while ((seg = next_segment(len, offset)) != 0) { 1754 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 1755 if (ret < 0) 1756 return ret; 1757 offset = 0; 1758 len -= seg; 1759 data += seg; 1760 ++gfn; 1761 } 1762 return 0; 1763 } 1764 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 1765 1766 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1767 gpa_t gpa, unsigned long len) 1768 { 1769 struct kvm_memslots *slots = kvm_memslots(kvm); 1770 int offset = offset_in_page(gpa); 1771 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1772 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1773 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1774 gfn_t nr_pages_avail; 1775 1776 ghc->gpa = gpa; 1777 ghc->generation = slots->generation; 1778 ghc->len = len; 1779 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1780 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL); 1781 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) { 1782 ghc->hva += offset; 1783 } else { 1784 /* 1785 * If the requested region crosses two memslots, we still 1786 * verify that the entire region is valid here. 1787 */ 1788 while (start_gfn <= end_gfn) { 1789 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1790 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1791 &nr_pages_avail); 1792 if (kvm_is_error_hva(ghc->hva)) 1793 return -EFAULT; 1794 start_gfn += nr_pages_avail; 1795 } 1796 /* Use the slow path for cross page reads and writes. */ 1797 ghc->memslot = NULL; 1798 } 1799 return 0; 1800 } 1801 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1802 1803 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1804 void *data, unsigned long len) 1805 { 1806 struct kvm_memslots *slots = kvm_memslots(kvm); 1807 int r; 1808 1809 BUG_ON(len > ghc->len); 1810 1811 if (slots->generation != ghc->generation) 1812 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1813 1814 if (unlikely(!ghc->memslot)) 1815 return kvm_write_guest(kvm, ghc->gpa, data, len); 1816 1817 if (kvm_is_error_hva(ghc->hva)) 1818 return -EFAULT; 1819 1820 r = __copy_to_user((void __user *)ghc->hva, data, len); 1821 if (r) 1822 return -EFAULT; 1823 mark_page_dirty_in_slot(ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1824 1825 return 0; 1826 } 1827 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1828 1829 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1830 void *data, unsigned long len) 1831 { 1832 struct kvm_memslots *slots = kvm_memslots(kvm); 1833 int r; 1834 1835 BUG_ON(len > ghc->len); 1836 1837 if (slots->generation != ghc->generation) 1838 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1839 1840 if (unlikely(!ghc->memslot)) 1841 return kvm_read_guest(kvm, ghc->gpa, data, len); 1842 1843 if (kvm_is_error_hva(ghc->hva)) 1844 return -EFAULT; 1845 1846 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1847 if (r) 1848 return -EFAULT; 1849 1850 return 0; 1851 } 1852 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1853 1854 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1855 { 1856 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 1857 1858 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 1859 } 1860 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1861 1862 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1863 { 1864 gfn_t gfn = gpa >> PAGE_SHIFT; 1865 int seg; 1866 int offset = offset_in_page(gpa); 1867 int ret; 1868 1869 while ((seg = next_segment(len, offset)) != 0) { 1870 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1871 if (ret < 0) 1872 return ret; 1873 offset = 0; 1874 len -= seg; 1875 ++gfn; 1876 } 1877 return 0; 1878 } 1879 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1880 1881 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, 1882 gfn_t gfn) 1883 { 1884 if (memslot && memslot->dirty_bitmap) { 1885 unsigned long rel_gfn = gfn - memslot->base_gfn; 1886 1887 set_bit_le(rel_gfn, memslot->dirty_bitmap); 1888 } 1889 } 1890 1891 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 1892 { 1893 struct kvm_memory_slot *memslot; 1894 1895 memslot = gfn_to_memslot(kvm, gfn); 1896 mark_page_dirty_in_slot(memslot, gfn); 1897 } 1898 EXPORT_SYMBOL_GPL(mark_page_dirty); 1899 1900 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 1901 { 1902 struct kvm_memory_slot *memslot; 1903 1904 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1905 mark_page_dirty_in_slot(memslot, gfn); 1906 } 1907 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 1908 1909 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 1910 { 1911 if (kvm_arch_vcpu_runnable(vcpu)) { 1912 kvm_make_request(KVM_REQ_UNHALT, vcpu); 1913 return -EINTR; 1914 } 1915 if (kvm_cpu_has_pending_timer(vcpu)) 1916 return -EINTR; 1917 if (signal_pending(current)) 1918 return -EINTR; 1919 1920 return 0; 1921 } 1922 1923 /* 1924 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 1925 */ 1926 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 1927 { 1928 ktime_t start, cur; 1929 DEFINE_WAIT(wait); 1930 bool waited = false; 1931 1932 start = cur = ktime_get(); 1933 if (halt_poll_ns) { 1934 ktime_t stop = ktime_add_ns(ktime_get(), halt_poll_ns); 1935 1936 do { 1937 /* 1938 * This sets KVM_REQ_UNHALT if an interrupt 1939 * arrives. 1940 */ 1941 if (kvm_vcpu_check_block(vcpu) < 0) { 1942 ++vcpu->stat.halt_successful_poll; 1943 goto out; 1944 } 1945 cur = ktime_get(); 1946 } while (single_task_running() && ktime_before(cur, stop)); 1947 } 1948 1949 for (;;) { 1950 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 1951 1952 if (kvm_vcpu_check_block(vcpu) < 0) 1953 break; 1954 1955 waited = true; 1956 schedule(); 1957 } 1958 1959 finish_wait(&vcpu->wq, &wait); 1960 cur = ktime_get(); 1961 1962 out: 1963 trace_kvm_vcpu_wakeup(ktime_to_ns(cur) - ktime_to_ns(start), waited); 1964 } 1965 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 1966 1967 #ifndef CONFIG_S390 1968 /* 1969 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 1970 */ 1971 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 1972 { 1973 int me; 1974 int cpu = vcpu->cpu; 1975 wait_queue_head_t *wqp; 1976 1977 wqp = kvm_arch_vcpu_wq(vcpu); 1978 if (waitqueue_active(wqp)) { 1979 wake_up_interruptible(wqp); 1980 ++vcpu->stat.halt_wakeup; 1981 } 1982 1983 me = get_cpu(); 1984 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 1985 if (kvm_arch_vcpu_should_kick(vcpu)) 1986 smp_send_reschedule(cpu); 1987 put_cpu(); 1988 } 1989 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 1990 #endif /* !CONFIG_S390 */ 1991 1992 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 1993 { 1994 struct pid *pid; 1995 struct task_struct *task = NULL; 1996 int ret = 0; 1997 1998 rcu_read_lock(); 1999 pid = rcu_dereference(target->pid); 2000 if (pid) 2001 task = get_pid_task(pid, PIDTYPE_PID); 2002 rcu_read_unlock(); 2003 if (!task) 2004 return ret; 2005 ret = yield_to(task, 1); 2006 put_task_struct(task); 2007 2008 return ret; 2009 } 2010 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 2011 2012 /* 2013 * Helper that checks whether a VCPU is eligible for directed yield. 2014 * Most eligible candidate to yield is decided by following heuristics: 2015 * 2016 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 2017 * (preempted lock holder), indicated by @in_spin_loop. 2018 * Set at the beiginning and cleared at the end of interception/PLE handler. 2019 * 2020 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 2021 * chance last time (mostly it has become eligible now since we have probably 2022 * yielded to lockholder in last iteration. This is done by toggling 2023 * @dy_eligible each time a VCPU checked for eligibility.) 2024 * 2025 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 2026 * to preempted lock-holder could result in wrong VCPU selection and CPU 2027 * burning. Giving priority for a potential lock-holder increases lock 2028 * progress. 2029 * 2030 * Since algorithm is based on heuristics, accessing another VCPU data without 2031 * locking does not harm. It may result in trying to yield to same VCPU, fail 2032 * and continue with next VCPU and so on. 2033 */ 2034 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 2035 { 2036 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2037 bool eligible; 2038 2039 eligible = !vcpu->spin_loop.in_spin_loop || 2040 vcpu->spin_loop.dy_eligible; 2041 2042 if (vcpu->spin_loop.in_spin_loop) 2043 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 2044 2045 return eligible; 2046 #else 2047 return true; 2048 #endif 2049 } 2050 2051 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 2052 { 2053 struct kvm *kvm = me->kvm; 2054 struct kvm_vcpu *vcpu; 2055 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 2056 int yielded = 0; 2057 int try = 3; 2058 int pass; 2059 int i; 2060 2061 kvm_vcpu_set_in_spin_loop(me, true); 2062 /* 2063 * We boost the priority of a VCPU that is runnable but not 2064 * currently running, because it got preempted by something 2065 * else and called schedule in __vcpu_run. Hopefully that 2066 * VCPU is holding the lock that we need and will release it. 2067 * We approximate round-robin by starting at the last boosted VCPU. 2068 */ 2069 for (pass = 0; pass < 2 && !yielded && try; pass++) { 2070 kvm_for_each_vcpu(i, vcpu, kvm) { 2071 if (!pass && i <= last_boosted_vcpu) { 2072 i = last_boosted_vcpu; 2073 continue; 2074 } else if (pass && i > last_boosted_vcpu) 2075 break; 2076 if (!ACCESS_ONCE(vcpu->preempted)) 2077 continue; 2078 if (vcpu == me) 2079 continue; 2080 if (waitqueue_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 2081 continue; 2082 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 2083 continue; 2084 2085 yielded = kvm_vcpu_yield_to(vcpu); 2086 if (yielded > 0) { 2087 kvm->last_boosted_vcpu = i; 2088 break; 2089 } else if (yielded < 0) { 2090 try--; 2091 if (!try) 2092 break; 2093 } 2094 } 2095 } 2096 kvm_vcpu_set_in_spin_loop(me, false); 2097 2098 /* Ensure vcpu is not eligible during next spinloop */ 2099 kvm_vcpu_set_dy_eligible(me, false); 2100 } 2101 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 2102 2103 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2104 { 2105 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 2106 struct page *page; 2107 2108 if (vmf->pgoff == 0) 2109 page = virt_to_page(vcpu->run); 2110 #ifdef CONFIG_X86 2111 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 2112 page = virt_to_page(vcpu->arch.pio_data); 2113 #endif 2114 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2115 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 2116 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 2117 #endif 2118 else 2119 return kvm_arch_vcpu_fault(vcpu, vmf); 2120 get_page(page); 2121 vmf->page = page; 2122 return 0; 2123 } 2124 2125 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 2126 .fault = kvm_vcpu_fault, 2127 }; 2128 2129 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 2130 { 2131 vma->vm_ops = &kvm_vcpu_vm_ops; 2132 return 0; 2133 } 2134 2135 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 2136 { 2137 struct kvm_vcpu *vcpu = filp->private_data; 2138 2139 kvm_put_kvm(vcpu->kvm); 2140 return 0; 2141 } 2142 2143 static struct file_operations kvm_vcpu_fops = { 2144 .release = kvm_vcpu_release, 2145 .unlocked_ioctl = kvm_vcpu_ioctl, 2146 #ifdef CONFIG_KVM_COMPAT 2147 .compat_ioctl = kvm_vcpu_compat_ioctl, 2148 #endif 2149 .mmap = kvm_vcpu_mmap, 2150 .llseek = noop_llseek, 2151 }; 2152 2153 /* 2154 * Allocates an inode for the vcpu. 2155 */ 2156 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 2157 { 2158 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 2159 } 2160 2161 /* 2162 * Creates some virtual cpus. Good luck creating more than one. 2163 */ 2164 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 2165 { 2166 int r; 2167 struct kvm_vcpu *vcpu, *v; 2168 2169 if (id >= KVM_MAX_VCPUS) 2170 return -EINVAL; 2171 2172 vcpu = kvm_arch_vcpu_create(kvm, id); 2173 if (IS_ERR(vcpu)) 2174 return PTR_ERR(vcpu); 2175 2176 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 2177 2178 r = kvm_arch_vcpu_setup(vcpu); 2179 if (r) 2180 goto vcpu_destroy; 2181 2182 mutex_lock(&kvm->lock); 2183 if (!kvm_vcpu_compatible(vcpu)) { 2184 r = -EINVAL; 2185 goto unlock_vcpu_destroy; 2186 } 2187 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 2188 r = -EINVAL; 2189 goto unlock_vcpu_destroy; 2190 } 2191 2192 kvm_for_each_vcpu(r, v, kvm) 2193 if (v->vcpu_id == id) { 2194 r = -EEXIST; 2195 goto unlock_vcpu_destroy; 2196 } 2197 2198 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 2199 2200 /* Now it's all set up, let userspace reach it */ 2201 kvm_get_kvm(kvm); 2202 r = create_vcpu_fd(vcpu); 2203 if (r < 0) { 2204 kvm_put_kvm(kvm); 2205 goto unlock_vcpu_destroy; 2206 } 2207 2208 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 2209 2210 /* 2211 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus 2212 * before kvm->online_vcpu's incremented value. 2213 */ 2214 smp_wmb(); 2215 atomic_inc(&kvm->online_vcpus); 2216 2217 mutex_unlock(&kvm->lock); 2218 kvm_arch_vcpu_postcreate(vcpu); 2219 return r; 2220 2221 unlock_vcpu_destroy: 2222 mutex_unlock(&kvm->lock); 2223 vcpu_destroy: 2224 kvm_arch_vcpu_destroy(vcpu); 2225 return r; 2226 } 2227 2228 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 2229 { 2230 if (sigset) { 2231 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2232 vcpu->sigset_active = 1; 2233 vcpu->sigset = *sigset; 2234 } else 2235 vcpu->sigset_active = 0; 2236 return 0; 2237 } 2238 2239 static long kvm_vcpu_ioctl(struct file *filp, 2240 unsigned int ioctl, unsigned long arg) 2241 { 2242 struct kvm_vcpu *vcpu = filp->private_data; 2243 void __user *argp = (void __user *)arg; 2244 int r; 2245 struct kvm_fpu *fpu = NULL; 2246 struct kvm_sregs *kvm_sregs = NULL; 2247 2248 if (vcpu->kvm->mm != current->mm) 2249 return -EIO; 2250 2251 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 2252 return -EINVAL; 2253 2254 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS) 2255 /* 2256 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 2257 * so vcpu_load() would break it. 2258 */ 2259 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT) 2260 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2261 #endif 2262 2263 2264 r = vcpu_load(vcpu); 2265 if (r) 2266 return r; 2267 switch (ioctl) { 2268 case KVM_RUN: 2269 r = -EINVAL; 2270 if (arg) 2271 goto out; 2272 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 2273 /* The thread running this VCPU changed. */ 2274 struct pid *oldpid = vcpu->pid; 2275 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 2276 2277 rcu_assign_pointer(vcpu->pid, newpid); 2278 if (oldpid) 2279 synchronize_rcu(); 2280 put_pid(oldpid); 2281 } 2282 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 2283 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 2284 break; 2285 case KVM_GET_REGS: { 2286 struct kvm_regs *kvm_regs; 2287 2288 r = -ENOMEM; 2289 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2290 if (!kvm_regs) 2291 goto out; 2292 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2293 if (r) 2294 goto out_free1; 2295 r = -EFAULT; 2296 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2297 goto out_free1; 2298 r = 0; 2299 out_free1: 2300 kfree(kvm_regs); 2301 break; 2302 } 2303 case KVM_SET_REGS: { 2304 struct kvm_regs *kvm_regs; 2305 2306 r = -ENOMEM; 2307 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2308 if (IS_ERR(kvm_regs)) { 2309 r = PTR_ERR(kvm_regs); 2310 goto out; 2311 } 2312 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2313 kfree(kvm_regs); 2314 break; 2315 } 2316 case KVM_GET_SREGS: { 2317 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2318 r = -ENOMEM; 2319 if (!kvm_sregs) 2320 goto out; 2321 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2322 if (r) 2323 goto out; 2324 r = -EFAULT; 2325 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2326 goto out; 2327 r = 0; 2328 break; 2329 } 2330 case KVM_SET_SREGS: { 2331 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2332 if (IS_ERR(kvm_sregs)) { 2333 r = PTR_ERR(kvm_sregs); 2334 kvm_sregs = NULL; 2335 goto out; 2336 } 2337 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2338 break; 2339 } 2340 case KVM_GET_MP_STATE: { 2341 struct kvm_mp_state mp_state; 2342 2343 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2344 if (r) 2345 goto out; 2346 r = -EFAULT; 2347 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 2348 goto out; 2349 r = 0; 2350 break; 2351 } 2352 case KVM_SET_MP_STATE: { 2353 struct kvm_mp_state mp_state; 2354 2355 r = -EFAULT; 2356 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 2357 goto out; 2358 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2359 break; 2360 } 2361 case KVM_TRANSLATE: { 2362 struct kvm_translation tr; 2363 2364 r = -EFAULT; 2365 if (copy_from_user(&tr, argp, sizeof(tr))) 2366 goto out; 2367 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2368 if (r) 2369 goto out; 2370 r = -EFAULT; 2371 if (copy_to_user(argp, &tr, sizeof(tr))) 2372 goto out; 2373 r = 0; 2374 break; 2375 } 2376 case KVM_SET_GUEST_DEBUG: { 2377 struct kvm_guest_debug dbg; 2378 2379 r = -EFAULT; 2380 if (copy_from_user(&dbg, argp, sizeof(dbg))) 2381 goto out; 2382 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2383 break; 2384 } 2385 case KVM_SET_SIGNAL_MASK: { 2386 struct kvm_signal_mask __user *sigmask_arg = argp; 2387 struct kvm_signal_mask kvm_sigmask; 2388 sigset_t sigset, *p; 2389 2390 p = NULL; 2391 if (argp) { 2392 r = -EFAULT; 2393 if (copy_from_user(&kvm_sigmask, argp, 2394 sizeof(kvm_sigmask))) 2395 goto out; 2396 r = -EINVAL; 2397 if (kvm_sigmask.len != sizeof(sigset)) 2398 goto out; 2399 r = -EFAULT; 2400 if (copy_from_user(&sigset, sigmask_arg->sigset, 2401 sizeof(sigset))) 2402 goto out; 2403 p = &sigset; 2404 } 2405 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2406 break; 2407 } 2408 case KVM_GET_FPU: { 2409 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2410 r = -ENOMEM; 2411 if (!fpu) 2412 goto out; 2413 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2414 if (r) 2415 goto out; 2416 r = -EFAULT; 2417 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2418 goto out; 2419 r = 0; 2420 break; 2421 } 2422 case KVM_SET_FPU: { 2423 fpu = memdup_user(argp, sizeof(*fpu)); 2424 if (IS_ERR(fpu)) { 2425 r = PTR_ERR(fpu); 2426 fpu = NULL; 2427 goto out; 2428 } 2429 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2430 break; 2431 } 2432 default: 2433 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2434 } 2435 out: 2436 vcpu_put(vcpu); 2437 kfree(fpu); 2438 kfree(kvm_sregs); 2439 return r; 2440 } 2441 2442 #ifdef CONFIG_KVM_COMPAT 2443 static long kvm_vcpu_compat_ioctl(struct file *filp, 2444 unsigned int ioctl, unsigned long arg) 2445 { 2446 struct kvm_vcpu *vcpu = filp->private_data; 2447 void __user *argp = compat_ptr(arg); 2448 int r; 2449 2450 if (vcpu->kvm->mm != current->mm) 2451 return -EIO; 2452 2453 switch (ioctl) { 2454 case KVM_SET_SIGNAL_MASK: { 2455 struct kvm_signal_mask __user *sigmask_arg = argp; 2456 struct kvm_signal_mask kvm_sigmask; 2457 compat_sigset_t csigset; 2458 sigset_t sigset; 2459 2460 if (argp) { 2461 r = -EFAULT; 2462 if (copy_from_user(&kvm_sigmask, argp, 2463 sizeof(kvm_sigmask))) 2464 goto out; 2465 r = -EINVAL; 2466 if (kvm_sigmask.len != sizeof(csigset)) 2467 goto out; 2468 r = -EFAULT; 2469 if (copy_from_user(&csigset, sigmask_arg->sigset, 2470 sizeof(csigset))) 2471 goto out; 2472 sigset_from_compat(&sigset, &csigset); 2473 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2474 } else 2475 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2476 break; 2477 } 2478 default: 2479 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2480 } 2481 2482 out: 2483 return r; 2484 } 2485 #endif 2486 2487 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2488 int (*accessor)(struct kvm_device *dev, 2489 struct kvm_device_attr *attr), 2490 unsigned long arg) 2491 { 2492 struct kvm_device_attr attr; 2493 2494 if (!accessor) 2495 return -EPERM; 2496 2497 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2498 return -EFAULT; 2499 2500 return accessor(dev, &attr); 2501 } 2502 2503 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2504 unsigned long arg) 2505 { 2506 struct kvm_device *dev = filp->private_data; 2507 2508 switch (ioctl) { 2509 case KVM_SET_DEVICE_ATTR: 2510 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2511 case KVM_GET_DEVICE_ATTR: 2512 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2513 case KVM_HAS_DEVICE_ATTR: 2514 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2515 default: 2516 if (dev->ops->ioctl) 2517 return dev->ops->ioctl(dev, ioctl, arg); 2518 2519 return -ENOTTY; 2520 } 2521 } 2522 2523 static int kvm_device_release(struct inode *inode, struct file *filp) 2524 { 2525 struct kvm_device *dev = filp->private_data; 2526 struct kvm *kvm = dev->kvm; 2527 2528 kvm_put_kvm(kvm); 2529 return 0; 2530 } 2531 2532 static const struct file_operations kvm_device_fops = { 2533 .unlocked_ioctl = kvm_device_ioctl, 2534 #ifdef CONFIG_KVM_COMPAT 2535 .compat_ioctl = kvm_device_ioctl, 2536 #endif 2537 .release = kvm_device_release, 2538 }; 2539 2540 struct kvm_device *kvm_device_from_filp(struct file *filp) 2541 { 2542 if (filp->f_op != &kvm_device_fops) 2543 return NULL; 2544 2545 return filp->private_data; 2546 } 2547 2548 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 2549 #ifdef CONFIG_KVM_MPIC 2550 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 2551 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 2552 #endif 2553 2554 #ifdef CONFIG_KVM_XICS 2555 [KVM_DEV_TYPE_XICS] = &kvm_xics_ops, 2556 #endif 2557 }; 2558 2559 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type) 2560 { 2561 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 2562 return -ENOSPC; 2563 2564 if (kvm_device_ops_table[type] != NULL) 2565 return -EEXIST; 2566 2567 kvm_device_ops_table[type] = ops; 2568 return 0; 2569 } 2570 2571 void kvm_unregister_device_ops(u32 type) 2572 { 2573 if (kvm_device_ops_table[type] != NULL) 2574 kvm_device_ops_table[type] = NULL; 2575 } 2576 2577 static int kvm_ioctl_create_device(struct kvm *kvm, 2578 struct kvm_create_device *cd) 2579 { 2580 struct kvm_device_ops *ops = NULL; 2581 struct kvm_device *dev; 2582 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2583 int ret; 2584 2585 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 2586 return -ENODEV; 2587 2588 ops = kvm_device_ops_table[cd->type]; 2589 if (ops == NULL) 2590 return -ENODEV; 2591 2592 if (test) 2593 return 0; 2594 2595 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2596 if (!dev) 2597 return -ENOMEM; 2598 2599 dev->ops = ops; 2600 dev->kvm = kvm; 2601 2602 ret = ops->create(dev, cd->type); 2603 if (ret < 0) { 2604 kfree(dev); 2605 return ret; 2606 } 2607 2608 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2609 if (ret < 0) { 2610 ops->destroy(dev); 2611 return ret; 2612 } 2613 2614 list_add(&dev->vm_node, &kvm->devices); 2615 kvm_get_kvm(kvm); 2616 cd->fd = ret; 2617 return 0; 2618 } 2619 2620 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 2621 { 2622 switch (arg) { 2623 case KVM_CAP_USER_MEMORY: 2624 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2625 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2626 case KVM_CAP_INTERNAL_ERROR_DATA: 2627 #ifdef CONFIG_HAVE_KVM_MSI 2628 case KVM_CAP_SIGNAL_MSI: 2629 #endif 2630 #ifdef CONFIG_HAVE_KVM_IRQFD 2631 case KVM_CAP_IRQFD: 2632 case KVM_CAP_IRQFD_RESAMPLE: 2633 #endif 2634 case KVM_CAP_CHECK_EXTENSION_VM: 2635 return 1; 2636 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2637 case KVM_CAP_IRQ_ROUTING: 2638 return KVM_MAX_IRQ_ROUTES; 2639 #endif 2640 #if KVM_ADDRESS_SPACE_NUM > 1 2641 case KVM_CAP_MULTI_ADDRESS_SPACE: 2642 return KVM_ADDRESS_SPACE_NUM; 2643 #endif 2644 default: 2645 break; 2646 } 2647 return kvm_vm_ioctl_check_extension(kvm, arg); 2648 } 2649 2650 static long kvm_vm_ioctl(struct file *filp, 2651 unsigned int ioctl, unsigned long arg) 2652 { 2653 struct kvm *kvm = filp->private_data; 2654 void __user *argp = (void __user *)arg; 2655 int r; 2656 2657 if (kvm->mm != current->mm) 2658 return -EIO; 2659 switch (ioctl) { 2660 case KVM_CREATE_VCPU: 2661 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2662 break; 2663 case KVM_SET_USER_MEMORY_REGION: { 2664 struct kvm_userspace_memory_region kvm_userspace_mem; 2665 2666 r = -EFAULT; 2667 if (copy_from_user(&kvm_userspace_mem, argp, 2668 sizeof(kvm_userspace_mem))) 2669 goto out; 2670 2671 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2672 break; 2673 } 2674 case KVM_GET_DIRTY_LOG: { 2675 struct kvm_dirty_log log; 2676 2677 r = -EFAULT; 2678 if (copy_from_user(&log, argp, sizeof(log))) 2679 goto out; 2680 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2681 break; 2682 } 2683 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2684 case KVM_REGISTER_COALESCED_MMIO: { 2685 struct kvm_coalesced_mmio_zone zone; 2686 2687 r = -EFAULT; 2688 if (copy_from_user(&zone, argp, sizeof(zone))) 2689 goto out; 2690 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2691 break; 2692 } 2693 case KVM_UNREGISTER_COALESCED_MMIO: { 2694 struct kvm_coalesced_mmio_zone zone; 2695 2696 r = -EFAULT; 2697 if (copy_from_user(&zone, argp, sizeof(zone))) 2698 goto out; 2699 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2700 break; 2701 } 2702 #endif 2703 case KVM_IRQFD: { 2704 struct kvm_irqfd data; 2705 2706 r = -EFAULT; 2707 if (copy_from_user(&data, argp, sizeof(data))) 2708 goto out; 2709 r = kvm_irqfd(kvm, &data); 2710 break; 2711 } 2712 case KVM_IOEVENTFD: { 2713 struct kvm_ioeventfd data; 2714 2715 r = -EFAULT; 2716 if (copy_from_user(&data, argp, sizeof(data))) 2717 goto out; 2718 r = kvm_ioeventfd(kvm, &data); 2719 break; 2720 } 2721 #ifdef CONFIG_HAVE_KVM_MSI 2722 case KVM_SIGNAL_MSI: { 2723 struct kvm_msi msi; 2724 2725 r = -EFAULT; 2726 if (copy_from_user(&msi, argp, sizeof(msi))) 2727 goto out; 2728 r = kvm_send_userspace_msi(kvm, &msi); 2729 break; 2730 } 2731 #endif 2732 #ifdef __KVM_HAVE_IRQ_LINE 2733 case KVM_IRQ_LINE_STATUS: 2734 case KVM_IRQ_LINE: { 2735 struct kvm_irq_level irq_event; 2736 2737 r = -EFAULT; 2738 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 2739 goto out; 2740 2741 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 2742 ioctl == KVM_IRQ_LINE_STATUS); 2743 if (r) 2744 goto out; 2745 2746 r = -EFAULT; 2747 if (ioctl == KVM_IRQ_LINE_STATUS) { 2748 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 2749 goto out; 2750 } 2751 2752 r = 0; 2753 break; 2754 } 2755 #endif 2756 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2757 case KVM_SET_GSI_ROUTING: { 2758 struct kvm_irq_routing routing; 2759 struct kvm_irq_routing __user *urouting; 2760 struct kvm_irq_routing_entry *entries; 2761 2762 r = -EFAULT; 2763 if (copy_from_user(&routing, argp, sizeof(routing))) 2764 goto out; 2765 r = -EINVAL; 2766 if (routing.nr >= KVM_MAX_IRQ_ROUTES) 2767 goto out; 2768 if (routing.flags) 2769 goto out; 2770 r = -ENOMEM; 2771 entries = vmalloc(routing.nr * sizeof(*entries)); 2772 if (!entries) 2773 goto out; 2774 r = -EFAULT; 2775 urouting = argp; 2776 if (copy_from_user(entries, urouting->entries, 2777 routing.nr * sizeof(*entries))) 2778 goto out_free_irq_routing; 2779 r = kvm_set_irq_routing(kvm, entries, routing.nr, 2780 routing.flags); 2781 out_free_irq_routing: 2782 vfree(entries); 2783 break; 2784 } 2785 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 2786 case KVM_CREATE_DEVICE: { 2787 struct kvm_create_device cd; 2788 2789 r = -EFAULT; 2790 if (copy_from_user(&cd, argp, sizeof(cd))) 2791 goto out; 2792 2793 r = kvm_ioctl_create_device(kvm, &cd); 2794 if (r) 2795 goto out; 2796 2797 r = -EFAULT; 2798 if (copy_to_user(argp, &cd, sizeof(cd))) 2799 goto out; 2800 2801 r = 0; 2802 break; 2803 } 2804 case KVM_CHECK_EXTENSION: 2805 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 2806 break; 2807 default: 2808 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2809 } 2810 out: 2811 return r; 2812 } 2813 2814 #ifdef CONFIG_KVM_COMPAT 2815 struct compat_kvm_dirty_log { 2816 __u32 slot; 2817 __u32 padding1; 2818 union { 2819 compat_uptr_t dirty_bitmap; /* one bit per page */ 2820 __u64 padding2; 2821 }; 2822 }; 2823 2824 static long kvm_vm_compat_ioctl(struct file *filp, 2825 unsigned int ioctl, unsigned long arg) 2826 { 2827 struct kvm *kvm = filp->private_data; 2828 int r; 2829 2830 if (kvm->mm != current->mm) 2831 return -EIO; 2832 switch (ioctl) { 2833 case KVM_GET_DIRTY_LOG: { 2834 struct compat_kvm_dirty_log compat_log; 2835 struct kvm_dirty_log log; 2836 2837 r = -EFAULT; 2838 if (copy_from_user(&compat_log, (void __user *)arg, 2839 sizeof(compat_log))) 2840 goto out; 2841 log.slot = compat_log.slot; 2842 log.padding1 = compat_log.padding1; 2843 log.padding2 = compat_log.padding2; 2844 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 2845 2846 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2847 break; 2848 } 2849 default: 2850 r = kvm_vm_ioctl(filp, ioctl, arg); 2851 } 2852 2853 out: 2854 return r; 2855 } 2856 #endif 2857 2858 static struct file_operations kvm_vm_fops = { 2859 .release = kvm_vm_release, 2860 .unlocked_ioctl = kvm_vm_ioctl, 2861 #ifdef CONFIG_KVM_COMPAT 2862 .compat_ioctl = kvm_vm_compat_ioctl, 2863 #endif 2864 .llseek = noop_llseek, 2865 }; 2866 2867 static int kvm_dev_ioctl_create_vm(unsigned long type) 2868 { 2869 int r; 2870 struct kvm *kvm; 2871 2872 kvm = kvm_create_vm(type); 2873 if (IS_ERR(kvm)) 2874 return PTR_ERR(kvm); 2875 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2876 r = kvm_coalesced_mmio_init(kvm); 2877 if (r < 0) { 2878 kvm_put_kvm(kvm); 2879 return r; 2880 } 2881 #endif 2882 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC); 2883 if (r < 0) 2884 kvm_put_kvm(kvm); 2885 2886 return r; 2887 } 2888 2889 static long kvm_dev_ioctl(struct file *filp, 2890 unsigned int ioctl, unsigned long arg) 2891 { 2892 long r = -EINVAL; 2893 2894 switch (ioctl) { 2895 case KVM_GET_API_VERSION: 2896 if (arg) 2897 goto out; 2898 r = KVM_API_VERSION; 2899 break; 2900 case KVM_CREATE_VM: 2901 r = kvm_dev_ioctl_create_vm(arg); 2902 break; 2903 case KVM_CHECK_EXTENSION: 2904 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 2905 break; 2906 case KVM_GET_VCPU_MMAP_SIZE: 2907 if (arg) 2908 goto out; 2909 r = PAGE_SIZE; /* struct kvm_run */ 2910 #ifdef CONFIG_X86 2911 r += PAGE_SIZE; /* pio data page */ 2912 #endif 2913 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2914 r += PAGE_SIZE; /* coalesced mmio ring page */ 2915 #endif 2916 break; 2917 case KVM_TRACE_ENABLE: 2918 case KVM_TRACE_PAUSE: 2919 case KVM_TRACE_DISABLE: 2920 r = -EOPNOTSUPP; 2921 break; 2922 default: 2923 return kvm_arch_dev_ioctl(filp, ioctl, arg); 2924 } 2925 out: 2926 return r; 2927 } 2928 2929 static struct file_operations kvm_chardev_ops = { 2930 .unlocked_ioctl = kvm_dev_ioctl, 2931 .compat_ioctl = kvm_dev_ioctl, 2932 .llseek = noop_llseek, 2933 }; 2934 2935 static struct miscdevice kvm_dev = { 2936 KVM_MINOR, 2937 "kvm", 2938 &kvm_chardev_ops, 2939 }; 2940 2941 static void hardware_enable_nolock(void *junk) 2942 { 2943 int cpu = raw_smp_processor_id(); 2944 int r; 2945 2946 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2947 return; 2948 2949 cpumask_set_cpu(cpu, cpus_hardware_enabled); 2950 2951 r = kvm_arch_hardware_enable(); 2952 2953 if (r) { 2954 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2955 atomic_inc(&hardware_enable_failed); 2956 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu); 2957 } 2958 } 2959 2960 static void hardware_enable(void) 2961 { 2962 raw_spin_lock(&kvm_count_lock); 2963 if (kvm_usage_count) 2964 hardware_enable_nolock(NULL); 2965 raw_spin_unlock(&kvm_count_lock); 2966 } 2967 2968 static void hardware_disable_nolock(void *junk) 2969 { 2970 int cpu = raw_smp_processor_id(); 2971 2972 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 2973 return; 2974 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 2975 kvm_arch_hardware_disable(); 2976 } 2977 2978 static void hardware_disable(void) 2979 { 2980 raw_spin_lock(&kvm_count_lock); 2981 if (kvm_usage_count) 2982 hardware_disable_nolock(NULL); 2983 raw_spin_unlock(&kvm_count_lock); 2984 } 2985 2986 static void hardware_disable_all_nolock(void) 2987 { 2988 BUG_ON(!kvm_usage_count); 2989 2990 kvm_usage_count--; 2991 if (!kvm_usage_count) 2992 on_each_cpu(hardware_disable_nolock, NULL, 1); 2993 } 2994 2995 static void hardware_disable_all(void) 2996 { 2997 raw_spin_lock(&kvm_count_lock); 2998 hardware_disable_all_nolock(); 2999 raw_spin_unlock(&kvm_count_lock); 3000 } 3001 3002 static int hardware_enable_all(void) 3003 { 3004 int r = 0; 3005 3006 raw_spin_lock(&kvm_count_lock); 3007 3008 kvm_usage_count++; 3009 if (kvm_usage_count == 1) { 3010 atomic_set(&hardware_enable_failed, 0); 3011 on_each_cpu(hardware_enable_nolock, NULL, 1); 3012 3013 if (atomic_read(&hardware_enable_failed)) { 3014 hardware_disable_all_nolock(); 3015 r = -EBUSY; 3016 } 3017 } 3018 3019 raw_spin_unlock(&kvm_count_lock); 3020 3021 return r; 3022 } 3023 3024 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 3025 void *v) 3026 { 3027 val &= ~CPU_TASKS_FROZEN; 3028 switch (val) { 3029 case CPU_DYING: 3030 hardware_disable(); 3031 break; 3032 case CPU_STARTING: 3033 hardware_enable(); 3034 break; 3035 } 3036 return NOTIFY_OK; 3037 } 3038 3039 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 3040 void *v) 3041 { 3042 /* 3043 * Some (well, at least mine) BIOSes hang on reboot if 3044 * in vmx root mode. 3045 * 3046 * And Intel TXT required VMX off for all cpu when system shutdown. 3047 */ 3048 pr_info("kvm: exiting hardware virtualization\n"); 3049 kvm_rebooting = true; 3050 on_each_cpu(hardware_disable_nolock, NULL, 1); 3051 return NOTIFY_OK; 3052 } 3053 3054 static struct notifier_block kvm_reboot_notifier = { 3055 .notifier_call = kvm_reboot, 3056 .priority = 0, 3057 }; 3058 3059 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 3060 { 3061 int i; 3062 3063 for (i = 0; i < bus->dev_count; i++) { 3064 struct kvm_io_device *pos = bus->range[i].dev; 3065 3066 kvm_iodevice_destructor(pos); 3067 } 3068 kfree(bus); 3069 } 3070 3071 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 3072 const struct kvm_io_range *r2) 3073 { 3074 if (r1->addr < r2->addr) 3075 return -1; 3076 if (r1->addr + r1->len > r2->addr + r2->len) 3077 return 1; 3078 return 0; 3079 } 3080 3081 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 3082 { 3083 return kvm_io_bus_cmp(p1, p2); 3084 } 3085 3086 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 3087 gpa_t addr, int len) 3088 { 3089 bus->range[bus->dev_count++] = (struct kvm_io_range) { 3090 .addr = addr, 3091 .len = len, 3092 .dev = dev, 3093 }; 3094 3095 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 3096 kvm_io_bus_sort_cmp, NULL); 3097 3098 return 0; 3099 } 3100 3101 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 3102 gpa_t addr, int len) 3103 { 3104 struct kvm_io_range *range, key; 3105 int off; 3106 3107 key = (struct kvm_io_range) { 3108 .addr = addr, 3109 .len = len, 3110 }; 3111 3112 range = bsearch(&key, bus->range, bus->dev_count, 3113 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 3114 if (range == NULL) 3115 return -ENOENT; 3116 3117 off = range - bus->range; 3118 3119 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 3120 off--; 3121 3122 return off; 3123 } 3124 3125 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3126 struct kvm_io_range *range, const void *val) 3127 { 3128 int idx; 3129 3130 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3131 if (idx < 0) 3132 return -EOPNOTSUPP; 3133 3134 while (idx < bus->dev_count && 3135 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3136 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 3137 range->len, val)) 3138 return idx; 3139 idx++; 3140 } 3141 3142 return -EOPNOTSUPP; 3143 } 3144 3145 /* kvm_io_bus_write - called under kvm->slots_lock */ 3146 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3147 int len, const void *val) 3148 { 3149 struct kvm_io_bus *bus; 3150 struct kvm_io_range range; 3151 int r; 3152 3153 range = (struct kvm_io_range) { 3154 .addr = addr, 3155 .len = len, 3156 }; 3157 3158 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3159 r = __kvm_io_bus_write(vcpu, bus, &range, val); 3160 return r < 0 ? r : 0; 3161 } 3162 3163 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 3164 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 3165 gpa_t addr, int len, const void *val, long cookie) 3166 { 3167 struct kvm_io_bus *bus; 3168 struct kvm_io_range range; 3169 3170 range = (struct kvm_io_range) { 3171 .addr = addr, 3172 .len = len, 3173 }; 3174 3175 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3176 3177 /* First try the device referenced by cookie. */ 3178 if ((cookie >= 0) && (cookie < bus->dev_count) && 3179 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 3180 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 3181 val)) 3182 return cookie; 3183 3184 /* 3185 * cookie contained garbage; fall back to search and return the 3186 * correct cookie value. 3187 */ 3188 return __kvm_io_bus_write(vcpu, bus, &range, val); 3189 } 3190 3191 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3192 struct kvm_io_range *range, void *val) 3193 { 3194 int idx; 3195 3196 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3197 if (idx < 0) 3198 return -EOPNOTSUPP; 3199 3200 while (idx < bus->dev_count && 3201 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3202 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 3203 range->len, val)) 3204 return idx; 3205 idx++; 3206 } 3207 3208 return -EOPNOTSUPP; 3209 } 3210 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3211 3212 /* kvm_io_bus_read - called under kvm->slots_lock */ 3213 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3214 int len, void *val) 3215 { 3216 struct kvm_io_bus *bus; 3217 struct kvm_io_range range; 3218 int r; 3219 3220 range = (struct kvm_io_range) { 3221 .addr = addr, 3222 .len = len, 3223 }; 3224 3225 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3226 r = __kvm_io_bus_read(vcpu, bus, &range, val); 3227 return r < 0 ? r : 0; 3228 } 3229 3230 3231 /* Caller must hold slots_lock. */ 3232 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3233 int len, struct kvm_io_device *dev) 3234 { 3235 struct kvm_io_bus *new_bus, *bus; 3236 3237 bus = kvm->buses[bus_idx]; 3238 /* exclude ioeventfd which is limited by maximum fd */ 3239 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 3240 return -ENOSPC; 3241 3242 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) * 3243 sizeof(struct kvm_io_range)), GFP_KERNEL); 3244 if (!new_bus) 3245 return -ENOMEM; 3246 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 3247 sizeof(struct kvm_io_range))); 3248 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 3249 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3250 synchronize_srcu_expedited(&kvm->srcu); 3251 kfree(bus); 3252 3253 return 0; 3254 } 3255 3256 /* Caller must hold slots_lock. */ 3257 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3258 struct kvm_io_device *dev) 3259 { 3260 int i, r; 3261 struct kvm_io_bus *new_bus, *bus; 3262 3263 bus = kvm->buses[bus_idx]; 3264 r = -ENOENT; 3265 for (i = 0; i < bus->dev_count; i++) 3266 if (bus->range[i].dev == dev) { 3267 r = 0; 3268 break; 3269 } 3270 3271 if (r) 3272 return r; 3273 3274 new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) * 3275 sizeof(struct kvm_io_range)), GFP_KERNEL); 3276 if (!new_bus) 3277 return -ENOMEM; 3278 3279 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3280 new_bus->dev_count--; 3281 memcpy(new_bus->range + i, bus->range + i + 1, 3282 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3283 3284 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3285 synchronize_srcu_expedited(&kvm->srcu); 3286 kfree(bus); 3287 return r; 3288 } 3289 3290 static struct notifier_block kvm_cpu_notifier = { 3291 .notifier_call = kvm_cpu_hotplug, 3292 }; 3293 3294 static int vm_stat_get(void *_offset, u64 *val) 3295 { 3296 unsigned offset = (long)_offset; 3297 struct kvm *kvm; 3298 3299 *val = 0; 3300 spin_lock(&kvm_lock); 3301 list_for_each_entry(kvm, &vm_list, vm_list) 3302 *val += *(u32 *)((void *)kvm + offset); 3303 spin_unlock(&kvm_lock); 3304 return 0; 3305 } 3306 3307 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 3308 3309 static int vcpu_stat_get(void *_offset, u64 *val) 3310 { 3311 unsigned offset = (long)_offset; 3312 struct kvm *kvm; 3313 struct kvm_vcpu *vcpu; 3314 int i; 3315 3316 *val = 0; 3317 spin_lock(&kvm_lock); 3318 list_for_each_entry(kvm, &vm_list, vm_list) 3319 kvm_for_each_vcpu(i, vcpu, kvm) 3320 *val += *(u32 *)((void *)vcpu + offset); 3321 3322 spin_unlock(&kvm_lock); 3323 return 0; 3324 } 3325 3326 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 3327 3328 static const struct file_operations *stat_fops[] = { 3329 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3330 [KVM_STAT_VM] = &vm_stat_fops, 3331 }; 3332 3333 static int kvm_init_debug(void) 3334 { 3335 int r = -EEXIST; 3336 struct kvm_stats_debugfs_item *p; 3337 3338 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3339 if (kvm_debugfs_dir == NULL) 3340 goto out; 3341 3342 for (p = debugfs_entries; p->name; ++p) { 3343 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 3344 (void *)(long)p->offset, 3345 stat_fops[p->kind]); 3346 if (p->dentry == NULL) 3347 goto out_dir; 3348 } 3349 3350 return 0; 3351 3352 out_dir: 3353 debugfs_remove_recursive(kvm_debugfs_dir); 3354 out: 3355 return r; 3356 } 3357 3358 static void kvm_exit_debug(void) 3359 { 3360 struct kvm_stats_debugfs_item *p; 3361 3362 for (p = debugfs_entries; p->name; ++p) 3363 debugfs_remove(p->dentry); 3364 debugfs_remove(kvm_debugfs_dir); 3365 } 3366 3367 static int kvm_suspend(void) 3368 { 3369 if (kvm_usage_count) 3370 hardware_disable_nolock(NULL); 3371 return 0; 3372 } 3373 3374 static void kvm_resume(void) 3375 { 3376 if (kvm_usage_count) { 3377 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3378 hardware_enable_nolock(NULL); 3379 } 3380 } 3381 3382 static struct syscore_ops kvm_syscore_ops = { 3383 .suspend = kvm_suspend, 3384 .resume = kvm_resume, 3385 }; 3386 3387 static inline 3388 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3389 { 3390 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3391 } 3392 3393 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3394 { 3395 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3396 3397 if (vcpu->preempted) 3398 vcpu->preempted = false; 3399 3400 kvm_arch_sched_in(vcpu, cpu); 3401 3402 kvm_arch_vcpu_load(vcpu, cpu); 3403 } 3404 3405 static void kvm_sched_out(struct preempt_notifier *pn, 3406 struct task_struct *next) 3407 { 3408 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3409 3410 if (current->state == TASK_RUNNING) 3411 vcpu->preempted = true; 3412 kvm_arch_vcpu_put(vcpu); 3413 } 3414 3415 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3416 struct module *module) 3417 { 3418 int r; 3419 int cpu; 3420 3421 r = kvm_arch_init(opaque); 3422 if (r) 3423 goto out_fail; 3424 3425 /* 3426 * kvm_arch_init makes sure there's at most one caller 3427 * for architectures that support multiple implementations, 3428 * like intel and amd on x86. 3429 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3430 * conflicts in case kvm is already setup for another implementation. 3431 */ 3432 r = kvm_irqfd_init(); 3433 if (r) 3434 goto out_irqfd; 3435 3436 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 3437 r = -ENOMEM; 3438 goto out_free_0; 3439 } 3440 3441 r = kvm_arch_hardware_setup(); 3442 if (r < 0) 3443 goto out_free_0a; 3444 3445 for_each_online_cpu(cpu) { 3446 smp_call_function_single(cpu, 3447 kvm_arch_check_processor_compat, 3448 &r, 1); 3449 if (r < 0) 3450 goto out_free_1; 3451 } 3452 3453 r = register_cpu_notifier(&kvm_cpu_notifier); 3454 if (r) 3455 goto out_free_2; 3456 register_reboot_notifier(&kvm_reboot_notifier); 3457 3458 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 3459 if (!vcpu_align) 3460 vcpu_align = __alignof__(struct kvm_vcpu); 3461 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 3462 0, NULL); 3463 if (!kvm_vcpu_cache) { 3464 r = -ENOMEM; 3465 goto out_free_3; 3466 } 3467 3468 r = kvm_async_pf_init(); 3469 if (r) 3470 goto out_free; 3471 3472 kvm_chardev_ops.owner = module; 3473 kvm_vm_fops.owner = module; 3474 kvm_vcpu_fops.owner = module; 3475 3476 r = misc_register(&kvm_dev); 3477 if (r) { 3478 pr_err("kvm: misc device register failed\n"); 3479 goto out_unreg; 3480 } 3481 3482 register_syscore_ops(&kvm_syscore_ops); 3483 3484 kvm_preempt_ops.sched_in = kvm_sched_in; 3485 kvm_preempt_ops.sched_out = kvm_sched_out; 3486 3487 r = kvm_init_debug(); 3488 if (r) { 3489 pr_err("kvm: create debugfs files failed\n"); 3490 goto out_undebugfs; 3491 } 3492 3493 r = kvm_vfio_ops_init(); 3494 WARN_ON(r); 3495 3496 return 0; 3497 3498 out_undebugfs: 3499 unregister_syscore_ops(&kvm_syscore_ops); 3500 misc_deregister(&kvm_dev); 3501 out_unreg: 3502 kvm_async_pf_deinit(); 3503 out_free: 3504 kmem_cache_destroy(kvm_vcpu_cache); 3505 out_free_3: 3506 unregister_reboot_notifier(&kvm_reboot_notifier); 3507 unregister_cpu_notifier(&kvm_cpu_notifier); 3508 out_free_2: 3509 out_free_1: 3510 kvm_arch_hardware_unsetup(); 3511 out_free_0a: 3512 free_cpumask_var(cpus_hardware_enabled); 3513 out_free_0: 3514 kvm_irqfd_exit(); 3515 out_irqfd: 3516 kvm_arch_exit(); 3517 out_fail: 3518 return r; 3519 } 3520 EXPORT_SYMBOL_GPL(kvm_init); 3521 3522 void kvm_exit(void) 3523 { 3524 kvm_exit_debug(); 3525 misc_deregister(&kvm_dev); 3526 kmem_cache_destroy(kvm_vcpu_cache); 3527 kvm_async_pf_deinit(); 3528 unregister_syscore_ops(&kvm_syscore_ops); 3529 unregister_reboot_notifier(&kvm_reboot_notifier); 3530 unregister_cpu_notifier(&kvm_cpu_notifier); 3531 on_each_cpu(hardware_disable_nolock, NULL, 1); 3532 kvm_arch_hardware_unsetup(); 3533 kvm_arch_exit(); 3534 kvm_irqfd_exit(); 3535 free_cpumask_var(cpus_hardware_enabled); 3536 kvm_vfio_ops_exit(); 3537 } 3538 EXPORT_SYMBOL_GPL(kvm_exit); 3539