1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include <kvm/iodev.h> 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 #include <linux/sort.h> 51 #include <linux/bsearch.h> 52 53 #include <asm/processor.h> 54 #include <asm/io.h> 55 #include <asm/ioctl.h> 56 #include <asm/uaccess.h> 57 #include <asm/pgtable.h> 58 59 #include "coalesced_mmio.h" 60 #include "async_pf.h" 61 #include "vfio.h" 62 63 #define CREATE_TRACE_POINTS 64 #include <trace/events/kvm.h> 65 66 MODULE_AUTHOR("Qumranet"); 67 MODULE_LICENSE("GPL"); 68 69 /* Architectures should define their poll value according to the halt latency */ 70 static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; 71 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR); 72 73 /* Default doubles per-vcpu halt_poll_ns. */ 74 static unsigned int halt_poll_ns_grow = 2; 75 module_param(halt_poll_ns_grow, uint, S_IRUGO | S_IWUSR); 76 77 /* Default resets per-vcpu halt_poll_ns . */ 78 static unsigned int halt_poll_ns_shrink; 79 module_param(halt_poll_ns_shrink, uint, S_IRUGO | S_IWUSR); 80 81 /* 82 * Ordering of locks: 83 * 84 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 85 */ 86 87 DEFINE_SPINLOCK(kvm_lock); 88 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 89 LIST_HEAD(vm_list); 90 91 static cpumask_var_t cpus_hardware_enabled; 92 static int kvm_usage_count; 93 static atomic_t hardware_enable_failed; 94 95 struct kmem_cache *kvm_vcpu_cache; 96 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 97 98 static __read_mostly struct preempt_ops kvm_preempt_ops; 99 100 struct dentry *kvm_debugfs_dir; 101 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 102 103 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 104 unsigned long arg); 105 #ifdef CONFIG_KVM_COMPAT 106 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 107 unsigned long arg); 108 #endif 109 static int hardware_enable_all(void); 110 static void hardware_disable_all(void); 111 112 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 113 114 static void kvm_release_pfn_dirty(kvm_pfn_t pfn); 115 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn); 116 117 __visible bool kvm_rebooting; 118 EXPORT_SYMBOL_GPL(kvm_rebooting); 119 120 static bool largepages_enabled = true; 121 122 bool kvm_is_reserved_pfn(kvm_pfn_t pfn) 123 { 124 if (pfn_valid(pfn)) 125 return PageReserved(pfn_to_page(pfn)); 126 127 return true; 128 } 129 130 /* 131 * Switches to specified vcpu, until a matching vcpu_put() 132 */ 133 int vcpu_load(struct kvm_vcpu *vcpu) 134 { 135 int cpu; 136 137 if (mutex_lock_killable(&vcpu->mutex)) 138 return -EINTR; 139 cpu = get_cpu(); 140 preempt_notifier_register(&vcpu->preempt_notifier); 141 kvm_arch_vcpu_load(vcpu, cpu); 142 put_cpu(); 143 return 0; 144 } 145 146 void vcpu_put(struct kvm_vcpu *vcpu) 147 { 148 preempt_disable(); 149 kvm_arch_vcpu_put(vcpu); 150 preempt_notifier_unregister(&vcpu->preempt_notifier); 151 preempt_enable(); 152 mutex_unlock(&vcpu->mutex); 153 } 154 155 static void ack_flush(void *_completed) 156 { 157 } 158 159 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 160 { 161 int i, cpu, me; 162 cpumask_var_t cpus; 163 bool called = true; 164 struct kvm_vcpu *vcpu; 165 166 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 167 168 me = get_cpu(); 169 kvm_for_each_vcpu(i, vcpu, kvm) { 170 kvm_make_request(req, vcpu); 171 cpu = vcpu->cpu; 172 173 /* Set ->requests bit before we read ->mode. */ 174 smp_mb__after_atomic(); 175 176 if (cpus != NULL && cpu != -1 && cpu != me && 177 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 178 cpumask_set_cpu(cpu, cpus); 179 } 180 if (unlikely(cpus == NULL)) 181 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 182 else if (!cpumask_empty(cpus)) 183 smp_call_function_many(cpus, ack_flush, NULL, 1); 184 else 185 called = false; 186 put_cpu(); 187 free_cpumask_var(cpus); 188 return called; 189 } 190 191 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 192 void kvm_flush_remote_tlbs(struct kvm *kvm) 193 { 194 /* 195 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in 196 * kvm_make_all_cpus_request. 197 */ 198 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty); 199 200 /* 201 * We want to publish modifications to the page tables before reading 202 * mode. Pairs with a memory barrier in arch-specific code. 203 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest 204 * and smp_mb in walk_shadow_page_lockless_begin/end. 205 * - powerpc: smp_mb in kvmppc_prepare_to_enter. 206 * 207 * There is already an smp_mb__after_atomic() before 208 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that 209 * barrier here. 210 */ 211 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 212 ++kvm->stat.remote_tlb_flush; 213 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 214 } 215 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 216 #endif 217 218 void kvm_reload_remote_mmus(struct kvm *kvm) 219 { 220 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 221 } 222 223 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 224 { 225 struct page *page; 226 int r; 227 228 mutex_init(&vcpu->mutex); 229 vcpu->cpu = -1; 230 vcpu->kvm = kvm; 231 vcpu->vcpu_id = id; 232 vcpu->pid = NULL; 233 init_swait_queue_head(&vcpu->wq); 234 kvm_async_pf_vcpu_init(vcpu); 235 236 vcpu->pre_pcpu = -1; 237 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list); 238 239 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 240 if (!page) { 241 r = -ENOMEM; 242 goto fail; 243 } 244 vcpu->run = page_address(page); 245 246 kvm_vcpu_set_in_spin_loop(vcpu, false); 247 kvm_vcpu_set_dy_eligible(vcpu, false); 248 vcpu->preempted = false; 249 250 r = kvm_arch_vcpu_init(vcpu); 251 if (r < 0) 252 goto fail_free_run; 253 return 0; 254 255 fail_free_run: 256 free_page((unsigned long)vcpu->run); 257 fail: 258 return r; 259 } 260 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 261 262 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 263 { 264 put_pid(vcpu->pid); 265 kvm_arch_vcpu_uninit(vcpu); 266 free_page((unsigned long)vcpu->run); 267 } 268 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 269 270 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 271 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 272 { 273 return container_of(mn, struct kvm, mmu_notifier); 274 } 275 276 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 277 struct mm_struct *mm, 278 unsigned long address) 279 { 280 struct kvm *kvm = mmu_notifier_to_kvm(mn); 281 int need_tlb_flush, idx; 282 283 /* 284 * When ->invalidate_page runs, the linux pte has been zapped 285 * already but the page is still allocated until 286 * ->invalidate_page returns. So if we increase the sequence 287 * here the kvm page fault will notice if the spte can't be 288 * established because the page is going to be freed. If 289 * instead the kvm page fault establishes the spte before 290 * ->invalidate_page runs, kvm_unmap_hva will release it 291 * before returning. 292 * 293 * The sequence increase only need to be seen at spin_unlock 294 * time, and not at spin_lock time. 295 * 296 * Increasing the sequence after the spin_unlock would be 297 * unsafe because the kvm page fault could then establish the 298 * pte after kvm_unmap_hva returned, without noticing the page 299 * is going to be freed. 300 */ 301 idx = srcu_read_lock(&kvm->srcu); 302 spin_lock(&kvm->mmu_lock); 303 304 kvm->mmu_notifier_seq++; 305 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 306 /* we've to flush the tlb before the pages can be freed */ 307 if (need_tlb_flush) 308 kvm_flush_remote_tlbs(kvm); 309 310 spin_unlock(&kvm->mmu_lock); 311 312 kvm_arch_mmu_notifier_invalidate_page(kvm, address); 313 314 srcu_read_unlock(&kvm->srcu, idx); 315 } 316 317 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 318 struct mm_struct *mm, 319 unsigned long address, 320 pte_t pte) 321 { 322 struct kvm *kvm = mmu_notifier_to_kvm(mn); 323 int idx; 324 325 idx = srcu_read_lock(&kvm->srcu); 326 spin_lock(&kvm->mmu_lock); 327 kvm->mmu_notifier_seq++; 328 kvm_set_spte_hva(kvm, address, pte); 329 spin_unlock(&kvm->mmu_lock); 330 srcu_read_unlock(&kvm->srcu, idx); 331 } 332 333 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 334 struct mm_struct *mm, 335 unsigned long start, 336 unsigned long end) 337 { 338 struct kvm *kvm = mmu_notifier_to_kvm(mn); 339 int need_tlb_flush = 0, idx; 340 341 idx = srcu_read_lock(&kvm->srcu); 342 spin_lock(&kvm->mmu_lock); 343 /* 344 * The count increase must become visible at unlock time as no 345 * spte can be established without taking the mmu_lock and 346 * count is also read inside the mmu_lock critical section. 347 */ 348 kvm->mmu_notifier_count++; 349 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 350 need_tlb_flush |= kvm->tlbs_dirty; 351 /* we've to flush the tlb before the pages can be freed */ 352 if (need_tlb_flush) 353 kvm_flush_remote_tlbs(kvm); 354 355 spin_unlock(&kvm->mmu_lock); 356 srcu_read_unlock(&kvm->srcu, idx); 357 } 358 359 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 360 struct mm_struct *mm, 361 unsigned long start, 362 unsigned long end) 363 { 364 struct kvm *kvm = mmu_notifier_to_kvm(mn); 365 366 spin_lock(&kvm->mmu_lock); 367 /* 368 * This sequence increase will notify the kvm page fault that 369 * the page that is going to be mapped in the spte could have 370 * been freed. 371 */ 372 kvm->mmu_notifier_seq++; 373 smp_wmb(); 374 /* 375 * The above sequence increase must be visible before the 376 * below count decrease, which is ensured by the smp_wmb above 377 * in conjunction with the smp_rmb in mmu_notifier_retry(). 378 */ 379 kvm->mmu_notifier_count--; 380 spin_unlock(&kvm->mmu_lock); 381 382 BUG_ON(kvm->mmu_notifier_count < 0); 383 } 384 385 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 386 struct mm_struct *mm, 387 unsigned long start, 388 unsigned long end) 389 { 390 struct kvm *kvm = mmu_notifier_to_kvm(mn); 391 int young, idx; 392 393 idx = srcu_read_lock(&kvm->srcu); 394 spin_lock(&kvm->mmu_lock); 395 396 young = kvm_age_hva(kvm, start, end); 397 if (young) 398 kvm_flush_remote_tlbs(kvm); 399 400 spin_unlock(&kvm->mmu_lock); 401 srcu_read_unlock(&kvm->srcu, idx); 402 403 return young; 404 } 405 406 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, 407 struct mm_struct *mm, 408 unsigned long start, 409 unsigned long end) 410 { 411 struct kvm *kvm = mmu_notifier_to_kvm(mn); 412 int young, idx; 413 414 idx = srcu_read_lock(&kvm->srcu); 415 spin_lock(&kvm->mmu_lock); 416 /* 417 * Even though we do not flush TLB, this will still adversely 418 * affect performance on pre-Haswell Intel EPT, where there is 419 * no EPT Access Bit to clear so that we have to tear down EPT 420 * tables instead. If we find this unacceptable, we can always 421 * add a parameter to kvm_age_hva so that it effectively doesn't 422 * do anything on clear_young. 423 * 424 * Also note that currently we never issue secondary TLB flushes 425 * from clear_young, leaving this job up to the regular system 426 * cadence. If we find this inaccurate, we might come up with a 427 * more sophisticated heuristic later. 428 */ 429 young = kvm_age_hva(kvm, start, end); 430 spin_unlock(&kvm->mmu_lock); 431 srcu_read_unlock(&kvm->srcu, idx); 432 433 return young; 434 } 435 436 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 437 struct mm_struct *mm, 438 unsigned long address) 439 { 440 struct kvm *kvm = mmu_notifier_to_kvm(mn); 441 int young, idx; 442 443 idx = srcu_read_lock(&kvm->srcu); 444 spin_lock(&kvm->mmu_lock); 445 young = kvm_test_age_hva(kvm, address); 446 spin_unlock(&kvm->mmu_lock); 447 srcu_read_unlock(&kvm->srcu, idx); 448 449 return young; 450 } 451 452 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 453 struct mm_struct *mm) 454 { 455 struct kvm *kvm = mmu_notifier_to_kvm(mn); 456 int idx; 457 458 idx = srcu_read_lock(&kvm->srcu); 459 kvm_arch_flush_shadow_all(kvm); 460 srcu_read_unlock(&kvm->srcu, idx); 461 } 462 463 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 464 .invalidate_page = kvm_mmu_notifier_invalidate_page, 465 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 466 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 467 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 468 .clear_young = kvm_mmu_notifier_clear_young, 469 .test_young = kvm_mmu_notifier_test_young, 470 .change_pte = kvm_mmu_notifier_change_pte, 471 .release = kvm_mmu_notifier_release, 472 }; 473 474 static int kvm_init_mmu_notifier(struct kvm *kvm) 475 { 476 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 477 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 478 } 479 480 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 481 482 static int kvm_init_mmu_notifier(struct kvm *kvm) 483 { 484 return 0; 485 } 486 487 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 488 489 static struct kvm_memslots *kvm_alloc_memslots(void) 490 { 491 int i; 492 struct kvm_memslots *slots; 493 494 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 495 if (!slots) 496 return NULL; 497 498 /* 499 * Init kvm generation close to the maximum to easily test the 500 * code of handling generation number wrap-around. 501 */ 502 slots->generation = -150; 503 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 504 slots->id_to_index[i] = slots->memslots[i].id = i; 505 506 return slots; 507 } 508 509 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 510 { 511 if (!memslot->dirty_bitmap) 512 return; 513 514 kvfree(memslot->dirty_bitmap); 515 memslot->dirty_bitmap = NULL; 516 } 517 518 /* 519 * Free any memory in @free but not in @dont. 520 */ 521 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 522 struct kvm_memory_slot *dont) 523 { 524 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 525 kvm_destroy_dirty_bitmap(free); 526 527 kvm_arch_free_memslot(kvm, free, dont); 528 529 free->npages = 0; 530 } 531 532 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 533 { 534 struct kvm_memory_slot *memslot; 535 536 if (!slots) 537 return; 538 539 kvm_for_each_memslot(memslot, slots) 540 kvm_free_memslot(kvm, memslot, NULL); 541 542 kvfree(slots); 543 } 544 545 static struct kvm *kvm_create_vm(unsigned long type) 546 { 547 int r, i; 548 struct kvm *kvm = kvm_arch_alloc_vm(); 549 550 if (!kvm) 551 return ERR_PTR(-ENOMEM); 552 553 spin_lock_init(&kvm->mmu_lock); 554 atomic_inc(¤t->mm->mm_count); 555 kvm->mm = current->mm; 556 kvm_eventfd_init(kvm); 557 mutex_init(&kvm->lock); 558 mutex_init(&kvm->irq_lock); 559 mutex_init(&kvm->slots_lock); 560 atomic_set(&kvm->users_count, 1); 561 INIT_LIST_HEAD(&kvm->devices); 562 563 r = kvm_arch_init_vm(kvm, type); 564 if (r) 565 goto out_err_no_disable; 566 567 r = hardware_enable_all(); 568 if (r) 569 goto out_err_no_disable; 570 571 #ifdef CONFIG_HAVE_KVM_IRQFD 572 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 573 #endif 574 575 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 576 577 r = -ENOMEM; 578 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 579 kvm->memslots[i] = kvm_alloc_memslots(); 580 if (!kvm->memslots[i]) 581 goto out_err_no_srcu; 582 } 583 584 if (init_srcu_struct(&kvm->srcu)) 585 goto out_err_no_srcu; 586 if (init_srcu_struct(&kvm->irq_srcu)) 587 goto out_err_no_irq_srcu; 588 for (i = 0; i < KVM_NR_BUSES; i++) { 589 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 590 GFP_KERNEL); 591 if (!kvm->buses[i]) 592 goto out_err; 593 } 594 595 r = kvm_init_mmu_notifier(kvm); 596 if (r) 597 goto out_err; 598 599 spin_lock(&kvm_lock); 600 list_add(&kvm->vm_list, &vm_list); 601 spin_unlock(&kvm_lock); 602 603 preempt_notifier_inc(); 604 605 return kvm; 606 607 out_err: 608 cleanup_srcu_struct(&kvm->irq_srcu); 609 out_err_no_irq_srcu: 610 cleanup_srcu_struct(&kvm->srcu); 611 out_err_no_srcu: 612 hardware_disable_all(); 613 out_err_no_disable: 614 for (i = 0; i < KVM_NR_BUSES; i++) 615 kfree(kvm->buses[i]); 616 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 617 kvm_free_memslots(kvm, kvm->memslots[i]); 618 kvm_arch_free_vm(kvm); 619 mmdrop(current->mm); 620 return ERR_PTR(r); 621 } 622 623 /* 624 * Avoid using vmalloc for a small buffer. 625 * Should not be used when the size is statically known. 626 */ 627 void *kvm_kvzalloc(unsigned long size) 628 { 629 if (size > PAGE_SIZE) 630 return vzalloc(size); 631 else 632 return kzalloc(size, GFP_KERNEL); 633 } 634 635 static void kvm_destroy_devices(struct kvm *kvm) 636 { 637 struct kvm_device *dev, *tmp; 638 639 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { 640 list_del(&dev->vm_node); 641 dev->ops->destroy(dev); 642 } 643 } 644 645 static void kvm_destroy_vm(struct kvm *kvm) 646 { 647 int i; 648 struct mm_struct *mm = kvm->mm; 649 650 kvm_arch_sync_events(kvm); 651 spin_lock(&kvm_lock); 652 list_del(&kvm->vm_list); 653 spin_unlock(&kvm_lock); 654 kvm_free_irq_routing(kvm); 655 for (i = 0; i < KVM_NR_BUSES; i++) 656 kvm_io_bus_destroy(kvm->buses[i]); 657 kvm_coalesced_mmio_free(kvm); 658 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 659 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 660 #else 661 kvm_arch_flush_shadow_all(kvm); 662 #endif 663 kvm_arch_destroy_vm(kvm); 664 kvm_destroy_devices(kvm); 665 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 666 kvm_free_memslots(kvm, kvm->memslots[i]); 667 cleanup_srcu_struct(&kvm->irq_srcu); 668 cleanup_srcu_struct(&kvm->srcu); 669 kvm_arch_free_vm(kvm); 670 preempt_notifier_dec(); 671 hardware_disable_all(); 672 mmdrop(mm); 673 } 674 675 void kvm_get_kvm(struct kvm *kvm) 676 { 677 atomic_inc(&kvm->users_count); 678 } 679 EXPORT_SYMBOL_GPL(kvm_get_kvm); 680 681 void kvm_put_kvm(struct kvm *kvm) 682 { 683 if (atomic_dec_and_test(&kvm->users_count)) 684 kvm_destroy_vm(kvm); 685 } 686 EXPORT_SYMBOL_GPL(kvm_put_kvm); 687 688 689 static int kvm_vm_release(struct inode *inode, struct file *filp) 690 { 691 struct kvm *kvm = filp->private_data; 692 693 kvm_irqfd_release(kvm); 694 695 kvm_put_kvm(kvm); 696 return 0; 697 } 698 699 /* 700 * Allocation size is twice as large as the actual dirty bitmap size. 701 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 702 */ 703 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 704 { 705 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 706 707 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes); 708 if (!memslot->dirty_bitmap) 709 return -ENOMEM; 710 711 return 0; 712 } 713 714 /* 715 * Insert memslot and re-sort memslots based on their GFN, 716 * so binary search could be used to lookup GFN. 717 * Sorting algorithm takes advantage of having initially 718 * sorted array and known changed memslot position. 719 */ 720 static void update_memslots(struct kvm_memslots *slots, 721 struct kvm_memory_slot *new) 722 { 723 int id = new->id; 724 int i = slots->id_to_index[id]; 725 struct kvm_memory_slot *mslots = slots->memslots; 726 727 WARN_ON(mslots[i].id != id); 728 if (!new->npages) { 729 WARN_ON(!mslots[i].npages); 730 if (mslots[i].npages) 731 slots->used_slots--; 732 } else { 733 if (!mslots[i].npages) 734 slots->used_slots++; 735 } 736 737 while (i < KVM_MEM_SLOTS_NUM - 1 && 738 new->base_gfn <= mslots[i + 1].base_gfn) { 739 if (!mslots[i + 1].npages) 740 break; 741 mslots[i] = mslots[i + 1]; 742 slots->id_to_index[mslots[i].id] = i; 743 i++; 744 } 745 746 /* 747 * The ">=" is needed when creating a slot with base_gfn == 0, 748 * so that it moves before all those with base_gfn == npages == 0. 749 * 750 * On the other hand, if new->npages is zero, the above loop has 751 * already left i pointing to the beginning of the empty part of 752 * mslots, and the ">=" would move the hole backwards in this 753 * case---which is wrong. So skip the loop when deleting a slot. 754 */ 755 if (new->npages) { 756 while (i > 0 && 757 new->base_gfn >= mslots[i - 1].base_gfn) { 758 mslots[i] = mslots[i - 1]; 759 slots->id_to_index[mslots[i].id] = i; 760 i--; 761 } 762 } else 763 WARN_ON_ONCE(i != slots->used_slots); 764 765 mslots[i] = *new; 766 slots->id_to_index[mslots[i].id] = i; 767 } 768 769 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 770 { 771 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 772 773 #ifdef __KVM_HAVE_READONLY_MEM 774 valid_flags |= KVM_MEM_READONLY; 775 #endif 776 777 if (mem->flags & ~valid_flags) 778 return -EINVAL; 779 780 return 0; 781 } 782 783 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 784 int as_id, struct kvm_memslots *slots) 785 { 786 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); 787 788 /* 789 * Set the low bit in the generation, which disables SPTE caching 790 * until the end of synchronize_srcu_expedited. 791 */ 792 WARN_ON(old_memslots->generation & 1); 793 slots->generation = old_memslots->generation + 1; 794 795 rcu_assign_pointer(kvm->memslots[as_id], slots); 796 synchronize_srcu_expedited(&kvm->srcu); 797 798 /* 799 * Increment the new memslot generation a second time. This prevents 800 * vm exits that race with memslot updates from caching a memslot 801 * generation that will (potentially) be valid forever. 802 */ 803 slots->generation++; 804 805 kvm_arch_memslots_updated(kvm, slots); 806 807 return old_memslots; 808 } 809 810 /* 811 * Allocate some memory and give it an address in the guest physical address 812 * space. 813 * 814 * Discontiguous memory is allowed, mostly for framebuffers. 815 * 816 * Must be called holding kvm->slots_lock for write. 817 */ 818 int __kvm_set_memory_region(struct kvm *kvm, 819 const struct kvm_userspace_memory_region *mem) 820 { 821 int r; 822 gfn_t base_gfn; 823 unsigned long npages; 824 struct kvm_memory_slot *slot; 825 struct kvm_memory_slot old, new; 826 struct kvm_memslots *slots = NULL, *old_memslots; 827 int as_id, id; 828 enum kvm_mr_change change; 829 830 r = check_memory_region_flags(mem); 831 if (r) 832 goto out; 833 834 r = -EINVAL; 835 as_id = mem->slot >> 16; 836 id = (u16)mem->slot; 837 838 /* General sanity checks */ 839 if (mem->memory_size & (PAGE_SIZE - 1)) 840 goto out; 841 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 842 goto out; 843 /* We can read the guest memory with __xxx_user() later on. */ 844 if ((id < KVM_USER_MEM_SLOTS) && 845 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 846 !access_ok(VERIFY_WRITE, 847 (void __user *)(unsigned long)mem->userspace_addr, 848 mem->memory_size))) 849 goto out; 850 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 851 goto out; 852 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 853 goto out; 854 855 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); 856 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 857 npages = mem->memory_size >> PAGE_SHIFT; 858 859 if (npages > KVM_MEM_MAX_NR_PAGES) 860 goto out; 861 862 new = old = *slot; 863 864 new.id = id; 865 new.base_gfn = base_gfn; 866 new.npages = npages; 867 new.flags = mem->flags; 868 869 if (npages) { 870 if (!old.npages) 871 change = KVM_MR_CREATE; 872 else { /* Modify an existing slot. */ 873 if ((mem->userspace_addr != old.userspace_addr) || 874 (npages != old.npages) || 875 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 876 goto out; 877 878 if (base_gfn != old.base_gfn) 879 change = KVM_MR_MOVE; 880 else if (new.flags != old.flags) 881 change = KVM_MR_FLAGS_ONLY; 882 else { /* Nothing to change. */ 883 r = 0; 884 goto out; 885 } 886 } 887 } else { 888 if (!old.npages) 889 goto out; 890 891 change = KVM_MR_DELETE; 892 new.base_gfn = 0; 893 new.flags = 0; 894 } 895 896 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 897 /* Check for overlaps */ 898 r = -EEXIST; 899 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { 900 if ((slot->id >= KVM_USER_MEM_SLOTS) || 901 (slot->id == id)) 902 continue; 903 if (!((base_gfn + npages <= slot->base_gfn) || 904 (base_gfn >= slot->base_gfn + slot->npages))) 905 goto out; 906 } 907 } 908 909 /* Free page dirty bitmap if unneeded */ 910 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 911 new.dirty_bitmap = NULL; 912 913 r = -ENOMEM; 914 if (change == KVM_MR_CREATE) { 915 new.userspace_addr = mem->userspace_addr; 916 917 if (kvm_arch_create_memslot(kvm, &new, npages)) 918 goto out_free; 919 } 920 921 /* Allocate page dirty bitmap if needed */ 922 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 923 if (kvm_create_dirty_bitmap(&new) < 0) 924 goto out_free; 925 } 926 927 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 928 if (!slots) 929 goto out_free; 930 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); 931 932 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 933 slot = id_to_memslot(slots, id); 934 slot->flags |= KVM_MEMSLOT_INVALID; 935 936 old_memslots = install_new_memslots(kvm, as_id, slots); 937 938 /* slot was deleted or moved, clear iommu mapping */ 939 kvm_iommu_unmap_pages(kvm, &old); 940 /* From this point no new shadow pages pointing to a deleted, 941 * or moved, memslot will be created. 942 * 943 * validation of sp->gfn happens in: 944 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 945 * - kvm_is_visible_gfn (mmu_check_roots) 946 */ 947 kvm_arch_flush_shadow_memslot(kvm, slot); 948 949 /* 950 * We can re-use the old_memslots from above, the only difference 951 * from the currently installed memslots is the invalid flag. This 952 * will get overwritten by update_memslots anyway. 953 */ 954 slots = old_memslots; 955 } 956 957 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 958 if (r) 959 goto out_slots; 960 961 /* actual memory is freed via old in kvm_free_memslot below */ 962 if (change == KVM_MR_DELETE) { 963 new.dirty_bitmap = NULL; 964 memset(&new.arch, 0, sizeof(new.arch)); 965 } 966 967 update_memslots(slots, &new); 968 old_memslots = install_new_memslots(kvm, as_id, slots); 969 970 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); 971 972 kvm_free_memslot(kvm, &old, &new); 973 kvfree(old_memslots); 974 975 /* 976 * IOMMU mapping: New slots need to be mapped. Old slots need to be 977 * un-mapped and re-mapped if their base changes. Since base change 978 * unmapping is handled above with slot deletion, mapping alone is 979 * needed here. Anything else the iommu might care about for existing 980 * slots (size changes, userspace addr changes and read-only flag 981 * changes) is disallowed above, so any other attribute changes getting 982 * here can be skipped. 983 */ 984 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 985 r = kvm_iommu_map_pages(kvm, &new); 986 return r; 987 } 988 989 return 0; 990 991 out_slots: 992 kvfree(slots); 993 out_free: 994 kvm_free_memslot(kvm, &new, &old); 995 out: 996 return r; 997 } 998 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 999 1000 int kvm_set_memory_region(struct kvm *kvm, 1001 const struct kvm_userspace_memory_region *mem) 1002 { 1003 int r; 1004 1005 mutex_lock(&kvm->slots_lock); 1006 r = __kvm_set_memory_region(kvm, mem); 1007 mutex_unlock(&kvm->slots_lock); 1008 return r; 1009 } 1010 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 1011 1012 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 1013 struct kvm_userspace_memory_region *mem) 1014 { 1015 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 1016 return -EINVAL; 1017 1018 return kvm_set_memory_region(kvm, mem); 1019 } 1020 1021 int kvm_get_dirty_log(struct kvm *kvm, 1022 struct kvm_dirty_log *log, int *is_dirty) 1023 { 1024 struct kvm_memslots *slots; 1025 struct kvm_memory_slot *memslot; 1026 int r, i, as_id, id; 1027 unsigned long n; 1028 unsigned long any = 0; 1029 1030 r = -EINVAL; 1031 as_id = log->slot >> 16; 1032 id = (u16)log->slot; 1033 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1034 goto out; 1035 1036 slots = __kvm_memslots(kvm, as_id); 1037 memslot = id_to_memslot(slots, id); 1038 r = -ENOENT; 1039 if (!memslot->dirty_bitmap) 1040 goto out; 1041 1042 n = kvm_dirty_bitmap_bytes(memslot); 1043 1044 for (i = 0; !any && i < n/sizeof(long); ++i) 1045 any = memslot->dirty_bitmap[i]; 1046 1047 r = -EFAULT; 1048 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 1049 goto out; 1050 1051 if (any) 1052 *is_dirty = 1; 1053 1054 r = 0; 1055 out: 1056 return r; 1057 } 1058 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1059 1060 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1061 /** 1062 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages 1063 * are dirty write protect them for next write. 1064 * @kvm: pointer to kvm instance 1065 * @log: slot id and address to which we copy the log 1066 * @is_dirty: flag set if any page is dirty 1067 * 1068 * We need to keep it in mind that VCPU threads can write to the bitmap 1069 * concurrently. So, to avoid losing track of dirty pages we keep the 1070 * following order: 1071 * 1072 * 1. Take a snapshot of the bit and clear it if needed. 1073 * 2. Write protect the corresponding page. 1074 * 3. Copy the snapshot to the userspace. 1075 * 4. Upon return caller flushes TLB's if needed. 1076 * 1077 * Between 2 and 4, the guest may write to the page using the remaining TLB 1078 * entry. This is not a problem because the page is reported dirty using 1079 * the snapshot taken before and step 4 ensures that writes done after 1080 * exiting to userspace will be logged for the next call. 1081 * 1082 */ 1083 int kvm_get_dirty_log_protect(struct kvm *kvm, 1084 struct kvm_dirty_log *log, bool *is_dirty) 1085 { 1086 struct kvm_memslots *slots; 1087 struct kvm_memory_slot *memslot; 1088 int r, i, as_id, id; 1089 unsigned long n; 1090 unsigned long *dirty_bitmap; 1091 unsigned long *dirty_bitmap_buffer; 1092 1093 r = -EINVAL; 1094 as_id = log->slot >> 16; 1095 id = (u16)log->slot; 1096 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1097 goto out; 1098 1099 slots = __kvm_memslots(kvm, as_id); 1100 memslot = id_to_memslot(slots, id); 1101 1102 dirty_bitmap = memslot->dirty_bitmap; 1103 r = -ENOENT; 1104 if (!dirty_bitmap) 1105 goto out; 1106 1107 n = kvm_dirty_bitmap_bytes(memslot); 1108 1109 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long); 1110 memset(dirty_bitmap_buffer, 0, n); 1111 1112 spin_lock(&kvm->mmu_lock); 1113 *is_dirty = false; 1114 for (i = 0; i < n / sizeof(long); i++) { 1115 unsigned long mask; 1116 gfn_t offset; 1117 1118 if (!dirty_bitmap[i]) 1119 continue; 1120 1121 *is_dirty = true; 1122 1123 mask = xchg(&dirty_bitmap[i], 0); 1124 dirty_bitmap_buffer[i] = mask; 1125 1126 if (mask) { 1127 offset = i * BITS_PER_LONG; 1128 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1129 offset, mask); 1130 } 1131 } 1132 1133 spin_unlock(&kvm->mmu_lock); 1134 1135 r = -EFAULT; 1136 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 1137 goto out; 1138 1139 r = 0; 1140 out: 1141 return r; 1142 } 1143 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect); 1144 #endif 1145 1146 bool kvm_largepages_enabled(void) 1147 { 1148 return largepages_enabled; 1149 } 1150 1151 void kvm_disable_largepages(void) 1152 { 1153 largepages_enabled = false; 1154 } 1155 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1156 1157 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1158 { 1159 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1160 } 1161 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1162 1163 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 1164 { 1165 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); 1166 } 1167 1168 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1169 { 1170 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1171 1172 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1173 memslot->flags & KVM_MEMSLOT_INVALID) 1174 return false; 1175 1176 return true; 1177 } 1178 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1179 1180 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1181 { 1182 struct vm_area_struct *vma; 1183 unsigned long addr, size; 1184 1185 size = PAGE_SIZE; 1186 1187 addr = gfn_to_hva(kvm, gfn); 1188 if (kvm_is_error_hva(addr)) 1189 return PAGE_SIZE; 1190 1191 down_read(¤t->mm->mmap_sem); 1192 vma = find_vma(current->mm, addr); 1193 if (!vma) 1194 goto out; 1195 1196 size = vma_kernel_pagesize(vma); 1197 1198 out: 1199 up_read(¤t->mm->mmap_sem); 1200 1201 return size; 1202 } 1203 1204 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1205 { 1206 return slot->flags & KVM_MEM_READONLY; 1207 } 1208 1209 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1210 gfn_t *nr_pages, bool write) 1211 { 1212 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1213 return KVM_HVA_ERR_BAD; 1214 1215 if (memslot_is_readonly(slot) && write) 1216 return KVM_HVA_ERR_RO_BAD; 1217 1218 if (nr_pages) 1219 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1220 1221 return __gfn_to_hva_memslot(slot, gfn); 1222 } 1223 1224 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1225 gfn_t *nr_pages) 1226 { 1227 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1228 } 1229 1230 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1231 gfn_t gfn) 1232 { 1233 return gfn_to_hva_many(slot, gfn, NULL); 1234 } 1235 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1236 1237 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1238 { 1239 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1240 } 1241 EXPORT_SYMBOL_GPL(gfn_to_hva); 1242 1243 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 1244 { 1245 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 1246 } 1247 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 1248 1249 /* 1250 * If writable is set to false, the hva returned by this function is only 1251 * allowed to be read. 1252 */ 1253 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1254 gfn_t gfn, bool *writable) 1255 { 1256 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1257 1258 if (!kvm_is_error_hva(hva) && writable) 1259 *writable = !memslot_is_readonly(slot); 1260 1261 return hva; 1262 } 1263 1264 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1265 { 1266 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1267 1268 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1269 } 1270 1271 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 1272 { 1273 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1274 1275 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1276 } 1277 1278 static int get_user_page_nowait(unsigned long start, int write, 1279 struct page **page) 1280 { 1281 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1282 1283 if (write) 1284 flags |= FOLL_WRITE; 1285 1286 return __get_user_pages(current, current->mm, start, 1, flags, page, 1287 NULL, NULL); 1288 } 1289 1290 static inline int check_user_page_hwpoison(unsigned long addr) 1291 { 1292 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1293 1294 rc = __get_user_pages(current, current->mm, addr, 1, 1295 flags, NULL, NULL, NULL); 1296 return rc == -EHWPOISON; 1297 } 1298 1299 /* 1300 * The atomic path to get the writable pfn which will be stored in @pfn, 1301 * true indicates success, otherwise false is returned. 1302 */ 1303 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1304 bool write_fault, bool *writable, kvm_pfn_t *pfn) 1305 { 1306 struct page *page[1]; 1307 int npages; 1308 1309 if (!(async || atomic)) 1310 return false; 1311 1312 /* 1313 * Fast pin a writable pfn only if it is a write fault request 1314 * or the caller allows to map a writable pfn for a read fault 1315 * request. 1316 */ 1317 if (!(write_fault || writable)) 1318 return false; 1319 1320 npages = __get_user_pages_fast(addr, 1, 1, page); 1321 if (npages == 1) { 1322 *pfn = page_to_pfn(page[0]); 1323 1324 if (writable) 1325 *writable = true; 1326 return true; 1327 } 1328 1329 return false; 1330 } 1331 1332 /* 1333 * The slow path to get the pfn of the specified host virtual address, 1334 * 1 indicates success, -errno is returned if error is detected. 1335 */ 1336 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1337 bool *writable, kvm_pfn_t *pfn) 1338 { 1339 struct page *page[1]; 1340 int npages = 0; 1341 1342 might_sleep(); 1343 1344 if (writable) 1345 *writable = write_fault; 1346 1347 if (async) { 1348 down_read(¤t->mm->mmap_sem); 1349 npages = get_user_page_nowait(addr, write_fault, page); 1350 up_read(¤t->mm->mmap_sem); 1351 } else 1352 npages = __get_user_pages_unlocked(current, current->mm, addr, 1, 1353 write_fault, 0, page, 1354 FOLL_TOUCH|FOLL_HWPOISON); 1355 if (npages != 1) 1356 return npages; 1357 1358 /* map read fault as writable if possible */ 1359 if (unlikely(!write_fault) && writable) { 1360 struct page *wpage[1]; 1361 1362 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1363 if (npages == 1) { 1364 *writable = true; 1365 put_page(page[0]); 1366 page[0] = wpage[0]; 1367 } 1368 1369 npages = 1; 1370 } 1371 *pfn = page_to_pfn(page[0]); 1372 return npages; 1373 } 1374 1375 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1376 { 1377 if (unlikely(!(vma->vm_flags & VM_READ))) 1378 return false; 1379 1380 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1381 return false; 1382 1383 return true; 1384 } 1385 1386 /* 1387 * Pin guest page in memory and return its pfn. 1388 * @addr: host virtual address which maps memory to the guest 1389 * @atomic: whether this function can sleep 1390 * @async: whether this function need to wait IO complete if the 1391 * host page is not in the memory 1392 * @write_fault: whether we should get a writable host page 1393 * @writable: whether it allows to map a writable host page for !@write_fault 1394 * 1395 * The function will map a writable host page for these two cases: 1396 * 1): @write_fault = true 1397 * 2): @write_fault = false && @writable, @writable will tell the caller 1398 * whether the mapping is writable. 1399 */ 1400 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1401 bool write_fault, bool *writable) 1402 { 1403 struct vm_area_struct *vma; 1404 kvm_pfn_t pfn = 0; 1405 int npages; 1406 1407 /* we can do it either atomically or asynchronously, not both */ 1408 BUG_ON(atomic && async); 1409 1410 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1411 return pfn; 1412 1413 if (atomic) 1414 return KVM_PFN_ERR_FAULT; 1415 1416 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1417 if (npages == 1) 1418 return pfn; 1419 1420 down_read(¤t->mm->mmap_sem); 1421 if (npages == -EHWPOISON || 1422 (!async && check_user_page_hwpoison(addr))) { 1423 pfn = KVM_PFN_ERR_HWPOISON; 1424 goto exit; 1425 } 1426 1427 vma = find_vma_intersection(current->mm, addr, addr + 1); 1428 1429 if (vma == NULL) 1430 pfn = KVM_PFN_ERR_FAULT; 1431 else if ((vma->vm_flags & VM_PFNMAP)) { 1432 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1433 vma->vm_pgoff; 1434 BUG_ON(!kvm_is_reserved_pfn(pfn)); 1435 } else { 1436 if (async && vma_is_valid(vma, write_fault)) 1437 *async = true; 1438 pfn = KVM_PFN_ERR_FAULT; 1439 } 1440 exit: 1441 up_read(¤t->mm->mmap_sem); 1442 return pfn; 1443 } 1444 1445 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, 1446 bool atomic, bool *async, bool write_fault, 1447 bool *writable) 1448 { 1449 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1450 1451 if (addr == KVM_HVA_ERR_RO_BAD) { 1452 if (writable) 1453 *writable = false; 1454 return KVM_PFN_ERR_RO_FAULT; 1455 } 1456 1457 if (kvm_is_error_hva(addr)) { 1458 if (writable) 1459 *writable = false; 1460 return KVM_PFN_NOSLOT; 1461 } 1462 1463 /* Do not map writable pfn in the readonly memslot. */ 1464 if (writable && memslot_is_readonly(slot)) { 1465 *writable = false; 1466 writable = NULL; 1467 } 1468 1469 return hva_to_pfn(addr, atomic, async, write_fault, 1470 writable); 1471 } 1472 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 1473 1474 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1475 bool *writable) 1476 { 1477 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, 1478 write_fault, writable); 1479 } 1480 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1481 1482 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1483 { 1484 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1485 } 1486 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 1487 1488 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1489 { 1490 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1491 } 1492 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1493 1494 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1495 { 1496 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); 1497 } 1498 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1499 1500 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 1501 { 1502 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1503 } 1504 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 1505 1506 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1507 { 1508 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 1509 } 1510 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1511 1512 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 1513 { 1514 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1515 } 1516 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 1517 1518 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1519 struct page **pages, int nr_pages) 1520 { 1521 unsigned long addr; 1522 gfn_t entry; 1523 1524 addr = gfn_to_hva_many(slot, gfn, &entry); 1525 if (kvm_is_error_hva(addr)) 1526 return -1; 1527 1528 if (entry < nr_pages) 1529 return 0; 1530 1531 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1532 } 1533 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1534 1535 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn) 1536 { 1537 if (is_error_noslot_pfn(pfn)) 1538 return KVM_ERR_PTR_BAD_PAGE; 1539 1540 if (kvm_is_reserved_pfn(pfn)) { 1541 WARN_ON(1); 1542 return KVM_ERR_PTR_BAD_PAGE; 1543 } 1544 1545 return pfn_to_page(pfn); 1546 } 1547 1548 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1549 { 1550 kvm_pfn_t pfn; 1551 1552 pfn = gfn_to_pfn(kvm, gfn); 1553 1554 return kvm_pfn_to_page(pfn); 1555 } 1556 EXPORT_SYMBOL_GPL(gfn_to_page); 1557 1558 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) 1559 { 1560 kvm_pfn_t pfn; 1561 1562 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); 1563 1564 return kvm_pfn_to_page(pfn); 1565 } 1566 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); 1567 1568 void kvm_release_page_clean(struct page *page) 1569 { 1570 WARN_ON(is_error_page(page)); 1571 1572 kvm_release_pfn_clean(page_to_pfn(page)); 1573 } 1574 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1575 1576 void kvm_release_pfn_clean(kvm_pfn_t pfn) 1577 { 1578 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) 1579 put_page(pfn_to_page(pfn)); 1580 } 1581 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1582 1583 void kvm_release_page_dirty(struct page *page) 1584 { 1585 WARN_ON(is_error_page(page)); 1586 1587 kvm_release_pfn_dirty(page_to_pfn(page)); 1588 } 1589 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1590 1591 static void kvm_release_pfn_dirty(kvm_pfn_t pfn) 1592 { 1593 kvm_set_pfn_dirty(pfn); 1594 kvm_release_pfn_clean(pfn); 1595 } 1596 1597 void kvm_set_pfn_dirty(kvm_pfn_t pfn) 1598 { 1599 if (!kvm_is_reserved_pfn(pfn)) { 1600 struct page *page = pfn_to_page(pfn); 1601 1602 if (!PageReserved(page)) 1603 SetPageDirty(page); 1604 } 1605 } 1606 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1607 1608 void kvm_set_pfn_accessed(kvm_pfn_t pfn) 1609 { 1610 if (!kvm_is_reserved_pfn(pfn)) 1611 mark_page_accessed(pfn_to_page(pfn)); 1612 } 1613 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1614 1615 void kvm_get_pfn(kvm_pfn_t pfn) 1616 { 1617 if (!kvm_is_reserved_pfn(pfn)) 1618 get_page(pfn_to_page(pfn)); 1619 } 1620 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1621 1622 static int next_segment(unsigned long len, int offset) 1623 { 1624 if (len > PAGE_SIZE - offset) 1625 return PAGE_SIZE - offset; 1626 else 1627 return len; 1628 } 1629 1630 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 1631 void *data, int offset, int len) 1632 { 1633 int r; 1634 unsigned long addr; 1635 1636 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1637 if (kvm_is_error_hva(addr)) 1638 return -EFAULT; 1639 r = __copy_from_user(data, (void __user *)addr + offset, len); 1640 if (r) 1641 return -EFAULT; 1642 return 0; 1643 } 1644 1645 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1646 int len) 1647 { 1648 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1649 1650 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1651 } 1652 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1653 1654 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 1655 int offset, int len) 1656 { 1657 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1658 1659 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1660 } 1661 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 1662 1663 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1664 { 1665 gfn_t gfn = gpa >> PAGE_SHIFT; 1666 int seg; 1667 int offset = offset_in_page(gpa); 1668 int ret; 1669 1670 while ((seg = next_segment(len, offset)) != 0) { 1671 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1672 if (ret < 0) 1673 return ret; 1674 offset = 0; 1675 len -= seg; 1676 data += seg; 1677 ++gfn; 1678 } 1679 return 0; 1680 } 1681 EXPORT_SYMBOL_GPL(kvm_read_guest); 1682 1683 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 1684 { 1685 gfn_t gfn = gpa >> PAGE_SHIFT; 1686 int seg; 1687 int offset = offset_in_page(gpa); 1688 int ret; 1689 1690 while ((seg = next_segment(len, offset)) != 0) { 1691 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 1692 if (ret < 0) 1693 return ret; 1694 offset = 0; 1695 len -= seg; 1696 data += seg; 1697 ++gfn; 1698 } 1699 return 0; 1700 } 1701 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 1702 1703 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1704 void *data, int offset, unsigned long len) 1705 { 1706 int r; 1707 unsigned long addr; 1708 1709 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1710 if (kvm_is_error_hva(addr)) 1711 return -EFAULT; 1712 pagefault_disable(); 1713 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1714 pagefault_enable(); 1715 if (r) 1716 return -EFAULT; 1717 return 0; 1718 } 1719 1720 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1721 unsigned long len) 1722 { 1723 gfn_t gfn = gpa >> PAGE_SHIFT; 1724 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1725 int offset = offset_in_page(gpa); 1726 1727 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1728 } 1729 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic); 1730 1731 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 1732 void *data, unsigned long len) 1733 { 1734 gfn_t gfn = gpa >> PAGE_SHIFT; 1735 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1736 int offset = offset_in_page(gpa); 1737 1738 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1739 } 1740 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 1741 1742 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn, 1743 const void *data, int offset, int len) 1744 { 1745 int r; 1746 unsigned long addr; 1747 1748 addr = gfn_to_hva_memslot(memslot, gfn); 1749 if (kvm_is_error_hva(addr)) 1750 return -EFAULT; 1751 r = __copy_to_user((void __user *)addr + offset, data, len); 1752 if (r) 1753 return -EFAULT; 1754 mark_page_dirty_in_slot(memslot, gfn); 1755 return 0; 1756 } 1757 1758 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 1759 const void *data, int offset, int len) 1760 { 1761 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1762 1763 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1764 } 1765 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1766 1767 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 1768 const void *data, int offset, int len) 1769 { 1770 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1771 1772 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1773 } 1774 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 1775 1776 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1777 unsigned long len) 1778 { 1779 gfn_t gfn = gpa >> PAGE_SHIFT; 1780 int seg; 1781 int offset = offset_in_page(gpa); 1782 int ret; 1783 1784 while ((seg = next_segment(len, offset)) != 0) { 1785 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1786 if (ret < 0) 1787 return ret; 1788 offset = 0; 1789 len -= seg; 1790 data += seg; 1791 ++gfn; 1792 } 1793 return 0; 1794 } 1795 EXPORT_SYMBOL_GPL(kvm_write_guest); 1796 1797 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1798 unsigned long len) 1799 { 1800 gfn_t gfn = gpa >> PAGE_SHIFT; 1801 int seg; 1802 int offset = offset_in_page(gpa); 1803 int ret; 1804 1805 while ((seg = next_segment(len, offset)) != 0) { 1806 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 1807 if (ret < 0) 1808 return ret; 1809 offset = 0; 1810 len -= seg; 1811 data += seg; 1812 ++gfn; 1813 } 1814 return 0; 1815 } 1816 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 1817 1818 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1819 gpa_t gpa, unsigned long len) 1820 { 1821 struct kvm_memslots *slots = kvm_memslots(kvm); 1822 int offset = offset_in_page(gpa); 1823 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1824 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1825 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1826 gfn_t nr_pages_avail; 1827 1828 ghc->gpa = gpa; 1829 ghc->generation = slots->generation; 1830 ghc->len = len; 1831 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1832 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL); 1833 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) { 1834 ghc->hva += offset; 1835 } else { 1836 /* 1837 * If the requested region crosses two memslots, we still 1838 * verify that the entire region is valid here. 1839 */ 1840 while (start_gfn <= end_gfn) { 1841 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1842 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1843 &nr_pages_avail); 1844 if (kvm_is_error_hva(ghc->hva)) 1845 return -EFAULT; 1846 start_gfn += nr_pages_avail; 1847 } 1848 /* Use the slow path for cross page reads and writes. */ 1849 ghc->memslot = NULL; 1850 } 1851 return 0; 1852 } 1853 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1854 1855 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1856 void *data, unsigned long len) 1857 { 1858 struct kvm_memslots *slots = kvm_memslots(kvm); 1859 int r; 1860 1861 BUG_ON(len > ghc->len); 1862 1863 if (slots->generation != ghc->generation) 1864 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1865 1866 if (unlikely(!ghc->memslot)) 1867 return kvm_write_guest(kvm, ghc->gpa, data, len); 1868 1869 if (kvm_is_error_hva(ghc->hva)) 1870 return -EFAULT; 1871 1872 r = __copy_to_user((void __user *)ghc->hva, data, len); 1873 if (r) 1874 return -EFAULT; 1875 mark_page_dirty_in_slot(ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1876 1877 return 0; 1878 } 1879 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1880 1881 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1882 void *data, unsigned long len) 1883 { 1884 struct kvm_memslots *slots = kvm_memslots(kvm); 1885 int r; 1886 1887 BUG_ON(len > ghc->len); 1888 1889 if (slots->generation != ghc->generation) 1890 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1891 1892 if (unlikely(!ghc->memslot)) 1893 return kvm_read_guest(kvm, ghc->gpa, data, len); 1894 1895 if (kvm_is_error_hva(ghc->hva)) 1896 return -EFAULT; 1897 1898 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1899 if (r) 1900 return -EFAULT; 1901 1902 return 0; 1903 } 1904 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1905 1906 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1907 { 1908 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 1909 1910 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 1911 } 1912 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1913 1914 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1915 { 1916 gfn_t gfn = gpa >> PAGE_SHIFT; 1917 int seg; 1918 int offset = offset_in_page(gpa); 1919 int ret; 1920 1921 while ((seg = next_segment(len, offset)) != 0) { 1922 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1923 if (ret < 0) 1924 return ret; 1925 offset = 0; 1926 len -= seg; 1927 ++gfn; 1928 } 1929 return 0; 1930 } 1931 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1932 1933 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, 1934 gfn_t gfn) 1935 { 1936 if (memslot && memslot->dirty_bitmap) { 1937 unsigned long rel_gfn = gfn - memslot->base_gfn; 1938 1939 set_bit_le(rel_gfn, memslot->dirty_bitmap); 1940 } 1941 } 1942 1943 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 1944 { 1945 struct kvm_memory_slot *memslot; 1946 1947 memslot = gfn_to_memslot(kvm, gfn); 1948 mark_page_dirty_in_slot(memslot, gfn); 1949 } 1950 EXPORT_SYMBOL_GPL(mark_page_dirty); 1951 1952 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 1953 { 1954 struct kvm_memory_slot *memslot; 1955 1956 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1957 mark_page_dirty_in_slot(memslot, gfn); 1958 } 1959 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 1960 1961 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) 1962 { 1963 unsigned int old, val, grow; 1964 1965 old = val = vcpu->halt_poll_ns; 1966 grow = READ_ONCE(halt_poll_ns_grow); 1967 /* 10us base */ 1968 if (val == 0 && grow) 1969 val = 10000; 1970 else 1971 val *= grow; 1972 1973 if (val > halt_poll_ns) 1974 val = halt_poll_ns; 1975 1976 vcpu->halt_poll_ns = val; 1977 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); 1978 } 1979 1980 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu) 1981 { 1982 unsigned int old, val, shrink; 1983 1984 old = val = vcpu->halt_poll_ns; 1985 shrink = READ_ONCE(halt_poll_ns_shrink); 1986 if (shrink == 0) 1987 val = 0; 1988 else 1989 val /= shrink; 1990 1991 vcpu->halt_poll_ns = val; 1992 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); 1993 } 1994 1995 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 1996 { 1997 if (kvm_arch_vcpu_runnable(vcpu)) { 1998 kvm_make_request(KVM_REQ_UNHALT, vcpu); 1999 return -EINTR; 2000 } 2001 if (kvm_cpu_has_pending_timer(vcpu)) 2002 return -EINTR; 2003 if (signal_pending(current)) 2004 return -EINTR; 2005 2006 return 0; 2007 } 2008 2009 /* 2010 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 2011 */ 2012 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 2013 { 2014 ktime_t start, cur; 2015 DECLARE_SWAITQUEUE(wait); 2016 bool waited = false; 2017 u64 block_ns; 2018 2019 start = cur = ktime_get(); 2020 if (vcpu->halt_poll_ns) { 2021 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); 2022 2023 ++vcpu->stat.halt_attempted_poll; 2024 do { 2025 /* 2026 * This sets KVM_REQ_UNHALT if an interrupt 2027 * arrives. 2028 */ 2029 if (kvm_vcpu_check_block(vcpu) < 0) { 2030 ++vcpu->stat.halt_successful_poll; 2031 if (!vcpu_valid_wakeup(vcpu)) 2032 ++vcpu->stat.halt_poll_invalid; 2033 goto out; 2034 } 2035 cur = ktime_get(); 2036 } while (single_task_running() && ktime_before(cur, stop)); 2037 } 2038 2039 kvm_arch_vcpu_blocking(vcpu); 2040 2041 for (;;) { 2042 prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 2043 2044 if (kvm_vcpu_check_block(vcpu) < 0) 2045 break; 2046 2047 waited = true; 2048 schedule(); 2049 } 2050 2051 finish_swait(&vcpu->wq, &wait); 2052 cur = ktime_get(); 2053 2054 kvm_arch_vcpu_unblocking(vcpu); 2055 out: 2056 block_ns = ktime_to_ns(cur) - ktime_to_ns(start); 2057 2058 if (!vcpu_valid_wakeup(vcpu)) 2059 shrink_halt_poll_ns(vcpu); 2060 else if (halt_poll_ns) { 2061 if (block_ns <= vcpu->halt_poll_ns) 2062 ; 2063 /* we had a long block, shrink polling */ 2064 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns) 2065 shrink_halt_poll_ns(vcpu); 2066 /* we had a short halt and our poll time is too small */ 2067 else if (vcpu->halt_poll_ns < halt_poll_ns && 2068 block_ns < halt_poll_ns) 2069 grow_halt_poll_ns(vcpu); 2070 } else 2071 vcpu->halt_poll_ns = 0; 2072 2073 trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu)); 2074 kvm_arch_vcpu_block_finish(vcpu); 2075 } 2076 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 2077 2078 #ifndef CONFIG_S390 2079 void kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) 2080 { 2081 struct swait_queue_head *wqp; 2082 2083 wqp = kvm_arch_vcpu_wq(vcpu); 2084 if (swait_active(wqp)) { 2085 swake_up(wqp); 2086 ++vcpu->stat.halt_wakeup; 2087 } 2088 2089 } 2090 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up); 2091 2092 /* 2093 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 2094 */ 2095 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 2096 { 2097 int me; 2098 int cpu = vcpu->cpu; 2099 2100 kvm_vcpu_wake_up(vcpu); 2101 me = get_cpu(); 2102 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 2103 if (kvm_arch_vcpu_should_kick(vcpu)) 2104 smp_send_reschedule(cpu); 2105 put_cpu(); 2106 } 2107 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 2108 #endif /* !CONFIG_S390 */ 2109 2110 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 2111 { 2112 struct pid *pid; 2113 struct task_struct *task = NULL; 2114 int ret = 0; 2115 2116 rcu_read_lock(); 2117 pid = rcu_dereference(target->pid); 2118 if (pid) 2119 task = get_pid_task(pid, PIDTYPE_PID); 2120 rcu_read_unlock(); 2121 if (!task) 2122 return ret; 2123 ret = yield_to(task, 1); 2124 put_task_struct(task); 2125 2126 return ret; 2127 } 2128 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 2129 2130 /* 2131 * Helper that checks whether a VCPU is eligible for directed yield. 2132 * Most eligible candidate to yield is decided by following heuristics: 2133 * 2134 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 2135 * (preempted lock holder), indicated by @in_spin_loop. 2136 * Set at the beiginning and cleared at the end of interception/PLE handler. 2137 * 2138 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 2139 * chance last time (mostly it has become eligible now since we have probably 2140 * yielded to lockholder in last iteration. This is done by toggling 2141 * @dy_eligible each time a VCPU checked for eligibility.) 2142 * 2143 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 2144 * to preempted lock-holder could result in wrong VCPU selection and CPU 2145 * burning. Giving priority for a potential lock-holder increases lock 2146 * progress. 2147 * 2148 * Since algorithm is based on heuristics, accessing another VCPU data without 2149 * locking does not harm. It may result in trying to yield to same VCPU, fail 2150 * and continue with next VCPU and so on. 2151 */ 2152 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 2153 { 2154 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2155 bool eligible; 2156 2157 eligible = !vcpu->spin_loop.in_spin_loop || 2158 vcpu->spin_loop.dy_eligible; 2159 2160 if (vcpu->spin_loop.in_spin_loop) 2161 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 2162 2163 return eligible; 2164 #else 2165 return true; 2166 #endif 2167 } 2168 2169 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 2170 { 2171 struct kvm *kvm = me->kvm; 2172 struct kvm_vcpu *vcpu; 2173 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 2174 int yielded = 0; 2175 int try = 3; 2176 int pass; 2177 int i; 2178 2179 kvm_vcpu_set_in_spin_loop(me, true); 2180 /* 2181 * We boost the priority of a VCPU that is runnable but not 2182 * currently running, because it got preempted by something 2183 * else and called schedule in __vcpu_run. Hopefully that 2184 * VCPU is holding the lock that we need and will release it. 2185 * We approximate round-robin by starting at the last boosted VCPU. 2186 */ 2187 for (pass = 0; pass < 2 && !yielded && try; pass++) { 2188 kvm_for_each_vcpu(i, vcpu, kvm) { 2189 if (!pass && i <= last_boosted_vcpu) { 2190 i = last_boosted_vcpu; 2191 continue; 2192 } else if (pass && i > last_boosted_vcpu) 2193 break; 2194 if (!ACCESS_ONCE(vcpu->preempted)) 2195 continue; 2196 if (vcpu == me) 2197 continue; 2198 if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 2199 continue; 2200 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 2201 continue; 2202 2203 yielded = kvm_vcpu_yield_to(vcpu); 2204 if (yielded > 0) { 2205 kvm->last_boosted_vcpu = i; 2206 break; 2207 } else if (yielded < 0) { 2208 try--; 2209 if (!try) 2210 break; 2211 } 2212 } 2213 } 2214 kvm_vcpu_set_in_spin_loop(me, false); 2215 2216 /* Ensure vcpu is not eligible during next spinloop */ 2217 kvm_vcpu_set_dy_eligible(me, false); 2218 } 2219 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 2220 2221 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2222 { 2223 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 2224 struct page *page; 2225 2226 if (vmf->pgoff == 0) 2227 page = virt_to_page(vcpu->run); 2228 #ifdef CONFIG_X86 2229 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 2230 page = virt_to_page(vcpu->arch.pio_data); 2231 #endif 2232 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2233 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 2234 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 2235 #endif 2236 else 2237 return kvm_arch_vcpu_fault(vcpu, vmf); 2238 get_page(page); 2239 vmf->page = page; 2240 return 0; 2241 } 2242 2243 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 2244 .fault = kvm_vcpu_fault, 2245 }; 2246 2247 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 2248 { 2249 vma->vm_ops = &kvm_vcpu_vm_ops; 2250 return 0; 2251 } 2252 2253 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 2254 { 2255 struct kvm_vcpu *vcpu = filp->private_data; 2256 2257 kvm_put_kvm(vcpu->kvm); 2258 return 0; 2259 } 2260 2261 static struct file_operations kvm_vcpu_fops = { 2262 .release = kvm_vcpu_release, 2263 .unlocked_ioctl = kvm_vcpu_ioctl, 2264 #ifdef CONFIG_KVM_COMPAT 2265 .compat_ioctl = kvm_vcpu_compat_ioctl, 2266 #endif 2267 .mmap = kvm_vcpu_mmap, 2268 .llseek = noop_llseek, 2269 }; 2270 2271 /* 2272 * Allocates an inode for the vcpu. 2273 */ 2274 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 2275 { 2276 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 2277 } 2278 2279 /* 2280 * Creates some virtual cpus. Good luck creating more than one. 2281 */ 2282 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 2283 { 2284 int r; 2285 struct kvm_vcpu *vcpu; 2286 2287 if (id >= KVM_MAX_VCPU_ID) 2288 return -EINVAL; 2289 2290 vcpu = kvm_arch_vcpu_create(kvm, id); 2291 if (IS_ERR(vcpu)) 2292 return PTR_ERR(vcpu); 2293 2294 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 2295 2296 r = kvm_arch_vcpu_setup(vcpu); 2297 if (r) 2298 goto vcpu_destroy; 2299 2300 mutex_lock(&kvm->lock); 2301 if (!kvm_vcpu_compatible(vcpu)) { 2302 r = -EINVAL; 2303 goto unlock_vcpu_destroy; 2304 } 2305 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 2306 r = -EINVAL; 2307 goto unlock_vcpu_destroy; 2308 } 2309 if (kvm_get_vcpu_by_id(kvm, id)) { 2310 r = -EEXIST; 2311 goto unlock_vcpu_destroy; 2312 } 2313 2314 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 2315 2316 /* Now it's all set up, let userspace reach it */ 2317 kvm_get_kvm(kvm); 2318 r = create_vcpu_fd(vcpu); 2319 if (r < 0) { 2320 kvm_put_kvm(kvm); 2321 goto unlock_vcpu_destroy; 2322 } 2323 2324 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 2325 2326 /* 2327 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus 2328 * before kvm->online_vcpu's incremented value. 2329 */ 2330 smp_wmb(); 2331 atomic_inc(&kvm->online_vcpus); 2332 2333 mutex_unlock(&kvm->lock); 2334 kvm_arch_vcpu_postcreate(vcpu); 2335 return r; 2336 2337 unlock_vcpu_destroy: 2338 mutex_unlock(&kvm->lock); 2339 vcpu_destroy: 2340 kvm_arch_vcpu_destroy(vcpu); 2341 return r; 2342 } 2343 2344 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 2345 { 2346 if (sigset) { 2347 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2348 vcpu->sigset_active = 1; 2349 vcpu->sigset = *sigset; 2350 } else 2351 vcpu->sigset_active = 0; 2352 return 0; 2353 } 2354 2355 static long kvm_vcpu_ioctl(struct file *filp, 2356 unsigned int ioctl, unsigned long arg) 2357 { 2358 struct kvm_vcpu *vcpu = filp->private_data; 2359 void __user *argp = (void __user *)arg; 2360 int r; 2361 struct kvm_fpu *fpu = NULL; 2362 struct kvm_sregs *kvm_sregs = NULL; 2363 2364 if (vcpu->kvm->mm != current->mm) 2365 return -EIO; 2366 2367 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 2368 return -EINVAL; 2369 2370 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS) 2371 /* 2372 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 2373 * so vcpu_load() would break it. 2374 */ 2375 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT) 2376 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2377 #endif 2378 2379 2380 r = vcpu_load(vcpu); 2381 if (r) 2382 return r; 2383 switch (ioctl) { 2384 case KVM_RUN: 2385 r = -EINVAL; 2386 if (arg) 2387 goto out; 2388 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 2389 /* The thread running this VCPU changed. */ 2390 struct pid *oldpid = vcpu->pid; 2391 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 2392 2393 rcu_assign_pointer(vcpu->pid, newpid); 2394 if (oldpid) 2395 synchronize_rcu(); 2396 put_pid(oldpid); 2397 } 2398 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 2399 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 2400 break; 2401 case KVM_GET_REGS: { 2402 struct kvm_regs *kvm_regs; 2403 2404 r = -ENOMEM; 2405 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2406 if (!kvm_regs) 2407 goto out; 2408 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2409 if (r) 2410 goto out_free1; 2411 r = -EFAULT; 2412 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2413 goto out_free1; 2414 r = 0; 2415 out_free1: 2416 kfree(kvm_regs); 2417 break; 2418 } 2419 case KVM_SET_REGS: { 2420 struct kvm_regs *kvm_regs; 2421 2422 r = -ENOMEM; 2423 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2424 if (IS_ERR(kvm_regs)) { 2425 r = PTR_ERR(kvm_regs); 2426 goto out; 2427 } 2428 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2429 kfree(kvm_regs); 2430 break; 2431 } 2432 case KVM_GET_SREGS: { 2433 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2434 r = -ENOMEM; 2435 if (!kvm_sregs) 2436 goto out; 2437 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2438 if (r) 2439 goto out; 2440 r = -EFAULT; 2441 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2442 goto out; 2443 r = 0; 2444 break; 2445 } 2446 case KVM_SET_SREGS: { 2447 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2448 if (IS_ERR(kvm_sregs)) { 2449 r = PTR_ERR(kvm_sregs); 2450 kvm_sregs = NULL; 2451 goto out; 2452 } 2453 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2454 break; 2455 } 2456 case KVM_GET_MP_STATE: { 2457 struct kvm_mp_state mp_state; 2458 2459 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2460 if (r) 2461 goto out; 2462 r = -EFAULT; 2463 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 2464 goto out; 2465 r = 0; 2466 break; 2467 } 2468 case KVM_SET_MP_STATE: { 2469 struct kvm_mp_state mp_state; 2470 2471 r = -EFAULT; 2472 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 2473 goto out; 2474 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2475 break; 2476 } 2477 case KVM_TRANSLATE: { 2478 struct kvm_translation tr; 2479 2480 r = -EFAULT; 2481 if (copy_from_user(&tr, argp, sizeof(tr))) 2482 goto out; 2483 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2484 if (r) 2485 goto out; 2486 r = -EFAULT; 2487 if (copy_to_user(argp, &tr, sizeof(tr))) 2488 goto out; 2489 r = 0; 2490 break; 2491 } 2492 case KVM_SET_GUEST_DEBUG: { 2493 struct kvm_guest_debug dbg; 2494 2495 r = -EFAULT; 2496 if (copy_from_user(&dbg, argp, sizeof(dbg))) 2497 goto out; 2498 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2499 break; 2500 } 2501 case KVM_SET_SIGNAL_MASK: { 2502 struct kvm_signal_mask __user *sigmask_arg = argp; 2503 struct kvm_signal_mask kvm_sigmask; 2504 sigset_t sigset, *p; 2505 2506 p = NULL; 2507 if (argp) { 2508 r = -EFAULT; 2509 if (copy_from_user(&kvm_sigmask, argp, 2510 sizeof(kvm_sigmask))) 2511 goto out; 2512 r = -EINVAL; 2513 if (kvm_sigmask.len != sizeof(sigset)) 2514 goto out; 2515 r = -EFAULT; 2516 if (copy_from_user(&sigset, sigmask_arg->sigset, 2517 sizeof(sigset))) 2518 goto out; 2519 p = &sigset; 2520 } 2521 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2522 break; 2523 } 2524 case KVM_GET_FPU: { 2525 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2526 r = -ENOMEM; 2527 if (!fpu) 2528 goto out; 2529 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2530 if (r) 2531 goto out; 2532 r = -EFAULT; 2533 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2534 goto out; 2535 r = 0; 2536 break; 2537 } 2538 case KVM_SET_FPU: { 2539 fpu = memdup_user(argp, sizeof(*fpu)); 2540 if (IS_ERR(fpu)) { 2541 r = PTR_ERR(fpu); 2542 fpu = NULL; 2543 goto out; 2544 } 2545 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2546 break; 2547 } 2548 default: 2549 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2550 } 2551 out: 2552 vcpu_put(vcpu); 2553 kfree(fpu); 2554 kfree(kvm_sregs); 2555 return r; 2556 } 2557 2558 #ifdef CONFIG_KVM_COMPAT 2559 static long kvm_vcpu_compat_ioctl(struct file *filp, 2560 unsigned int ioctl, unsigned long arg) 2561 { 2562 struct kvm_vcpu *vcpu = filp->private_data; 2563 void __user *argp = compat_ptr(arg); 2564 int r; 2565 2566 if (vcpu->kvm->mm != current->mm) 2567 return -EIO; 2568 2569 switch (ioctl) { 2570 case KVM_SET_SIGNAL_MASK: { 2571 struct kvm_signal_mask __user *sigmask_arg = argp; 2572 struct kvm_signal_mask kvm_sigmask; 2573 compat_sigset_t csigset; 2574 sigset_t sigset; 2575 2576 if (argp) { 2577 r = -EFAULT; 2578 if (copy_from_user(&kvm_sigmask, argp, 2579 sizeof(kvm_sigmask))) 2580 goto out; 2581 r = -EINVAL; 2582 if (kvm_sigmask.len != sizeof(csigset)) 2583 goto out; 2584 r = -EFAULT; 2585 if (copy_from_user(&csigset, sigmask_arg->sigset, 2586 sizeof(csigset))) 2587 goto out; 2588 sigset_from_compat(&sigset, &csigset); 2589 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2590 } else 2591 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2592 break; 2593 } 2594 default: 2595 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2596 } 2597 2598 out: 2599 return r; 2600 } 2601 #endif 2602 2603 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2604 int (*accessor)(struct kvm_device *dev, 2605 struct kvm_device_attr *attr), 2606 unsigned long arg) 2607 { 2608 struct kvm_device_attr attr; 2609 2610 if (!accessor) 2611 return -EPERM; 2612 2613 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2614 return -EFAULT; 2615 2616 return accessor(dev, &attr); 2617 } 2618 2619 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2620 unsigned long arg) 2621 { 2622 struct kvm_device *dev = filp->private_data; 2623 2624 switch (ioctl) { 2625 case KVM_SET_DEVICE_ATTR: 2626 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2627 case KVM_GET_DEVICE_ATTR: 2628 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2629 case KVM_HAS_DEVICE_ATTR: 2630 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2631 default: 2632 if (dev->ops->ioctl) 2633 return dev->ops->ioctl(dev, ioctl, arg); 2634 2635 return -ENOTTY; 2636 } 2637 } 2638 2639 static int kvm_device_release(struct inode *inode, struct file *filp) 2640 { 2641 struct kvm_device *dev = filp->private_data; 2642 struct kvm *kvm = dev->kvm; 2643 2644 kvm_put_kvm(kvm); 2645 return 0; 2646 } 2647 2648 static const struct file_operations kvm_device_fops = { 2649 .unlocked_ioctl = kvm_device_ioctl, 2650 #ifdef CONFIG_KVM_COMPAT 2651 .compat_ioctl = kvm_device_ioctl, 2652 #endif 2653 .release = kvm_device_release, 2654 }; 2655 2656 struct kvm_device *kvm_device_from_filp(struct file *filp) 2657 { 2658 if (filp->f_op != &kvm_device_fops) 2659 return NULL; 2660 2661 return filp->private_data; 2662 } 2663 2664 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 2665 #ifdef CONFIG_KVM_MPIC 2666 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 2667 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 2668 #endif 2669 2670 #ifdef CONFIG_KVM_XICS 2671 [KVM_DEV_TYPE_XICS] = &kvm_xics_ops, 2672 #endif 2673 }; 2674 2675 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type) 2676 { 2677 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 2678 return -ENOSPC; 2679 2680 if (kvm_device_ops_table[type] != NULL) 2681 return -EEXIST; 2682 2683 kvm_device_ops_table[type] = ops; 2684 return 0; 2685 } 2686 2687 void kvm_unregister_device_ops(u32 type) 2688 { 2689 if (kvm_device_ops_table[type] != NULL) 2690 kvm_device_ops_table[type] = NULL; 2691 } 2692 2693 static int kvm_ioctl_create_device(struct kvm *kvm, 2694 struct kvm_create_device *cd) 2695 { 2696 struct kvm_device_ops *ops = NULL; 2697 struct kvm_device *dev; 2698 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2699 int ret; 2700 2701 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 2702 return -ENODEV; 2703 2704 ops = kvm_device_ops_table[cd->type]; 2705 if (ops == NULL) 2706 return -ENODEV; 2707 2708 if (test) 2709 return 0; 2710 2711 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2712 if (!dev) 2713 return -ENOMEM; 2714 2715 dev->ops = ops; 2716 dev->kvm = kvm; 2717 2718 ret = ops->create(dev, cd->type); 2719 if (ret < 0) { 2720 kfree(dev); 2721 return ret; 2722 } 2723 2724 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2725 if (ret < 0) { 2726 ops->destroy(dev); 2727 return ret; 2728 } 2729 2730 list_add(&dev->vm_node, &kvm->devices); 2731 kvm_get_kvm(kvm); 2732 cd->fd = ret; 2733 return 0; 2734 } 2735 2736 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 2737 { 2738 switch (arg) { 2739 case KVM_CAP_USER_MEMORY: 2740 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2741 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2742 case KVM_CAP_INTERNAL_ERROR_DATA: 2743 #ifdef CONFIG_HAVE_KVM_MSI 2744 case KVM_CAP_SIGNAL_MSI: 2745 #endif 2746 #ifdef CONFIG_HAVE_KVM_IRQFD 2747 case KVM_CAP_IRQFD: 2748 case KVM_CAP_IRQFD_RESAMPLE: 2749 #endif 2750 case KVM_CAP_IOEVENTFD_ANY_LENGTH: 2751 case KVM_CAP_CHECK_EXTENSION_VM: 2752 return 1; 2753 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2754 case KVM_CAP_IRQ_ROUTING: 2755 return KVM_MAX_IRQ_ROUTES; 2756 #endif 2757 #if KVM_ADDRESS_SPACE_NUM > 1 2758 case KVM_CAP_MULTI_ADDRESS_SPACE: 2759 return KVM_ADDRESS_SPACE_NUM; 2760 #endif 2761 case KVM_CAP_MAX_VCPU_ID: 2762 return KVM_MAX_VCPU_ID; 2763 default: 2764 break; 2765 } 2766 return kvm_vm_ioctl_check_extension(kvm, arg); 2767 } 2768 2769 static long kvm_vm_ioctl(struct file *filp, 2770 unsigned int ioctl, unsigned long arg) 2771 { 2772 struct kvm *kvm = filp->private_data; 2773 void __user *argp = (void __user *)arg; 2774 int r; 2775 2776 if (kvm->mm != current->mm) 2777 return -EIO; 2778 switch (ioctl) { 2779 case KVM_CREATE_VCPU: 2780 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2781 break; 2782 case KVM_SET_USER_MEMORY_REGION: { 2783 struct kvm_userspace_memory_region kvm_userspace_mem; 2784 2785 r = -EFAULT; 2786 if (copy_from_user(&kvm_userspace_mem, argp, 2787 sizeof(kvm_userspace_mem))) 2788 goto out; 2789 2790 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2791 break; 2792 } 2793 case KVM_GET_DIRTY_LOG: { 2794 struct kvm_dirty_log log; 2795 2796 r = -EFAULT; 2797 if (copy_from_user(&log, argp, sizeof(log))) 2798 goto out; 2799 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2800 break; 2801 } 2802 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2803 case KVM_REGISTER_COALESCED_MMIO: { 2804 struct kvm_coalesced_mmio_zone zone; 2805 2806 r = -EFAULT; 2807 if (copy_from_user(&zone, argp, sizeof(zone))) 2808 goto out; 2809 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2810 break; 2811 } 2812 case KVM_UNREGISTER_COALESCED_MMIO: { 2813 struct kvm_coalesced_mmio_zone zone; 2814 2815 r = -EFAULT; 2816 if (copy_from_user(&zone, argp, sizeof(zone))) 2817 goto out; 2818 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2819 break; 2820 } 2821 #endif 2822 case KVM_IRQFD: { 2823 struct kvm_irqfd data; 2824 2825 r = -EFAULT; 2826 if (copy_from_user(&data, argp, sizeof(data))) 2827 goto out; 2828 r = kvm_irqfd(kvm, &data); 2829 break; 2830 } 2831 case KVM_IOEVENTFD: { 2832 struct kvm_ioeventfd data; 2833 2834 r = -EFAULT; 2835 if (copy_from_user(&data, argp, sizeof(data))) 2836 goto out; 2837 r = kvm_ioeventfd(kvm, &data); 2838 break; 2839 } 2840 #ifdef CONFIG_HAVE_KVM_MSI 2841 case KVM_SIGNAL_MSI: { 2842 struct kvm_msi msi; 2843 2844 r = -EFAULT; 2845 if (copy_from_user(&msi, argp, sizeof(msi))) 2846 goto out; 2847 r = kvm_send_userspace_msi(kvm, &msi); 2848 break; 2849 } 2850 #endif 2851 #ifdef __KVM_HAVE_IRQ_LINE 2852 case KVM_IRQ_LINE_STATUS: 2853 case KVM_IRQ_LINE: { 2854 struct kvm_irq_level irq_event; 2855 2856 r = -EFAULT; 2857 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 2858 goto out; 2859 2860 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 2861 ioctl == KVM_IRQ_LINE_STATUS); 2862 if (r) 2863 goto out; 2864 2865 r = -EFAULT; 2866 if (ioctl == KVM_IRQ_LINE_STATUS) { 2867 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 2868 goto out; 2869 } 2870 2871 r = 0; 2872 break; 2873 } 2874 #endif 2875 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2876 case KVM_SET_GSI_ROUTING: { 2877 struct kvm_irq_routing routing; 2878 struct kvm_irq_routing __user *urouting; 2879 struct kvm_irq_routing_entry *entries; 2880 2881 r = -EFAULT; 2882 if (copy_from_user(&routing, argp, sizeof(routing))) 2883 goto out; 2884 r = -EINVAL; 2885 if (routing.nr >= KVM_MAX_IRQ_ROUTES) 2886 goto out; 2887 if (routing.flags) 2888 goto out; 2889 r = -ENOMEM; 2890 entries = vmalloc(routing.nr * sizeof(*entries)); 2891 if (!entries) 2892 goto out; 2893 r = -EFAULT; 2894 urouting = argp; 2895 if (copy_from_user(entries, urouting->entries, 2896 routing.nr * sizeof(*entries))) 2897 goto out_free_irq_routing; 2898 r = kvm_set_irq_routing(kvm, entries, routing.nr, 2899 routing.flags); 2900 out_free_irq_routing: 2901 vfree(entries); 2902 break; 2903 } 2904 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 2905 case KVM_CREATE_DEVICE: { 2906 struct kvm_create_device cd; 2907 2908 r = -EFAULT; 2909 if (copy_from_user(&cd, argp, sizeof(cd))) 2910 goto out; 2911 2912 r = kvm_ioctl_create_device(kvm, &cd); 2913 if (r) 2914 goto out; 2915 2916 r = -EFAULT; 2917 if (copy_to_user(argp, &cd, sizeof(cd))) 2918 goto out; 2919 2920 r = 0; 2921 break; 2922 } 2923 case KVM_CHECK_EXTENSION: 2924 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 2925 break; 2926 default: 2927 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2928 } 2929 out: 2930 return r; 2931 } 2932 2933 #ifdef CONFIG_KVM_COMPAT 2934 struct compat_kvm_dirty_log { 2935 __u32 slot; 2936 __u32 padding1; 2937 union { 2938 compat_uptr_t dirty_bitmap; /* one bit per page */ 2939 __u64 padding2; 2940 }; 2941 }; 2942 2943 static long kvm_vm_compat_ioctl(struct file *filp, 2944 unsigned int ioctl, unsigned long arg) 2945 { 2946 struct kvm *kvm = filp->private_data; 2947 int r; 2948 2949 if (kvm->mm != current->mm) 2950 return -EIO; 2951 switch (ioctl) { 2952 case KVM_GET_DIRTY_LOG: { 2953 struct compat_kvm_dirty_log compat_log; 2954 struct kvm_dirty_log log; 2955 2956 r = -EFAULT; 2957 if (copy_from_user(&compat_log, (void __user *)arg, 2958 sizeof(compat_log))) 2959 goto out; 2960 log.slot = compat_log.slot; 2961 log.padding1 = compat_log.padding1; 2962 log.padding2 = compat_log.padding2; 2963 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 2964 2965 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2966 break; 2967 } 2968 default: 2969 r = kvm_vm_ioctl(filp, ioctl, arg); 2970 } 2971 2972 out: 2973 return r; 2974 } 2975 #endif 2976 2977 static struct file_operations kvm_vm_fops = { 2978 .release = kvm_vm_release, 2979 .unlocked_ioctl = kvm_vm_ioctl, 2980 #ifdef CONFIG_KVM_COMPAT 2981 .compat_ioctl = kvm_vm_compat_ioctl, 2982 #endif 2983 .llseek = noop_llseek, 2984 }; 2985 2986 static int kvm_dev_ioctl_create_vm(unsigned long type) 2987 { 2988 int r; 2989 struct kvm *kvm; 2990 2991 kvm = kvm_create_vm(type); 2992 if (IS_ERR(kvm)) 2993 return PTR_ERR(kvm); 2994 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2995 r = kvm_coalesced_mmio_init(kvm); 2996 if (r < 0) { 2997 kvm_put_kvm(kvm); 2998 return r; 2999 } 3000 #endif 3001 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC); 3002 if (r < 0) 3003 kvm_put_kvm(kvm); 3004 3005 return r; 3006 } 3007 3008 static long kvm_dev_ioctl(struct file *filp, 3009 unsigned int ioctl, unsigned long arg) 3010 { 3011 long r = -EINVAL; 3012 3013 switch (ioctl) { 3014 case KVM_GET_API_VERSION: 3015 if (arg) 3016 goto out; 3017 r = KVM_API_VERSION; 3018 break; 3019 case KVM_CREATE_VM: 3020 r = kvm_dev_ioctl_create_vm(arg); 3021 break; 3022 case KVM_CHECK_EXTENSION: 3023 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 3024 break; 3025 case KVM_GET_VCPU_MMAP_SIZE: 3026 if (arg) 3027 goto out; 3028 r = PAGE_SIZE; /* struct kvm_run */ 3029 #ifdef CONFIG_X86 3030 r += PAGE_SIZE; /* pio data page */ 3031 #endif 3032 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 3033 r += PAGE_SIZE; /* coalesced mmio ring page */ 3034 #endif 3035 break; 3036 case KVM_TRACE_ENABLE: 3037 case KVM_TRACE_PAUSE: 3038 case KVM_TRACE_DISABLE: 3039 r = -EOPNOTSUPP; 3040 break; 3041 default: 3042 return kvm_arch_dev_ioctl(filp, ioctl, arg); 3043 } 3044 out: 3045 return r; 3046 } 3047 3048 static struct file_operations kvm_chardev_ops = { 3049 .unlocked_ioctl = kvm_dev_ioctl, 3050 .compat_ioctl = kvm_dev_ioctl, 3051 .llseek = noop_llseek, 3052 }; 3053 3054 static struct miscdevice kvm_dev = { 3055 KVM_MINOR, 3056 "kvm", 3057 &kvm_chardev_ops, 3058 }; 3059 3060 static void hardware_enable_nolock(void *junk) 3061 { 3062 int cpu = raw_smp_processor_id(); 3063 int r; 3064 3065 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3066 return; 3067 3068 cpumask_set_cpu(cpu, cpus_hardware_enabled); 3069 3070 r = kvm_arch_hardware_enable(); 3071 3072 if (r) { 3073 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3074 atomic_inc(&hardware_enable_failed); 3075 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu); 3076 } 3077 } 3078 3079 static void hardware_enable(void) 3080 { 3081 raw_spin_lock(&kvm_count_lock); 3082 if (kvm_usage_count) 3083 hardware_enable_nolock(NULL); 3084 raw_spin_unlock(&kvm_count_lock); 3085 } 3086 3087 static void hardware_disable_nolock(void *junk) 3088 { 3089 int cpu = raw_smp_processor_id(); 3090 3091 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3092 return; 3093 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3094 kvm_arch_hardware_disable(); 3095 } 3096 3097 static void hardware_disable(void) 3098 { 3099 raw_spin_lock(&kvm_count_lock); 3100 if (kvm_usage_count) 3101 hardware_disable_nolock(NULL); 3102 raw_spin_unlock(&kvm_count_lock); 3103 } 3104 3105 static void hardware_disable_all_nolock(void) 3106 { 3107 BUG_ON(!kvm_usage_count); 3108 3109 kvm_usage_count--; 3110 if (!kvm_usage_count) 3111 on_each_cpu(hardware_disable_nolock, NULL, 1); 3112 } 3113 3114 static void hardware_disable_all(void) 3115 { 3116 raw_spin_lock(&kvm_count_lock); 3117 hardware_disable_all_nolock(); 3118 raw_spin_unlock(&kvm_count_lock); 3119 } 3120 3121 static int hardware_enable_all(void) 3122 { 3123 int r = 0; 3124 3125 raw_spin_lock(&kvm_count_lock); 3126 3127 kvm_usage_count++; 3128 if (kvm_usage_count == 1) { 3129 atomic_set(&hardware_enable_failed, 0); 3130 on_each_cpu(hardware_enable_nolock, NULL, 1); 3131 3132 if (atomic_read(&hardware_enable_failed)) { 3133 hardware_disable_all_nolock(); 3134 r = -EBUSY; 3135 } 3136 } 3137 3138 raw_spin_unlock(&kvm_count_lock); 3139 3140 return r; 3141 } 3142 3143 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 3144 void *v) 3145 { 3146 val &= ~CPU_TASKS_FROZEN; 3147 switch (val) { 3148 case CPU_DYING: 3149 hardware_disable(); 3150 break; 3151 case CPU_STARTING: 3152 hardware_enable(); 3153 break; 3154 } 3155 return NOTIFY_OK; 3156 } 3157 3158 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 3159 void *v) 3160 { 3161 /* 3162 * Some (well, at least mine) BIOSes hang on reboot if 3163 * in vmx root mode. 3164 * 3165 * And Intel TXT required VMX off for all cpu when system shutdown. 3166 */ 3167 pr_info("kvm: exiting hardware virtualization\n"); 3168 kvm_rebooting = true; 3169 on_each_cpu(hardware_disable_nolock, NULL, 1); 3170 return NOTIFY_OK; 3171 } 3172 3173 static struct notifier_block kvm_reboot_notifier = { 3174 .notifier_call = kvm_reboot, 3175 .priority = 0, 3176 }; 3177 3178 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 3179 { 3180 int i; 3181 3182 for (i = 0; i < bus->dev_count; i++) { 3183 struct kvm_io_device *pos = bus->range[i].dev; 3184 3185 kvm_iodevice_destructor(pos); 3186 } 3187 kfree(bus); 3188 } 3189 3190 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 3191 const struct kvm_io_range *r2) 3192 { 3193 gpa_t addr1 = r1->addr; 3194 gpa_t addr2 = r2->addr; 3195 3196 if (addr1 < addr2) 3197 return -1; 3198 3199 /* If r2->len == 0, match the exact address. If r2->len != 0, 3200 * accept any overlapping write. Any order is acceptable for 3201 * overlapping ranges, because kvm_io_bus_get_first_dev ensures 3202 * we process all of them. 3203 */ 3204 if (r2->len) { 3205 addr1 += r1->len; 3206 addr2 += r2->len; 3207 } 3208 3209 if (addr1 > addr2) 3210 return 1; 3211 3212 return 0; 3213 } 3214 3215 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 3216 { 3217 return kvm_io_bus_cmp(p1, p2); 3218 } 3219 3220 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 3221 gpa_t addr, int len) 3222 { 3223 bus->range[bus->dev_count++] = (struct kvm_io_range) { 3224 .addr = addr, 3225 .len = len, 3226 .dev = dev, 3227 }; 3228 3229 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 3230 kvm_io_bus_sort_cmp, NULL); 3231 3232 return 0; 3233 } 3234 3235 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 3236 gpa_t addr, int len) 3237 { 3238 struct kvm_io_range *range, key; 3239 int off; 3240 3241 key = (struct kvm_io_range) { 3242 .addr = addr, 3243 .len = len, 3244 }; 3245 3246 range = bsearch(&key, bus->range, bus->dev_count, 3247 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 3248 if (range == NULL) 3249 return -ENOENT; 3250 3251 off = range - bus->range; 3252 3253 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 3254 off--; 3255 3256 return off; 3257 } 3258 3259 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3260 struct kvm_io_range *range, const void *val) 3261 { 3262 int idx; 3263 3264 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3265 if (idx < 0) 3266 return -EOPNOTSUPP; 3267 3268 while (idx < bus->dev_count && 3269 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3270 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 3271 range->len, val)) 3272 return idx; 3273 idx++; 3274 } 3275 3276 return -EOPNOTSUPP; 3277 } 3278 3279 /* kvm_io_bus_write - called under kvm->slots_lock */ 3280 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3281 int len, const void *val) 3282 { 3283 struct kvm_io_bus *bus; 3284 struct kvm_io_range range; 3285 int r; 3286 3287 range = (struct kvm_io_range) { 3288 .addr = addr, 3289 .len = len, 3290 }; 3291 3292 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3293 r = __kvm_io_bus_write(vcpu, bus, &range, val); 3294 return r < 0 ? r : 0; 3295 } 3296 3297 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 3298 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 3299 gpa_t addr, int len, const void *val, long cookie) 3300 { 3301 struct kvm_io_bus *bus; 3302 struct kvm_io_range range; 3303 3304 range = (struct kvm_io_range) { 3305 .addr = addr, 3306 .len = len, 3307 }; 3308 3309 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3310 3311 /* First try the device referenced by cookie. */ 3312 if ((cookie >= 0) && (cookie < bus->dev_count) && 3313 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 3314 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 3315 val)) 3316 return cookie; 3317 3318 /* 3319 * cookie contained garbage; fall back to search and return the 3320 * correct cookie value. 3321 */ 3322 return __kvm_io_bus_write(vcpu, bus, &range, val); 3323 } 3324 3325 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3326 struct kvm_io_range *range, void *val) 3327 { 3328 int idx; 3329 3330 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3331 if (idx < 0) 3332 return -EOPNOTSUPP; 3333 3334 while (idx < bus->dev_count && 3335 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3336 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 3337 range->len, val)) 3338 return idx; 3339 idx++; 3340 } 3341 3342 return -EOPNOTSUPP; 3343 } 3344 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3345 3346 /* kvm_io_bus_read - called under kvm->slots_lock */ 3347 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3348 int len, void *val) 3349 { 3350 struct kvm_io_bus *bus; 3351 struct kvm_io_range range; 3352 int r; 3353 3354 range = (struct kvm_io_range) { 3355 .addr = addr, 3356 .len = len, 3357 }; 3358 3359 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3360 r = __kvm_io_bus_read(vcpu, bus, &range, val); 3361 return r < 0 ? r : 0; 3362 } 3363 3364 3365 /* Caller must hold slots_lock. */ 3366 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3367 int len, struct kvm_io_device *dev) 3368 { 3369 struct kvm_io_bus *new_bus, *bus; 3370 3371 bus = kvm->buses[bus_idx]; 3372 /* exclude ioeventfd which is limited by maximum fd */ 3373 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 3374 return -ENOSPC; 3375 3376 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) * 3377 sizeof(struct kvm_io_range)), GFP_KERNEL); 3378 if (!new_bus) 3379 return -ENOMEM; 3380 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 3381 sizeof(struct kvm_io_range))); 3382 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 3383 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3384 synchronize_srcu_expedited(&kvm->srcu); 3385 kfree(bus); 3386 3387 return 0; 3388 } 3389 3390 /* Caller must hold slots_lock. */ 3391 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3392 struct kvm_io_device *dev) 3393 { 3394 int i, r; 3395 struct kvm_io_bus *new_bus, *bus; 3396 3397 bus = kvm->buses[bus_idx]; 3398 r = -ENOENT; 3399 for (i = 0; i < bus->dev_count; i++) 3400 if (bus->range[i].dev == dev) { 3401 r = 0; 3402 break; 3403 } 3404 3405 if (r) 3406 return r; 3407 3408 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) * 3409 sizeof(struct kvm_io_range)), GFP_KERNEL); 3410 if (!new_bus) 3411 return -ENOMEM; 3412 3413 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3414 new_bus->dev_count--; 3415 memcpy(new_bus->range + i, bus->range + i + 1, 3416 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3417 3418 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3419 synchronize_srcu_expedited(&kvm->srcu); 3420 kfree(bus); 3421 return r; 3422 } 3423 3424 static struct notifier_block kvm_cpu_notifier = { 3425 .notifier_call = kvm_cpu_hotplug, 3426 }; 3427 3428 static int vm_stat_get(void *_offset, u64 *val) 3429 { 3430 unsigned offset = (long)_offset; 3431 struct kvm *kvm; 3432 3433 *val = 0; 3434 spin_lock(&kvm_lock); 3435 list_for_each_entry(kvm, &vm_list, vm_list) 3436 *val += *(u32 *)((void *)kvm + offset); 3437 spin_unlock(&kvm_lock); 3438 return 0; 3439 } 3440 3441 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 3442 3443 static int vcpu_stat_get(void *_offset, u64 *val) 3444 { 3445 unsigned offset = (long)_offset; 3446 struct kvm *kvm; 3447 struct kvm_vcpu *vcpu; 3448 int i; 3449 3450 *val = 0; 3451 spin_lock(&kvm_lock); 3452 list_for_each_entry(kvm, &vm_list, vm_list) 3453 kvm_for_each_vcpu(i, vcpu, kvm) 3454 *val += *(u32 *)((void *)vcpu + offset); 3455 3456 spin_unlock(&kvm_lock); 3457 return 0; 3458 } 3459 3460 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 3461 3462 static const struct file_operations *stat_fops[] = { 3463 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3464 [KVM_STAT_VM] = &vm_stat_fops, 3465 }; 3466 3467 static int kvm_init_debug(void) 3468 { 3469 int r = -EEXIST; 3470 struct kvm_stats_debugfs_item *p; 3471 3472 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3473 if (kvm_debugfs_dir == NULL) 3474 goto out; 3475 3476 for (p = debugfs_entries; p->name; ++p) { 3477 if (!debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 3478 (void *)(long)p->offset, 3479 stat_fops[p->kind])) 3480 goto out_dir; 3481 } 3482 3483 return 0; 3484 3485 out_dir: 3486 debugfs_remove_recursive(kvm_debugfs_dir); 3487 out: 3488 return r; 3489 } 3490 3491 static int kvm_suspend(void) 3492 { 3493 if (kvm_usage_count) 3494 hardware_disable_nolock(NULL); 3495 return 0; 3496 } 3497 3498 static void kvm_resume(void) 3499 { 3500 if (kvm_usage_count) { 3501 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3502 hardware_enable_nolock(NULL); 3503 } 3504 } 3505 3506 static struct syscore_ops kvm_syscore_ops = { 3507 .suspend = kvm_suspend, 3508 .resume = kvm_resume, 3509 }; 3510 3511 static inline 3512 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3513 { 3514 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3515 } 3516 3517 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3518 { 3519 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3520 3521 if (vcpu->preempted) 3522 vcpu->preempted = false; 3523 3524 kvm_arch_sched_in(vcpu, cpu); 3525 3526 kvm_arch_vcpu_load(vcpu, cpu); 3527 } 3528 3529 static void kvm_sched_out(struct preempt_notifier *pn, 3530 struct task_struct *next) 3531 { 3532 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3533 3534 if (current->state == TASK_RUNNING) 3535 vcpu->preempted = true; 3536 kvm_arch_vcpu_put(vcpu); 3537 } 3538 3539 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3540 struct module *module) 3541 { 3542 int r; 3543 int cpu; 3544 3545 r = kvm_arch_init(opaque); 3546 if (r) 3547 goto out_fail; 3548 3549 /* 3550 * kvm_arch_init makes sure there's at most one caller 3551 * for architectures that support multiple implementations, 3552 * like intel and amd on x86. 3553 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3554 * conflicts in case kvm is already setup for another implementation. 3555 */ 3556 r = kvm_irqfd_init(); 3557 if (r) 3558 goto out_irqfd; 3559 3560 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 3561 r = -ENOMEM; 3562 goto out_free_0; 3563 } 3564 3565 r = kvm_arch_hardware_setup(); 3566 if (r < 0) 3567 goto out_free_0a; 3568 3569 for_each_online_cpu(cpu) { 3570 smp_call_function_single(cpu, 3571 kvm_arch_check_processor_compat, 3572 &r, 1); 3573 if (r < 0) 3574 goto out_free_1; 3575 } 3576 3577 r = register_cpu_notifier(&kvm_cpu_notifier); 3578 if (r) 3579 goto out_free_2; 3580 register_reboot_notifier(&kvm_reboot_notifier); 3581 3582 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 3583 if (!vcpu_align) 3584 vcpu_align = __alignof__(struct kvm_vcpu); 3585 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 3586 0, NULL); 3587 if (!kvm_vcpu_cache) { 3588 r = -ENOMEM; 3589 goto out_free_3; 3590 } 3591 3592 r = kvm_async_pf_init(); 3593 if (r) 3594 goto out_free; 3595 3596 kvm_chardev_ops.owner = module; 3597 kvm_vm_fops.owner = module; 3598 kvm_vcpu_fops.owner = module; 3599 3600 r = misc_register(&kvm_dev); 3601 if (r) { 3602 pr_err("kvm: misc device register failed\n"); 3603 goto out_unreg; 3604 } 3605 3606 register_syscore_ops(&kvm_syscore_ops); 3607 3608 kvm_preempt_ops.sched_in = kvm_sched_in; 3609 kvm_preempt_ops.sched_out = kvm_sched_out; 3610 3611 r = kvm_init_debug(); 3612 if (r) { 3613 pr_err("kvm: create debugfs files failed\n"); 3614 goto out_undebugfs; 3615 } 3616 3617 r = kvm_vfio_ops_init(); 3618 WARN_ON(r); 3619 3620 return 0; 3621 3622 out_undebugfs: 3623 unregister_syscore_ops(&kvm_syscore_ops); 3624 misc_deregister(&kvm_dev); 3625 out_unreg: 3626 kvm_async_pf_deinit(); 3627 out_free: 3628 kmem_cache_destroy(kvm_vcpu_cache); 3629 out_free_3: 3630 unregister_reboot_notifier(&kvm_reboot_notifier); 3631 unregister_cpu_notifier(&kvm_cpu_notifier); 3632 out_free_2: 3633 out_free_1: 3634 kvm_arch_hardware_unsetup(); 3635 out_free_0a: 3636 free_cpumask_var(cpus_hardware_enabled); 3637 out_free_0: 3638 kvm_irqfd_exit(); 3639 out_irqfd: 3640 kvm_arch_exit(); 3641 out_fail: 3642 return r; 3643 } 3644 EXPORT_SYMBOL_GPL(kvm_init); 3645 3646 void kvm_exit(void) 3647 { 3648 debugfs_remove_recursive(kvm_debugfs_dir); 3649 misc_deregister(&kvm_dev); 3650 kmem_cache_destroy(kvm_vcpu_cache); 3651 kvm_async_pf_deinit(); 3652 unregister_syscore_ops(&kvm_syscore_ops); 3653 unregister_reboot_notifier(&kvm_reboot_notifier); 3654 unregister_cpu_notifier(&kvm_cpu_notifier); 3655 on_each_cpu(hardware_disable_nolock, NULL, 1); 3656 kvm_arch_hardware_unsetup(); 3657 kvm_arch_exit(); 3658 kvm_irqfd_exit(); 3659 free_cpumask_var(cpus_hardware_enabled); 3660 kvm_vfio_ops_exit(); 3661 } 3662 EXPORT_SYMBOL_GPL(kvm_exit); 3663