1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include <kvm/iodev.h> 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 #include <linux/sort.h> 51 #include <linux/bsearch.h> 52 53 #include <asm/processor.h> 54 #include <asm/io.h> 55 #include <asm/ioctl.h> 56 #include <asm/uaccess.h> 57 #include <asm/pgtable.h> 58 59 #include "coalesced_mmio.h" 60 #include "async_pf.h" 61 #include "vfio.h" 62 63 #define CREATE_TRACE_POINTS 64 #include <trace/events/kvm.h> 65 66 MODULE_AUTHOR("Qumranet"); 67 MODULE_LICENSE("GPL"); 68 69 /* Architectures should define their poll value according to the halt latency */ 70 static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; 71 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR); 72 73 /* Default doubles per-vcpu halt_poll_ns. */ 74 static unsigned int halt_poll_ns_grow = 2; 75 module_param(halt_poll_ns_grow, uint, S_IRUGO | S_IWUSR); 76 77 /* Default resets per-vcpu halt_poll_ns . */ 78 static unsigned int halt_poll_ns_shrink; 79 module_param(halt_poll_ns_shrink, uint, S_IRUGO | S_IWUSR); 80 81 /* 82 * Ordering of locks: 83 * 84 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 85 */ 86 87 DEFINE_SPINLOCK(kvm_lock); 88 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 89 LIST_HEAD(vm_list); 90 91 static cpumask_var_t cpus_hardware_enabled; 92 static int kvm_usage_count; 93 static atomic_t hardware_enable_failed; 94 95 struct kmem_cache *kvm_vcpu_cache; 96 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 97 98 static __read_mostly struct preempt_ops kvm_preempt_ops; 99 100 struct dentry *kvm_debugfs_dir; 101 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 102 103 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 104 unsigned long arg); 105 #ifdef CONFIG_KVM_COMPAT 106 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 107 unsigned long arg); 108 #endif 109 static int hardware_enable_all(void); 110 static void hardware_disable_all(void); 111 112 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 113 114 static void kvm_release_pfn_dirty(kvm_pfn_t pfn); 115 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn); 116 117 __visible bool kvm_rebooting; 118 EXPORT_SYMBOL_GPL(kvm_rebooting); 119 120 static bool largepages_enabled = true; 121 122 bool kvm_is_reserved_pfn(kvm_pfn_t pfn) 123 { 124 if (pfn_valid(pfn)) 125 return PageReserved(pfn_to_page(pfn)); 126 127 return true; 128 } 129 130 /* 131 * Switches to specified vcpu, until a matching vcpu_put() 132 */ 133 int vcpu_load(struct kvm_vcpu *vcpu) 134 { 135 int cpu; 136 137 if (mutex_lock_killable(&vcpu->mutex)) 138 return -EINTR; 139 cpu = get_cpu(); 140 preempt_notifier_register(&vcpu->preempt_notifier); 141 kvm_arch_vcpu_load(vcpu, cpu); 142 put_cpu(); 143 return 0; 144 } 145 146 void vcpu_put(struct kvm_vcpu *vcpu) 147 { 148 preempt_disable(); 149 kvm_arch_vcpu_put(vcpu); 150 preempt_notifier_unregister(&vcpu->preempt_notifier); 151 preempt_enable(); 152 mutex_unlock(&vcpu->mutex); 153 } 154 155 static void ack_flush(void *_completed) 156 { 157 } 158 159 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 160 { 161 int i, cpu, me; 162 cpumask_var_t cpus; 163 bool called = true; 164 struct kvm_vcpu *vcpu; 165 166 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 167 168 me = get_cpu(); 169 kvm_for_each_vcpu(i, vcpu, kvm) { 170 kvm_make_request(req, vcpu); 171 cpu = vcpu->cpu; 172 173 /* Set ->requests bit before we read ->mode. */ 174 smp_mb__after_atomic(); 175 176 if (cpus != NULL && cpu != -1 && cpu != me && 177 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 178 cpumask_set_cpu(cpu, cpus); 179 } 180 if (unlikely(cpus == NULL)) 181 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 182 else if (!cpumask_empty(cpus)) 183 smp_call_function_many(cpus, ack_flush, NULL, 1); 184 else 185 called = false; 186 put_cpu(); 187 free_cpumask_var(cpus); 188 return called; 189 } 190 191 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 192 void kvm_flush_remote_tlbs(struct kvm *kvm) 193 { 194 /* 195 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in 196 * kvm_make_all_cpus_request. 197 */ 198 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty); 199 200 /* 201 * We want to publish modifications to the page tables before reading 202 * mode. Pairs with a memory barrier in arch-specific code. 203 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest 204 * and smp_mb in walk_shadow_page_lockless_begin/end. 205 * - powerpc: smp_mb in kvmppc_prepare_to_enter. 206 * 207 * There is already an smp_mb__after_atomic() before 208 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that 209 * barrier here. 210 */ 211 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 212 ++kvm->stat.remote_tlb_flush; 213 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 214 } 215 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 216 #endif 217 218 void kvm_reload_remote_mmus(struct kvm *kvm) 219 { 220 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 221 } 222 223 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 224 { 225 struct page *page; 226 int r; 227 228 mutex_init(&vcpu->mutex); 229 vcpu->cpu = -1; 230 vcpu->kvm = kvm; 231 vcpu->vcpu_id = id; 232 vcpu->pid = NULL; 233 init_swait_queue_head(&vcpu->wq); 234 kvm_async_pf_vcpu_init(vcpu); 235 236 vcpu->pre_pcpu = -1; 237 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list); 238 239 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 240 if (!page) { 241 r = -ENOMEM; 242 goto fail; 243 } 244 vcpu->run = page_address(page); 245 246 kvm_vcpu_set_in_spin_loop(vcpu, false); 247 kvm_vcpu_set_dy_eligible(vcpu, false); 248 vcpu->preempted = false; 249 250 r = kvm_arch_vcpu_init(vcpu); 251 if (r < 0) 252 goto fail_free_run; 253 return 0; 254 255 fail_free_run: 256 free_page((unsigned long)vcpu->run); 257 fail: 258 return r; 259 } 260 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 261 262 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 263 { 264 put_pid(vcpu->pid); 265 kvm_arch_vcpu_uninit(vcpu); 266 free_page((unsigned long)vcpu->run); 267 } 268 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 269 270 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 271 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 272 { 273 return container_of(mn, struct kvm, mmu_notifier); 274 } 275 276 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 277 struct mm_struct *mm, 278 unsigned long address) 279 { 280 struct kvm *kvm = mmu_notifier_to_kvm(mn); 281 int need_tlb_flush, idx; 282 283 /* 284 * When ->invalidate_page runs, the linux pte has been zapped 285 * already but the page is still allocated until 286 * ->invalidate_page returns. So if we increase the sequence 287 * here the kvm page fault will notice if the spte can't be 288 * established because the page is going to be freed. If 289 * instead the kvm page fault establishes the spte before 290 * ->invalidate_page runs, kvm_unmap_hva will release it 291 * before returning. 292 * 293 * The sequence increase only need to be seen at spin_unlock 294 * time, and not at spin_lock time. 295 * 296 * Increasing the sequence after the spin_unlock would be 297 * unsafe because the kvm page fault could then establish the 298 * pte after kvm_unmap_hva returned, without noticing the page 299 * is going to be freed. 300 */ 301 idx = srcu_read_lock(&kvm->srcu); 302 spin_lock(&kvm->mmu_lock); 303 304 kvm->mmu_notifier_seq++; 305 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 306 /* we've to flush the tlb before the pages can be freed */ 307 if (need_tlb_flush) 308 kvm_flush_remote_tlbs(kvm); 309 310 spin_unlock(&kvm->mmu_lock); 311 312 kvm_arch_mmu_notifier_invalidate_page(kvm, address); 313 314 srcu_read_unlock(&kvm->srcu, idx); 315 } 316 317 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 318 struct mm_struct *mm, 319 unsigned long address, 320 pte_t pte) 321 { 322 struct kvm *kvm = mmu_notifier_to_kvm(mn); 323 int idx; 324 325 idx = srcu_read_lock(&kvm->srcu); 326 spin_lock(&kvm->mmu_lock); 327 kvm->mmu_notifier_seq++; 328 kvm_set_spte_hva(kvm, address, pte); 329 spin_unlock(&kvm->mmu_lock); 330 srcu_read_unlock(&kvm->srcu, idx); 331 } 332 333 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 334 struct mm_struct *mm, 335 unsigned long start, 336 unsigned long end) 337 { 338 struct kvm *kvm = mmu_notifier_to_kvm(mn); 339 int need_tlb_flush = 0, idx; 340 341 idx = srcu_read_lock(&kvm->srcu); 342 spin_lock(&kvm->mmu_lock); 343 /* 344 * The count increase must become visible at unlock time as no 345 * spte can be established without taking the mmu_lock and 346 * count is also read inside the mmu_lock critical section. 347 */ 348 kvm->mmu_notifier_count++; 349 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 350 need_tlb_flush |= kvm->tlbs_dirty; 351 /* we've to flush the tlb before the pages can be freed */ 352 if (need_tlb_flush) 353 kvm_flush_remote_tlbs(kvm); 354 355 spin_unlock(&kvm->mmu_lock); 356 srcu_read_unlock(&kvm->srcu, idx); 357 } 358 359 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 360 struct mm_struct *mm, 361 unsigned long start, 362 unsigned long end) 363 { 364 struct kvm *kvm = mmu_notifier_to_kvm(mn); 365 366 spin_lock(&kvm->mmu_lock); 367 /* 368 * This sequence increase will notify the kvm page fault that 369 * the page that is going to be mapped in the spte could have 370 * been freed. 371 */ 372 kvm->mmu_notifier_seq++; 373 smp_wmb(); 374 /* 375 * The above sequence increase must be visible before the 376 * below count decrease, which is ensured by the smp_wmb above 377 * in conjunction with the smp_rmb in mmu_notifier_retry(). 378 */ 379 kvm->mmu_notifier_count--; 380 spin_unlock(&kvm->mmu_lock); 381 382 BUG_ON(kvm->mmu_notifier_count < 0); 383 } 384 385 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 386 struct mm_struct *mm, 387 unsigned long start, 388 unsigned long end) 389 { 390 struct kvm *kvm = mmu_notifier_to_kvm(mn); 391 int young, idx; 392 393 idx = srcu_read_lock(&kvm->srcu); 394 spin_lock(&kvm->mmu_lock); 395 396 young = kvm_age_hva(kvm, start, end); 397 if (young) 398 kvm_flush_remote_tlbs(kvm); 399 400 spin_unlock(&kvm->mmu_lock); 401 srcu_read_unlock(&kvm->srcu, idx); 402 403 return young; 404 } 405 406 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, 407 struct mm_struct *mm, 408 unsigned long start, 409 unsigned long end) 410 { 411 struct kvm *kvm = mmu_notifier_to_kvm(mn); 412 int young, idx; 413 414 idx = srcu_read_lock(&kvm->srcu); 415 spin_lock(&kvm->mmu_lock); 416 /* 417 * Even though we do not flush TLB, this will still adversely 418 * affect performance on pre-Haswell Intel EPT, where there is 419 * no EPT Access Bit to clear so that we have to tear down EPT 420 * tables instead. If we find this unacceptable, we can always 421 * add a parameter to kvm_age_hva so that it effectively doesn't 422 * do anything on clear_young. 423 * 424 * Also note that currently we never issue secondary TLB flushes 425 * from clear_young, leaving this job up to the regular system 426 * cadence. If we find this inaccurate, we might come up with a 427 * more sophisticated heuristic later. 428 */ 429 young = kvm_age_hva(kvm, start, end); 430 spin_unlock(&kvm->mmu_lock); 431 srcu_read_unlock(&kvm->srcu, idx); 432 433 return young; 434 } 435 436 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 437 struct mm_struct *mm, 438 unsigned long address) 439 { 440 struct kvm *kvm = mmu_notifier_to_kvm(mn); 441 int young, idx; 442 443 idx = srcu_read_lock(&kvm->srcu); 444 spin_lock(&kvm->mmu_lock); 445 young = kvm_test_age_hva(kvm, address); 446 spin_unlock(&kvm->mmu_lock); 447 srcu_read_unlock(&kvm->srcu, idx); 448 449 return young; 450 } 451 452 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 453 struct mm_struct *mm) 454 { 455 struct kvm *kvm = mmu_notifier_to_kvm(mn); 456 int idx; 457 458 idx = srcu_read_lock(&kvm->srcu); 459 kvm_arch_flush_shadow_all(kvm); 460 srcu_read_unlock(&kvm->srcu, idx); 461 } 462 463 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 464 .invalidate_page = kvm_mmu_notifier_invalidate_page, 465 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 466 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 467 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 468 .clear_young = kvm_mmu_notifier_clear_young, 469 .test_young = kvm_mmu_notifier_test_young, 470 .change_pte = kvm_mmu_notifier_change_pte, 471 .release = kvm_mmu_notifier_release, 472 }; 473 474 static int kvm_init_mmu_notifier(struct kvm *kvm) 475 { 476 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 477 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 478 } 479 480 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 481 482 static int kvm_init_mmu_notifier(struct kvm *kvm) 483 { 484 return 0; 485 } 486 487 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 488 489 static struct kvm_memslots *kvm_alloc_memslots(void) 490 { 491 int i; 492 struct kvm_memslots *slots; 493 494 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 495 if (!slots) 496 return NULL; 497 498 /* 499 * Init kvm generation close to the maximum to easily test the 500 * code of handling generation number wrap-around. 501 */ 502 slots->generation = -150; 503 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 504 slots->id_to_index[i] = slots->memslots[i].id = i; 505 506 return slots; 507 } 508 509 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 510 { 511 if (!memslot->dirty_bitmap) 512 return; 513 514 kvfree(memslot->dirty_bitmap); 515 memslot->dirty_bitmap = NULL; 516 } 517 518 /* 519 * Free any memory in @free but not in @dont. 520 */ 521 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 522 struct kvm_memory_slot *dont) 523 { 524 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 525 kvm_destroy_dirty_bitmap(free); 526 527 kvm_arch_free_memslot(kvm, free, dont); 528 529 free->npages = 0; 530 } 531 532 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 533 { 534 struct kvm_memory_slot *memslot; 535 536 if (!slots) 537 return; 538 539 kvm_for_each_memslot(memslot, slots) 540 kvm_free_memslot(kvm, memslot, NULL); 541 542 kvfree(slots); 543 } 544 545 static struct kvm *kvm_create_vm(unsigned long type) 546 { 547 int r, i; 548 struct kvm *kvm = kvm_arch_alloc_vm(); 549 550 if (!kvm) 551 return ERR_PTR(-ENOMEM); 552 553 spin_lock_init(&kvm->mmu_lock); 554 atomic_inc(¤t->mm->mm_count); 555 kvm->mm = current->mm; 556 kvm_eventfd_init(kvm); 557 mutex_init(&kvm->lock); 558 mutex_init(&kvm->irq_lock); 559 mutex_init(&kvm->slots_lock); 560 atomic_set(&kvm->users_count, 1); 561 INIT_LIST_HEAD(&kvm->devices); 562 563 r = kvm_arch_init_vm(kvm, type); 564 if (r) 565 goto out_err_no_disable; 566 567 r = hardware_enable_all(); 568 if (r) 569 goto out_err_no_disable; 570 571 #ifdef CONFIG_HAVE_KVM_IRQFD 572 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 573 #endif 574 575 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 576 577 r = -ENOMEM; 578 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 579 kvm->memslots[i] = kvm_alloc_memslots(); 580 if (!kvm->memslots[i]) 581 goto out_err_no_srcu; 582 } 583 584 if (init_srcu_struct(&kvm->srcu)) 585 goto out_err_no_srcu; 586 if (init_srcu_struct(&kvm->irq_srcu)) 587 goto out_err_no_irq_srcu; 588 for (i = 0; i < KVM_NR_BUSES; i++) { 589 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 590 GFP_KERNEL); 591 if (!kvm->buses[i]) 592 goto out_err; 593 } 594 595 r = kvm_init_mmu_notifier(kvm); 596 if (r) 597 goto out_err; 598 599 spin_lock(&kvm_lock); 600 list_add(&kvm->vm_list, &vm_list); 601 spin_unlock(&kvm_lock); 602 603 preempt_notifier_inc(); 604 605 return kvm; 606 607 out_err: 608 cleanup_srcu_struct(&kvm->irq_srcu); 609 out_err_no_irq_srcu: 610 cleanup_srcu_struct(&kvm->srcu); 611 out_err_no_srcu: 612 hardware_disable_all(); 613 out_err_no_disable: 614 for (i = 0; i < KVM_NR_BUSES; i++) 615 kfree(kvm->buses[i]); 616 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 617 kvm_free_memslots(kvm, kvm->memslots[i]); 618 kvm_arch_free_vm(kvm); 619 mmdrop(current->mm); 620 return ERR_PTR(r); 621 } 622 623 /* 624 * Avoid using vmalloc for a small buffer. 625 * Should not be used when the size is statically known. 626 */ 627 void *kvm_kvzalloc(unsigned long size) 628 { 629 if (size > PAGE_SIZE) 630 return vzalloc(size); 631 else 632 return kzalloc(size, GFP_KERNEL); 633 } 634 635 static void kvm_destroy_devices(struct kvm *kvm) 636 { 637 struct kvm_device *dev, *tmp; 638 639 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { 640 list_del(&dev->vm_node); 641 dev->ops->destroy(dev); 642 } 643 } 644 645 static void kvm_destroy_vm(struct kvm *kvm) 646 { 647 int i; 648 struct mm_struct *mm = kvm->mm; 649 650 kvm_arch_sync_events(kvm); 651 spin_lock(&kvm_lock); 652 list_del(&kvm->vm_list); 653 spin_unlock(&kvm_lock); 654 kvm_free_irq_routing(kvm); 655 for (i = 0; i < KVM_NR_BUSES; i++) 656 kvm_io_bus_destroy(kvm->buses[i]); 657 kvm_coalesced_mmio_free(kvm); 658 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 659 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 660 #else 661 kvm_arch_flush_shadow_all(kvm); 662 #endif 663 kvm_arch_destroy_vm(kvm); 664 kvm_destroy_devices(kvm); 665 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 666 kvm_free_memslots(kvm, kvm->memslots[i]); 667 cleanup_srcu_struct(&kvm->irq_srcu); 668 cleanup_srcu_struct(&kvm->srcu); 669 kvm_arch_free_vm(kvm); 670 preempt_notifier_dec(); 671 hardware_disable_all(); 672 mmdrop(mm); 673 } 674 675 void kvm_get_kvm(struct kvm *kvm) 676 { 677 atomic_inc(&kvm->users_count); 678 } 679 EXPORT_SYMBOL_GPL(kvm_get_kvm); 680 681 void kvm_put_kvm(struct kvm *kvm) 682 { 683 if (atomic_dec_and_test(&kvm->users_count)) 684 kvm_destroy_vm(kvm); 685 } 686 EXPORT_SYMBOL_GPL(kvm_put_kvm); 687 688 689 static int kvm_vm_release(struct inode *inode, struct file *filp) 690 { 691 struct kvm *kvm = filp->private_data; 692 693 kvm_irqfd_release(kvm); 694 695 kvm_put_kvm(kvm); 696 return 0; 697 } 698 699 /* 700 * Allocation size is twice as large as the actual dirty bitmap size. 701 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 702 */ 703 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 704 { 705 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 706 707 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes); 708 if (!memslot->dirty_bitmap) 709 return -ENOMEM; 710 711 return 0; 712 } 713 714 /* 715 * Insert memslot and re-sort memslots based on their GFN, 716 * so binary search could be used to lookup GFN. 717 * Sorting algorithm takes advantage of having initially 718 * sorted array and known changed memslot position. 719 */ 720 static void update_memslots(struct kvm_memslots *slots, 721 struct kvm_memory_slot *new) 722 { 723 int id = new->id; 724 int i = slots->id_to_index[id]; 725 struct kvm_memory_slot *mslots = slots->memslots; 726 727 WARN_ON(mslots[i].id != id); 728 if (!new->npages) { 729 WARN_ON(!mslots[i].npages); 730 if (mslots[i].npages) 731 slots->used_slots--; 732 } else { 733 if (!mslots[i].npages) 734 slots->used_slots++; 735 } 736 737 while (i < KVM_MEM_SLOTS_NUM - 1 && 738 new->base_gfn <= mslots[i + 1].base_gfn) { 739 if (!mslots[i + 1].npages) 740 break; 741 mslots[i] = mslots[i + 1]; 742 slots->id_to_index[mslots[i].id] = i; 743 i++; 744 } 745 746 /* 747 * The ">=" is needed when creating a slot with base_gfn == 0, 748 * so that it moves before all those with base_gfn == npages == 0. 749 * 750 * On the other hand, if new->npages is zero, the above loop has 751 * already left i pointing to the beginning of the empty part of 752 * mslots, and the ">=" would move the hole backwards in this 753 * case---which is wrong. So skip the loop when deleting a slot. 754 */ 755 if (new->npages) { 756 while (i > 0 && 757 new->base_gfn >= mslots[i - 1].base_gfn) { 758 mslots[i] = mslots[i - 1]; 759 slots->id_to_index[mslots[i].id] = i; 760 i--; 761 } 762 } else 763 WARN_ON_ONCE(i != slots->used_slots); 764 765 mslots[i] = *new; 766 slots->id_to_index[mslots[i].id] = i; 767 } 768 769 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 770 { 771 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 772 773 #ifdef __KVM_HAVE_READONLY_MEM 774 valid_flags |= KVM_MEM_READONLY; 775 #endif 776 777 if (mem->flags & ~valid_flags) 778 return -EINVAL; 779 780 return 0; 781 } 782 783 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 784 int as_id, struct kvm_memslots *slots) 785 { 786 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); 787 788 /* 789 * Set the low bit in the generation, which disables SPTE caching 790 * until the end of synchronize_srcu_expedited. 791 */ 792 WARN_ON(old_memslots->generation & 1); 793 slots->generation = old_memslots->generation + 1; 794 795 rcu_assign_pointer(kvm->memslots[as_id], slots); 796 synchronize_srcu_expedited(&kvm->srcu); 797 798 /* 799 * Increment the new memslot generation a second time. This prevents 800 * vm exits that race with memslot updates from caching a memslot 801 * generation that will (potentially) be valid forever. 802 */ 803 slots->generation++; 804 805 kvm_arch_memslots_updated(kvm, slots); 806 807 return old_memslots; 808 } 809 810 /* 811 * Allocate some memory and give it an address in the guest physical address 812 * space. 813 * 814 * Discontiguous memory is allowed, mostly for framebuffers. 815 * 816 * Must be called holding kvm->slots_lock for write. 817 */ 818 int __kvm_set_memory_region(struct kvm *kvm, 819 const struct kvm_userspace_memory_region *mem) 820 { 821 int r; 822 gfn_t base_gfn; 823 unsigned long npages; 824 struct kvm_memory_slot *slot; 825 struct kvm_memory_slot old, new; 826 struct kvm_memslots *slots = NULL, *old_memslots; 827 int as_id, id; 828 enum kvm_mr_change change; 829 830 r = check_memory_region_flags(mem); 831 if (r) 832 goto out; 833 834 r = -EINVAL; 835 as_id = mem->slot >> 16; 836 id = (u16)mem->slot; 837 838 /* General sanity checks */ 839 if (mem->memory_size & (PAGE_SIZE - 1)) 840 goto out; 841 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 842 goto out; 843 /* We can read the guest memory with __xxx_user() later on. */ 844 if ((id < KVM_USER_MEM_SLOTS) && 845 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 846 !access_ok(VERIFY_WRITE, 847 (void __user *)(unsigned long)mem->userspace_addr, 848 mem->memory_size))) 849 goto out; 850 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 851 goto out; 852 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 853 goto out; 854 855 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); 856 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 857 npages = mem->memory_size >> PAGE_SHIFT; 858 859 if (npages > KVM_MEM_MAX_NR_PAGES) 860 goto out; 861 862 new = old = *slot; 863 864 new.id = id; 865 new.base_gfn = base_gfn; 866 new.npages = npages; 867 new.flags = mem->flags; 868 869 if (npages) { 870 if (!old.npages) 871 change = KVM_MR_CREATE; 872 else { /* Modify an existing slot. */ 873 if ((mem->userspace_addr != old.userspace_addr) || 874 (npages != old.npages) || 875 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 876 goto out; 877 878 if (base_gfn != old.base_gfn) 879 change = KVM_MR_MOVE; 880 else if (new.flags != old.flags) 881 change = KVM_MR_FLAGS_ONLY; 882 else { /* Nothing to change. */ 883 r = 0; 884 goto out; 885 } 886 } 887 } else { 888 if (!old.npages) 889 goto out; 890 891 change = KVM_MR_DELETE; 892 new.base_gfn = 0; 893 new.flags = 0; 894 } 895 896 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 897 /* Check for overlaps */ 898 r = -EEXIST; 899 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { 900 if ((slot->id >= KVM_USER_MEM_SLOTS) || 901 (slot->id == id)) 902 continue; 903 if (!((base_gfn + npages <= slot->base_gfn) || 904 (base_gfn >= slot->base_gfn + slot->npages))) 905 goto out; 906 } 907 } 908 909 /* Free page dirty bitmap if unneeded */ 910 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 911 new.dirty_bitmap = NULL; 912 913 r = -ENOMEM; 914 if (change == KVM_MR_CREATE) { 915 new.userspace_addr = mem->userspace_addr; 916 917 if (kvm_arch_create_memslot(kvm, &new, npages)) 918 goto out_free; 919 } 920 921 /* Allocate page dirty bitmap if needed */ 922 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 923 if (kvm_create_dirty_bitmap(&new) < 0) 924 goto out_free; 925 } 926 927 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 928 if (!slots) 929 goto out_free; 930 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); 931 932 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 933 slot = id_to_memslot(slots, id); 934 slot->flags |= KVM_MEMSLOT_INVALID; 935 936 old_memslots = install_new_memslots(kvm, as_id, slots); 937 938 /* slot was deleted or moved, clear iommu mapping */ 939 kvm_iommu_unmap_pages(kvm, &old); 940 /* From this point no new shadow pages pointing to a deleted, 941 * or moved, memslot will be created. 942 * 943 * validation of sp->gfn happens in: 944 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 945 * - kvm_is_visible_gfn (mmu_check_roots) 946 */ 947 kvm_arch_flush_shadow_memslot(kvm, slot); 948 949 /* 950 * We can re-use the old_memslots from above, the only difference 951 * from the currently installed memslots is the invalid flag. This 952 * will get overwritten by update_memslots anyway. 953 */ 954 slots = old_memslots; 955 } 956 957 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 958 if (r) 959 goto out_slots; 960 961 /* actual memory is freed via old in kvm_free_memslot below */ 962 if (change == KVM_MR_DELETE) { 963 new.dirty_bitmap = NULL; 964 memset(&new.arch, 0, sizeof(new.arch)); 965 } 966 967 update_memslots(slots, &new); 968 old_memslots = install_new_memslots(kvm, as_id, slots); 969 970 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); 971 972 kvm_free_memslot(kvm, &old, &new); 973 kvfree(old_memslots); 974 975 /* 976 * IOMMU mapping: New slots need to be mapped. Old slots need to be 977 * un-mapped and re-mapped if their base changes. Since base change 978 * unmapping is handled above with slot deletion, mapping alone is 979 * needed here. Anything else the iommu might care about for existing 980 * slots (size changes, userspace addr changes and read-only flag 981 * changes) is disallowed above, so any other attribute changes getting 982 * here can be skipped. 983 */ 984 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 985 r = kvm_iommu_map_pages(kvm, &new); 986 return r; 987 } 988 989 return 0; 990 991 out_slots: 992 kvfree(slots); 993 out_free: 994 kvm_free_memslot(kvm, &new, &old); 995 out: 996 return r; 997 } 998 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 999 1000 int kvm_set_memory_region(struct kvm *kvm, 1001 const struct kvm_userspace_memory_region *mem) 1002 { 1003 int r; 1004 1005 mutex_lock(&kvm->slots_lock); 1006 r = __kvm_set_memory_region(kvm, mem); 1007 mutex_unlock(&kvm->slots_lock); 1008 return r; 1009 } 1010 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 1011 1012 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 1013 struct kvm_userspace_memory_region *mem) 1014 { 1015 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 1016 return -EINVAL; 1017 1018 return kvm_set_memory_region(kvm, mem); 1019 } 1020 1021 int kvm_get_dirty_log(struct kvm *kvm, 1022 struct kvm_dirty_log *log, int *is_dirty) 1023 { 1024 struct kvm_memslots *slots; 1025 struct kvm_memory_slot *memslot; 1026 int r, i, as_id, id; 1027 unsigned long n; 1028 unsigned long any = 0; 1029 1030 r = -EINVAL; 1031 as_id = log->slot >> 16; 1032 id = (u16)log->slot; 1033 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1034 goto out; 1035 1036 slots = __kvm_memslots(kvm, as_id); 1037 memslot = id_to_memslot(slots, id); 1038 r = -ENOENT; 1039 if (!memslot->dirty_bitmap) 1040 goto out; 1041 1042 n = kvm_dirty_bitmap_bytes(memslot); 1043 1044 for (i = 0; !any && i < n/sizeof(long); ++i) 1045 any = memslot->dirty_bitmap[i]; 1046 1047 r = -EFAULT; 1048 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 1049 goto out; 1050 1051 if (any) 1052 *is_dirty = 1; 1053 1054 r = 0; 1055 out: 1056 return r; 1057 } 1058 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1059 1060 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1061 /** 1062 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages 1063 * are dirty write protect them for next write. 1064 * @kvm: pointer to kvm instance 1065 * @log: slot id and address to which we copy the log 1066 * @is_dirty: flag set if any page is dirty 1067 * 1068 * We need to keep it in mind that VCPU threads can write to the bitmap 1069 * concurrently. So, to avoid losing track of dirty pages we keep the 1070 * following order: 1071 * 1072 * 1. Take a snapshot of the bit and clear it if needed. 1073 * 2. Write protect the corresponding page. 1074 * 3. Copy the snapshot to the userspace. 1075 * 4. Upon return caller flushes TLB's if needed. 1076 * 1077 * Between 2 and 4, the guest may write to the page using the remaining TLB 1078 * entry. This is not a problem because the page is reported dirty using 1079 * the snapshot taken before and step 4 ensures that writes done after 1080 * exiting to userspace will be logged for the next call. 1081 * 1082 */ 1083 int kvm_get_dirty_log_protect(struct kvm *kvm, 1084 struct kvm_dirty_log *log, bool *is_dirty) 1085 { 1086 struct kvm_memslots *slots; 1087 struct kvm_memory_slot *memslot; 1088 int r, i, as_id, id; 1089 unsigned long n; 1090 unsigned long *dirty_bitmap; 1091 unsigned long *dirty_bitmap_buffer; 1092 1093 r = -EINVAL; 1094 as_id = log->slot >> 16; 1095 id = (u16)log->slot; 1096 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1097 goto out; 1098 1099 slots = __kvm_memslots(kvm, as_id); 1100 memslot = id_to_memslot(slots, id); 1101 1102 dirty_bitmap = memslot->dirty_bitmap; 1103 r = -ENOENT; 1104 if (!dirty_bitmap) 1105 goto out; 1106 1107 n = kvm_dirty_bitmap_bytes(memslot); 1108 1109 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long); 1110 memset(dirty_bitmap_buffer, 0, n); 1111 1112 spin_lock(&kvm->mmu_lock); 1113 *is_dirty = false; 1114 for (i = 0; i < n / sizeof(long); i++) { 1115 unsigned long mask; 1116 gfn_t offset; 1117 1118 if (!dirty_bitmap[i]) 1119 continue; 1120 1121 *is_dirty = true; 1122 1123 mask = xchg(&dirty_bitmap[i], 0); 1124 dirty_bitmap_buffer[i] = mask; 1125 1126 if (mask) { 1127 offset = i * BITS_PER_LONG; 1128 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1129 offset, mask); 1130 } 1131 } 1132 1133 spin_unlock(&kvm->mmu_lock); 1134 1135 r = -EFAULT; 1136 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 1137 goto out; 1138 1139 r = 0; 1140 out: 1141 return r; 1142 } 1143 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect); 1144 #endif 1145 1146 bool kvm_largepages_enabled(void) 1147 { 1148 return largepages_enabled; 1149 } 1150 1151 void kvm_disable_largepages(void) 1152 { 1153 largepages_enabled = false; 1154 } 1155 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1156 1157 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1158 { 1159 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1160 } 1161 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1162 1163 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 1164 { 1165 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); 1166 } 1167 1168 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1169 { 1170 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1171 1172 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1173 memslot->flags & KVM_MEMSLOT_INVALID) 1174 return false; 1175 1176 return true; 1177 } 1178 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1179 1180 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1181 { 1182 struct vm_area_struct *vma; 1183 unsigned long addr, size; 1184 1185 size = PAGE_SIZE; 1186 1187 addr = gfn_to_hva(kvm, gfn); 1188 if (kvm_is_error_hva(addr)) 1189 return PAGE_SIZE; 1190 1191 down_read(¤t->mm->mmap_sem); 1192 vma = find_vma(current->mm, addr); 1193 if (!vma) 1194 goto out; 1195 1196 size = vma_kernel_pagesize(vma); 1197 1198 out: 1199 up_read(¤t->mm->mmap_sem); 1200 1201 return size; 1202 } 1203 1204 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1205 { 1206 return slot->flags & KVM_MEM_READONLY; 1207 } 1208 1209 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1210 gfn_t *nr_pages, bool write) 1211 { 1212 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1213 return KVM_HVA_ERR_BAD; 1214 1215 if (memslot_is_readonly(slot) && write) 1216 return KVM_HVA_ERR_RO_BAD; 1217 1218 if (nr_pages) 1219 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1220 1221 return __gfn_to_hva_memslot(slot, gfn); 1222 } 1223 1224 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1225 gfn_t *nr_pages) 1226 { 1227 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1228 } 1229 1230 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1231 gfn_t gfn) 1232 { 1233 return gfn_to_hva_many(slot, gfn, NULL); 1234 } 1235 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1236 1237 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1238 { 1239 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1240 } 1241 EXPORT_SYMBOL_GPL(gfn_to_hva); 1242 1243 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 1244 { 1245 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 1246 } 1247 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 1248 1249 /* 1250 * If writable is set to false, the hva returned by this function is only 1251 * allowed to be read. 1252 */ 1253 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1254 gfn_t gfn, bool *writable) 1255 { 1256 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1257 1258 if (!kvm_is_error_hva(hva) && writable) 1259 *writable = !memslot_is_readonly(slot); 1260 1261 return hva; 1262 } 1263 1264 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1265 { 1266 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1267 1268 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1269 } 1270 1271 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 1272 { 1273 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1274 1275 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1276 } 1277 1278 static int get_user_page_nowait(unsigned long start, int write, 1279 struct page **page) 1280 { 1281 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1282 1283 if (write) 1284 flags |= FOLL_WRITE; 1285 1286 return __get_user_pages(current, current->mm, start, 1, flags, page, 1287 NULL, NULL); 1288 } 1289 1290 static inline int check_user_page_hwpoison(unsigned long addr) 1291 { 1292 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1293 1294 rc = __get_user_pages(current, current->mm, addr, 1, 1295 flags, NULL, NULL, NULL); 1296 return rc == -EHWPOISON; 1297 } 1298 1299 /* 1300 * The atomic path to get the writable pfn which will be stored in @pfn, 1301 * true indicates success, otherwise false is returned. 1302 */ 1303 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1304 bool write_fault, bool *writable, kvm_pfn_t *pfn) 1305 { 1306 struct page *page[1]; 1307 int npages; 1308 1309 if (!(async || atomic)) 1310 return false; 1311 1312 /* 1313 * Fast pin a writable pfn only if it is a write fault request 1314 * or the caller allows to map a writable pfn for a read fault 1315 * request. 1316 */ 1317 if (!(write_fault || writable)) 1318 return false; 1319 1320 npages = __get_user_pages_fast(addr, 1, 1, page); 1321 if (npages == 1) { 1322 *pfn = page_to_pfn(page[0]); 1323 1324 if (writable) 1325 *writable = true; 1326 return true; 1327 } 1328 1329 return false; 1330 } 1331 1332 /* 1333 * The slow path to get the pfn of the specified host virtual address, 1334 * 1 indicates success, -errno is returned if error is detected. 1335 */ 1336 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1337 bool *writable, kvm_pfn_t *pfn) 1338 { 1339 struct page *page[1]; 1340 int npages = 0; 1341 1342 might_sleep(); 1343 1344 if (writable) 1345 *writable = write_fault; 1346 1347 if (async) { 1348 down_read(¤t->mm->mmap_sem); 1349 npages = get_user_page_nowait(addr, write_fault, page); 1350 up_read(¤t->mm->mmap_sem); 1351 } else 1352 npages = __get_user_pages_unlocked(current, current->mm, addr, 1, 1353 write_fault, 0, page, 1354 FOLL_TOUCH|FOLL_HWPOISON); 1355 if (npages != 1) 1356 return npages; 1357 1358 /* map read fault as writable if possible */ 1359 if (unlikely(!write_fault) && writable) { 1360 struct page *wpage[1]; 1361 1362 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1363 if (npages == 1) { 1364 *writable = true; 1365 put_page(page[0]); 1366 page[0] = wpage[0]; 1367 } 1368 1369 npages = 1; 1370 } 1371 *pfn = page_to_pfn(page[0]); 1372 return npages; 1373 } 1374 1375 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1376 { 1377 if (unlikely(!(vma->vm_flags & VM_READ))) 1378 return false; 1379 1380 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1381 return false; 1382 1383 return true; 1384 } 1385 1386 /* 1387 * Pin guest page in memory and return its pfn. 1388 * @addr: host virtual address which maps memory to the guest 1389 * @atomic: whether this function can sleep 1390 * @async: whether this function need to wait IO complete if the 1391 * host page is not in the memory 1392 * @write_fault: whether we should get a writable host page 1393 * @writable: whether it allows to map a writable host page for !@write_fault 1394 * 1395 * The function will map a writable host page for these two cases: 1396 * 1): @write_fault = true 1397 * 2): @write_fault = false && @writable, @writable will tell the caller 1398 * whether the mapping is writable. 1399 */ 1400 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1401 bool write_fault, bool *writable) 1402 { 1403 struct vm_area_struct *vma; 1404 kvm_pfn_t pfn = 0; 1405 int npages; 1406 1407 /* we can do it either atomically or asynchronously, not both */ 1408 BUG_ON(atomic && async); 1409 1410 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1411 return pfn; 1412 1413 if (atomic) 1414 return KVM_PFN_ERR_FAULT; 1415 1416 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1417 if (npages == 1) 1418 return pfn; 1419 1420 down_read(¤t->mm->mmap_sem); 1421 if (npages == -EHWPOISON || 1422 (!async && check_user_page_hwpoison(addr))) { 1423 pfn = KVM_PFN_ERR_HWPOISON; 1424 goto exit; 1425 } 1426 1427 vma = find_vma_intersection(current->mm, addr, addr + 1); 1428 1429 if (vma == NULL) 1430 pfn = KVM_PFN_ERR_FAULT; 1431 else if ((vma->vm_flags & VM_PFNMAP)) { 1432 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1433 vma->vm_pgoff; 1434 BUG_ON(!kvm_is_reserved_pfn(pfn)); 1435 } else { 1436 if (async && vma_is_valid(vma, write_fault)) 1437 *async = true; 1438 pfn = KVM_PFN_ERR_FAULT; 1439 } 1440 exit: 1441 up_read(¤t->mm->mmap_sem); 1442 return pfn; 1443 } 1444 1445 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, 1446 bool atomic, bool *async, bool write_fault, 1447 bool *writable) 1448 { 1449 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1450 1451 if (addr == KVM_HVA_ERR_RO_BAD) { 1452 if (writable) 1453 *writable = false; 1454 return KVM_PFN_ERR_RO_FAULT; 1455 } 1456 1457 if (kvm_is_error_hva(addr)) { 1458 if (writable) 1459 *writable = false; 1460 return KVM_PFN_NOSLOT; 1461 } 1462 1463 /* Do not map writable pfn in the readonly memslot. */ 1464 if (writable && memslot_is_readonly(slot)) { 1465 *writable = false; 1466 writable = NULL; 1467 } 1468 1469 return hva_to_pfn(addr, atomic, async, write_fault, 1470 writable); 1471 } 1472 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 1473 1474 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1475 bool *writable) 1476 { 1477 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, 1478 write_fault, writable); 1479 } 1480 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1481 1482 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1483 { 1484 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1485 } 1486 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 1487 1488 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1489 { 1490 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1491 } 1492 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1493 1494 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1495 { 1496 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); 1497 } 1498 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1499 1500 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 1501 { 1502 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1503 } 1504 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 1505 1506 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1507 { 1508 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 1509 } 1510 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1511 1512 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 1513 { 1514 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1515 } 1516 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 1517 1518 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1519 struct page **pages, int nr_pages) 1520 { 1521 unsigned long addr; 1522 gfn_t entry; 1523 1524 addr = gfn_to_hva_many(slot, gfn, &entry); 1525 if (kvm_is_error_hva(addr)) 1526 return -1; 1527 1528 if (entry < nr_pages) 1529 return 0; 1530 1531 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1532 } 1533 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1534 1535 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn) 1536 { 1537 if (is_error_noslot_pfn(pfn)) 1538 return KVM_ERR_PTR_BAD_PAGE; 1539 1540 if (kvm_is_reserved_pfn(pfn)) { 1541 WARN_ON(1); 1542 return KVM_ERR_PTR_BAD_PAGE; 1543 } 1544 1545 return pfn_to_page(pfn); 1546 } 1547 1548 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1549 { 1550 kvm_pfn_t pfn; 1551 1552 pfn = gfn_to_pfn(kvm, gfn); 1553 1554 return kvm_pfn_to_page(pfn); 1555 } 1556 EXPORT_SYMBOL_GPL(gfn_to_page); 1557 1558 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) 1559 { 1560 kvm_pfn_t pfn; 1561 1562 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); 1563 1564 return kvm_pfn_to_page(pfn); 1565 } 1566 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); 1567 1568 void kvm_release_page_clean(struct page *page) 1569 { 1570 WARN_ON(is_error_page(page)); 1571 1572 kvm_release_pfn_clean(page_to_pfn(page)); 1573 } 1574 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1575 1576 void kvm_release_pfn_clean(kvm_pfn_t pfn) 1577 { 1578 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) 1579 put_page(pfn_to_page(pfn)); 1580 } 1581 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1582 1583 void kvm_release_page_dirty(struct page *page) 1584 { 1585 WARN_ON(is_error_page(page)); 1586 1587 kvm_release_pfn_dirty(page_to_pfn(page)); 1588 } 1589 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1590 1591 static void kvm_release_pfn_dirty(kvm_pfn_t pfn) 1592 { 1593 kvm_set_pfn_dirty(pfn); 1594 kvm_release_pfn_clean(pfn); 1595 } 1596 1597 void kvm_set_pfn_dirty(kvm_pfn_t pfn) 1598 { 1599 if (!kvm_is_reserved_pfn(pfn)) { 1600 struct page *page = pfn_to_page(pfn); 1601 1602 if (!PageReserved(page)) 1603 SetPageDirty(page); 1604 } 1605 } 1606 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1607 1608 void kvm_set_pfn_accessed(kvm_pfn_t pfn) 1609 { 1610 if (!kvm_is_reserved_pfn(pfn)) 1611 mark_page_accessed(pfn_to_page(pfn)); 1612 } 1613 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1614 1615 void kvm_get_pfn(kvm_pfn_t pfn) 1616 { 1617 if (!kvm_is_reserved_pfn(pfn)) 1618 get_page(pfn_to_page(pfn)); 1619 } 1620 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1621 1622 static int next_segment(unsigned long len, int offset) 1623 { 1624 if (len > PAGE_SIZE - offset) 1625 return PAGE_SIZE - offset; 1626 else 1627 return len; 1628 } 1629 1630 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 1631 void *data, int offset, int len) 1632 { 1633 int r; 1634 unsigned long addr; 1635 1636 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1637 if (kvm_is_error_hva(addr)) 1638 return -EFAULT; 1639 r = __copy_from_user(data, (void __user *)addr + offset, len); 1640 if (r) 1641 return -EFAULT; 1642 return 0; 1643 } 1644 1645 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1646 int len) 1647 { 1648 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1649 1650 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1651 } 1652 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1653 1654 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 1655 int offset, int len) 1656 { 1657 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1658 1659 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1660 } 1661 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 1662 1663 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1664 { 1665 gfn_t gfn = gpa >> PAGE_SHIFT; 1666 int seg; 1667 int offset = offset_in_page(gpa); 1668 int ret; 1669 1670 while ((seg = next_segment(len, offset)) != 0) { 1671 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1672 if (ret < 0) 1673 return ret; 1674 offset = 0; 1675 len -= seg; 1676 data += seg; 1677 ++gfn; 1678 } 1679 return 0; 1680 } 1681 EXPORT_SYMBOL_GPL(kvm_read_guest); 1682 1683 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 1684 { 1685 gfn_t gfn = gpa >> PAGE_SHIFT; 1686 int seg; 1687 int offset = offset_in_page(gpa); 1688 int ret; 1689 1690 while ((seg = next_segment(len, offset)) != 0) { 1691 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 1692 if (ret < 0) 1693 return ret; 1694 offset = 0; 1695 len -= seg; 1696 data += seg; 1697 ++gfn; 1698 } 1699 return 0; 1700 } 1701 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 1702 1703 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1704 void *data, int offset, unsigned long len) 1705 { 1706 int r; 1707 unsigned long addr; 1708 1709 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1710 if (kvm_is_error_hva(addr)) 1711 return -EFAULT; 1712 pagefault_disable(); 1713 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1714 pagefault_enable(); 1715 if (r) 1716 return -EFAULT; 1717 return 0; 1718 } 1719 1720 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1721 unsigned long len) 1722 { 1723 gfn_t gfn = gpa >> PAGE_SHIFT; 1724 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1725 int offset = offset_in_page(gpa); 1726 1727 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1728 } 1729 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic); 1730 1731 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 1732 void *data, unsigned long len) 1733 { 1734 gfn_t gfn = gpa >> PAGE_SHIFT; 1735 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1736 int offset = offset_in_page(gpa); 1737 1738 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1739 } 1740 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 1741 1742 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn, 1743 const void *data, int offset, int len) 1744 { 1745 int r; 1746 unsigned long addr; 1747 1748 addr = gfn_to_hva_memslot(memslot, gfn); 1749 if (kvm_is_error_hva(addr)) 1750 return -EFAULT; 1751 r = __copy_to_user((void __user *)addr + offset, data, len); 1752 if (r) 1753 return -EFAULT; 1754 mark_page_dirty_in_slot(memslot, gfn); 1755 return 0; 1756 } 1757 1758 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 1759 const void *data, int offset, int len) 1760 { 1761 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1762 1763 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1764 } 1765 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1766 1767 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 1768 const void *data, int offset, int len) 1769 { 1770 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1771 1772 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1773 } 1774 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 1775 1776 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1777 unsigned long len) 1778 { 1779 gfn_t gfn = gpa >> PAGE_SHIFT; 1780 int seg; 1781 int offset = offset_in_page(gpa); 1782 int ret; 1783 1784 while ((seg = next_segment(len, offset)) != 0) { 1785 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1786 if (ret < 0) 1787 return ret; 1788 offset = 0; 1789 len -= seg; 1790 data += seg; 1791 ++gfn; 1792 } 1793 return 0; 1794 } 1795 EXPORT_SYMBOL_GPL(kvm_write_guest); 1796 1797 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1798 unsigned long len) 1799 { 1800 gfn_t gfn = gpa >> PAGE_SHIFT; 1801 int seg; 1802 int offset = offset_in_page(gpa); 1803 int ret; 1804 1805 while ((seg = next_segment(len, offset)) != 0) { 1806 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 1807 if (ret < 0) 1808 return ret; 1809 offset = 0; 1810 len -= seg; 1811 data += seg; 1812 ++gfn; 1813 } 1814 return 0; 1815 } 1816 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 1817 1818 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1819 gpa_t gpa, unsigned long len) 1820 { 1821 struct kvm_memslots *slots = kvm_memslots(kvm); 1822 int offset = offset_in_page(gpa); 1823 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1824 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1825 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1826 gfn_t nr_pages_avail; 1827 1828 ghc->gpa = gpa; 1829 ghc->generation = slots->generation; 1830 ghc->len = len; 1831 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1832 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL); 1833 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) { 1834 ghc->hva += offset; 1835 } else { 1836 /* 1837 * If the requested region crosses two memslots, we still 1838 * verify that the entire region is valid here. 1839 */ 1840 while (start_gfn <= end_gfn) { 1841 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1842 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1843 &nr_pages_avail); 1844 if (kvm_is_error_hva(ghc->hva)) 1845 return -EFAULT; 1846 start_gfn += nr_pages_avail; 1847 } 1848 /* Use the slow path for cross page reads and writes. */ 1849 ghc->memslot = NULL; 1850 } 1851 return 0; 1852 } 1853 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1854 1855 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1856 void *data, unsigned long len) 1857 { 1858 struct kvm_memslots *slots = kvm_memslots(kvm); 1859 int r; 1860 1861 BUG_ON(len > ghc->len); 1862 1863 if (slots->generation != ghc->generation) 1864 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1865 1866 if (unlikely(!ghc->memslot)) 1867 return kvm_write_guest(kvm, ghc->gpa, data, len); 1868 1869 if (kvm_is_error_hva(ghc->hva)) 1870 return -EFAULT; 1871 1872 r = __copy_to_user((void __user *)ghc->hva, data, len); 1873 if (r) 1874 return -EFAULT; 1875 mark_page_dirty_in_slot(ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1876 1877 return 0; 1878 } 1879 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1880 1881 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1882 void *data, unsigned long len) 1883 { 1884 struct kvm_memslots *slots = kvm_memslots(kvm); 1885 int r; 1886 1887 BUG_ON(len > ghc->len); 1888 1889 if (slots->generation != ghc->generation) 1890 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1891 1892 if (unlikely(!ghc->memslot)) 1893 return kvm_read_guest(kvm, ghc->gpa, data, len); 1894 1895 if (kvm_is_error_hva(ghc->hva)) 1896 return -EFAULT; 1897 1898 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1899 if (r) 1900 return -EFAULT; 1901 1902 return 0; 1903 } 1904 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1905 1906 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1907 { 1908 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 1909 1910 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 1911 } 1912 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1913 1914 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1915 { 1916 gfn_t gfn = gpa >> PAGE_SHIFT; 1917 int seg; 1918 int offset = offset_in_page(gpa); 1919 int ret; 1920 1921 while ((seg = next_segment(len, offset)) != 0) { 1922 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1923 if (ret < 0) 1924 return ret; 1925 offset = 0; 1926 len -= seg; 1927 ++gfn; 1928 } 1929 return 0; 1930 } 1931 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1932 1933 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, 1934 gfn_t gfn) 1935 { 1936 if (memslot && memslot->dirty_bitmap) { 1937 unsigned long rel_gfn = gfn - memslot->base_gfn; 1938 1939 set_bit_le(rel_gfn, memslot->dirty_bitmap); 1940 } 1941 } 1942 1943 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 1944 { 1945 struct kvm_memory_slot *memslot; 1946 1947 memslot = gfn_to_memslot(kvm, gfn); 1948 mark_page_dirty_in_slot(memslot, gfn); 1949 } 1950 EXPORT_SYMBOL_GPL(mark_page_dirty); 1951 1952 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 1953 { 1954 struct kvm_memory_slot *memslot; 1955 1956 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1957 mark_page_dirty_in_slot(memslot, gfn); 1958 } 1959 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 1960 1961 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) 1962 { 1963 unsigned int old, val, grow; 1964 1965 old = val = vcpu->halt_poll_ns; 1966 grow = READ_ONCE(halt_poll_ns_grow); 1967 /* 10us base */ 1968 if (val == 0 && grow) 1969 val = 10000; 1970 else 1971 val *= grow; 1972 1973 if (val > halt_poll_ns) 1974 val = halt_poll_ns; 1975 1976 vcpu->halt_poll_ns = val; 1977 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); 1978 } 1979 1980 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu) 1981 { 1982 unsigned int old, val, shrink; 1983 1984 old = val = vcpu->halt_poll_ns; 1985 shrink = READ_ONCE(halt_poll_ns_shrink); 1986 if (shrink == 0) 1987 val = 0; 1988 else 1989 val /= shrink; 1990 1991 vcpu->halt_poll_ns = val; 1992 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); 1993 } 1994 1995 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 1996 { 1997 if (kvm_arch_vcpu_runnable(vcpu)) { 1998 kvm_make_request(KVM_REQ_UNHALT, vcpu); 1999 return -EINTR; 2000 } 2001 if (kvm_cpu_has_pending_timer(vcpu)) 2002 return -EINTR; 2003 if (signal_pending(current)) 2004 return -EINTR; 2005 2006 return 0; 2007 } 2008 2009 /* 2010 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 2011 */ 2012 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 2013 { 2014 ktime_t start, cur; 2015 DECLARE_SWAITQUEUE(wait); 2016 bool waited = false; 2017 u64 block_ns; 2018 2019 start = cur = ktime_get(); 2020 if (vcpu->halt_poll_ns) { 2021 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); 2022 2023 ++vcpu->stat.halt_attempted_poll; 2024 do { 2025 /* 2026 * This sets KVM_REQ_UNHALT if an interrupt 2027 * arrives. 2028 */ 2029 if (kvm_vcpu_check_block(vcpu) < 0) { 2030 ++vcpu->stat.halt_successful_poll; 2031 goto out; 2032 } 2033 cur = ktime_get(); 2034 } while (single_task_running() && ktime_before(cur, stop)); 2035 } 2036 2037 kvm_arch_vcpu_blocking(vcpu); 2038 2039 for (;;) { 2040 prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 2041 2042 if (kvm_vcpu_check_block(vcpu) < 0) 2043 break; 2044 2045 waited = true; 2046 schedule(); 2047 } 2048 2049 finish_swait(&vcpu->wq, &wait); 2050 cur = ktime_get(); 2051 2052 kvm_arch_vcpu_unblocking(vcpu); 2053 out: 2054 block_ns = ktime_to_ns(cur) - ktime_to_ns(start); 2055 2056 if (halt_poll_ns) { 2057 if (block_ns <= vcpu->halt_poll_ns) 2058 ; 2059 /* we had a long block, shrink polling */ 2060 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns) 2061 shrink_halt_poll_ns(vcpu); 2062 /* we had a short halt and our poll time is too small */ 2063 else if (vcpu->halt_poll_ns < halt_poll_ns && 2064 block_ns < halt_poll_ns) 2065 grow_halt_poll_ns(vcpu); 2066 } else 2067 vcpu->halt_poll_ns = 0; 2068 2069 trace_kvm_vcpu_wakeup(block_ns, waited); 2070 } 2071 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 2072 2073 #ifndef CONFIG_S390 2074 /* 2075 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 2076 */ 2077 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 2078 { 2079 int me; 2080 int cpu = vcpu->cpu; 2081 struct swait_queue_head *wqp; 2082 2083 wqp = kvm_arch_vcpu_wq(vcpu); 2084 if (swait_active(wqp)) { 2085 swake_up(wqp); 2086 ++vcpu->stat.halt_wakeup; 2087 } 2088 2089 me = get_cpu(); 2090 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 2091 if (kvm_arch_vcpu_should_kick(vcpu)) 2092 smp_send_reschedule(cpu); 2093 put_cpu(); 2094 } 2095 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 2096 #endif /* !CONFIG_S390 */ 2097 2098 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 2099 { 2100 struct pid *pid; 2101 struct task_struct *task = NULL; 2102 int ret = 0; 2103 2104 rcu_read_lock(); 2105 pid = rcu_dereference(target->pid); 2106 if (pid) 2107 task = get_pid_task(pid, PIDTYPE_PID); 2108 rcu_read_unlock(); 2109 if (!task) 2110 return ret; 2111 ret = yield_to(task, 1); 2112 put_task_struct(task); 2113 2114 return ret; 2115 } 2116 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 2117 2118 /* 2119 * Helper that checks whether a VCPU is eligible for directed yield. 2120 * Most eligible candidate to yield is decided by following heuristics: 2121 * 2122 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 2123 * (preempted lock holder), indicated by @in_spin_loop. 2124 * Set at the beiginning and cleared at the end of interception/PLE handler. 2125 * 2126 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 2127 * chance last time (mostly it has become eligible now since we have probably 2128 * yielded to lockholder in last iteration. This is done by toggling 2129 * @dy_eligible each time a VCPU checked for eligibility.) 2130 * 2131 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 2132 * to preempted lock-holder could result in wrong VCPU selection and CPU 2133 * burning. Giving priority for a potential lock-holder increases lock 2134 * progress. 2135 * 2136 * Since algorithm is based on heuristics, accessing another VCPU data without 2137 * locking does not harm. It may result in trying to yield to same VCPU, fail 2138 * and continue with next VCPU and so on. 2139 */ 2140 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 2141 { 2142 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2143 bool eligible; 2144 2145 eligible = !vcpu->spin_loop.in_spin_loop || 2146 vcpu->spin_loop.dy_eligible; 2147 2148 if (vcpu->spin_loop.in_spin_loop) 2149 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 2150 2151 return eligible; 2152 #else 2153 return true; 2154 #endif 2155 } 2156 2157 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 2158 { 2159 struct kvm *kvm = me->kvm; 2160 struct kvm_vcpu *vcpu; 2161 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 2162 int yielded = 0; 2163 int try = 3; 2164 int pass; 2165 int i; 2166 2167 kvm_vcpu_set_in_spin_loop(me, true); 2168 /* 2169 * We boost the priority of a VCPU that is runnable but not 2170 * currently running, because it got preempted by something 2171 * else and called schedule in __vcpu_run. Hopefully that 2172 * VCPU is holding the lock that we need and will release it. 2173 * We approximate round-robin by starting at the last boosted VCPU. 2174 */ 2175 for (pass = 0; pass < 2 && !yielded && try; pass++) { 2176 kvm_for_each_vcpu(i, vcpu, kvm) { 2177 if (!pass && i <= last_boosted_vcpu) { 2178 i = last_boosted_vcpu; 2179 continue; 2180 } else if (pass && i > last_boosted_vcpu) 2181 break; 2182 if (!ACCESS_ONCE(vcpu->preempted)) 2183 continue; 2184 if (vcpu == me) 2185 continue; 2186 if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 2187 continue; 2188 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 2189 continue; 2190 2191 yielded = kvm_vcpu_yield_to(vcpu); 2192 if (yielded > 0) { 2193 kvm->last_boosted_vcpu = i; 2194 break; 2195 } else if (yielded < 0) { 2196 try--; 2197 if (!try) 2198 break; 2199 } 2200 } 2201 } 2202 kvm_vcpu_set_in_spin_loop(me, false); 2203 2204 /* Ensure vcpu is not eligible during next spinloop */ 2205 kvm_vcpu_set_dy_eligible(me, false); 2206 } 2207 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 2208 2209 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2210 { 2211 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 2212 struct page *page; 2213 2214 if (vmf->pgoff == 0) 2215 page = virt_to_page(vcpu->run); 2216 #ifdef CONFIG_X86 2217 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 2218 page = virt_to_page(vcpu->arch.pio_data); 2219 #endif 2220 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2221 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 2222 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 2223 #endif 2224 else 2225 return kvm_arch_vcpu_fault(vcpu, vmf); 2226 get_page(page); 2227 vmf->page = page; 2228 return 0; 2229 } 2230 2231 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 2232 .fault = kvm_vcpu_fault, 2233 }; 2234 2235 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 2236 { 2237 vma->vm_ops = &kvm_vcpu_vm_ops; 2238 return 0; 2239 } 2240 2241 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 2242 { 2243 struct kvm_vcpu *vcpu = filp->private_data; 2244 2245 kvm_put_kvm(vcpu->kvm); 2246 return 0; 2247 } 2248 2249 static struct file_operations kvm_vcpu_fops = { 2250 .release = kvm_vcpu_release, 2251 .unlocked_ioctl = kvm_vcpu_ioctl, 2252 #ifdef CONFIG_KVM_COMPAT 2253 .compat_ioctl = kvm_vcpu_compat_ioctl, 2254 #endif 2255 .mmap = kvm_vcpu_mmap, 2256 .llseek = noop_llseek, 2257 }; 2258 2259 /* 2260 * Allocates an inode for the vcpu. 2261 */ 2262 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 2263 { 2264 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 2265 } 2266 2267 /* 2268 * Creates some virtual cpus. Good luck creating more than one. 2269 */ 2270 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 2271 { 2272 int r; 2273 struct kvm_vcpu *vcpu; 2274 2275 if (id >= KVM_MAX_VCPUS) 2276 return -EINVAL; 2277 2278 vcpu = kvm_arch_vcpu_create(kvm, id); 2279 if (IS_ERR(vcpu)) 2280 return PTR_ERR(vcpu); 2281 2282 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 2283 2284 r = kvm_arch_vcpu_setup(vcpu); 2285 if (r) 2286 goto vcpu_destroy; 2287 2288 mutex_lock(&kvm->lock); 2289 if (!kvm_vcpu_compatible(vcpu)) { 2290 r = -EINVAL; 2291 goto unlock_vcpu_destroy; 2292 } 2293 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 2294 r = -EINVAL; 2295 goto unlock_vcpu_destroy; 2296 } 2297 if (kvm_get_vcpu_by_id(kvm, id)) { 2298 r = -EEXIST; 2299 goto unlock_vcpu_destroy; 2300 } 2301 2302 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 2303 2304 /* Now it's all set up, let userspace reach it */ 2305 kvm_get_kvm(kvm); 2306 r = create_vcpu_fd(vcpu); 2307 if (r < 0) { 2308 kvm_put_kvm(kvm); 2309 goto unlock_vcpu_destroy; 2310 } 2311 2312 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 2313 2314 /* 2315 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus 2316 * before kvm->online_vcpu's incremented value. 2317 */ 2318 smp_wmb(); 2319 atomic_inc(&kvm->online_vcpus); 2320 2321 mutex_unlock(&kvm->lock); 2322 kvm_arch_vcpu_postcreate(vcpu); 2323 return r; 2324 2325 unlock_vcpu_destroy: 2326 mutex_unlock(&kvm->lock); 2327 vcpu_destroy: 2328 kvm_arch_vcpu_destroy(vcpu); 2329 return r; 2330 } 2331 2332 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 2333 { 2334 if (sigset) { 2335 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2336 vcpu->sigset_active = 1; 2337 vcpu->sigset = *sigset; 2338 } else 2339 vcpu->sigset_active = 0; 2340 return 0; 2341 } 2342 2343 static long kvm_vcpu_ioctl(struct file *filp, 2344 unsigned int ioctl, unsigned long arg) 2345 { 2346 struct kvm_vcpu *vcpu = filp->private_data; 2347 void __user *argp = (void __user *)arg; 2348 int r; 2349 struct kvm_fpu *fpu = NULL; 2350 struct kvm_sregs *kvm_sregs = NULL; 2351 2352 if (vcpu->kvm->mm != current->mm) 2353 return -EIO; 2354 2355 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 2356 return -EINVAL; 2357 2358 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS) 2359 /* 2360 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 2361 * so vcpu_load() would break it. 2362 */ 2363 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT) 2364 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2365 #endif 2366 2367 2368 r = vcpu_load(vcpu); 2369 if (r) 2370 return r; 2371 switch (ioctl) { 2372 case KVM_RUN: 2373 r = -EINVAL; 2374 if (arg) 2375 goto out; 2376 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 2377 /* The thread running this VCPU changed. */ 2378 struct pid *oldpid = vcpu->pid; 2379 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 2380 2381 rcu_assign_pointer(vcpu->pid, newpid); 2382 if (oldpid) 2383 synchronize_rcu(); 2384 put_pid(oldpid); 2385 } 2386 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 2387 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 2388 break; 2389 case KVM_GET_REGS: { 2390 struct kvm_regs *kvm_regs; 2391 2392 r = -ENOMEM; 2393 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2394 if (!kvm_regs) 2395 goto out; 2396 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2397 if (r) 2398 goto out_free1; 2399 r = -EFAULT; 2400 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2401 goto out_free1; 2402 r = 0; 2403 out_free1: 2404 kfree(kvm_regs); 2405 break; 2406 } 2407 case KVM_SET_REGS: { 2408 struct kvm_regs *kvm_regs; 2409 2410 r = -ENOMEM; 2411 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2412 if (IS_ERR(kvm_regs)) { 2413 r = PTR_ERR(kvm_regs); 2414 goto out; 2415 } 2416 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2417 kfree(kvm_regs); 2418 break; 2419 } 2420 case KVM_GET_SREGS: { 2421 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2422 r = -ENOMEM; 2423 if (!kvm_sregs) 2424 goto out; 2425 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2426 if (r) 2427 goto out; 2428 r = -EFAULT; 2429 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2430 goto out; 2431 r = 0; 2432 break; 2433 } 2434 case KVM_SET_SREGS: { 2435 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2436 if (IS_ERR(kvm_sregs)) { 2437 r = PTR_ERR(kvm_sregs); 2438 kvm_sregs = NULL; 2439 goto out; 2440 } 2441 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2442 break; 2443 } 2444 case KVM_GET_MP_STATE: { 2445 struct kvm_mp_state mp_state; 2446 2447 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2448 if (r) 2449 goto out; 2450 r = -EFAULT; 2451 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 2452 goto out; 2453 r = 0; 2454 break; 2455 } 2456 case KVM_SET_MP_STATE: { 2457 struct kvm_mp_state mp_state; 2458 2459 r = -EFAULT; 2460 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 2461 goto out; 2462 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2463 break; 2464 } 2465 case KVM_TRANSLATE: { 2466 struct kvm_translation tr; 2467 2468 r = -EFAULT; 2469 if (copy_from_user(&tr, argp, sizeof(tr))) 2470 goto out; 2471 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2472 if (r) 2473 goto out; 2474 r = -EFAULT; 2475 if (copy_to_user(argp, &tr, sizeof(tr))) 2476 goto out; 2477 r = 0; 2478 break; 2479 } 2480 case KVM_SET_GUEST_DEBUG: { 2481 struct kvm_guest_debug dbg; 2482 2483 r = -EFAULT; 2484 if (copy_from_user(&dbg, argp, sizeof(dbg))) 2485 goto out; 2486 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2487 break; 2488 } 2489 case KVM_SET_SIGNAL_MASK: { 2490 struct kvm_signal_mask __user *sigmask_arg = argp; 2491 struct kvm_signal_mask kvm_sigmask; 2492 sigset_t sigset, *p; 2493 2494 p = NULL; 2495 if (argp) { 2496 r = -EFAULT; 2497 if (copy_from_user(&kvm_sigmask, argp, 2498 sizeof(kvm_sigmask))) 2499 goto out; 2500 r = -EINVAL; 2501 if (kvm_sigmask.len != sizeof(sigset)) 2502 goto out; 2503 r = -EFAULT; 2504 if (copy_from_user(&sigset, sigmask_arg->sigset, 2505 sizeof(sigset))) 2506 goto out; 2507 p = &sigset; 2508 } 2509 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2510 break; 2511 } 2512 case KVM_GET_FPU: { 2513 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2514 r = -ENOMEM; 2515 if (!fpu) 2516 goto out; 2517 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2518 if (r) 2519 goto out; 2520 r = -EFAULT; 2521 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2522 goto out; 2523 r = 0; 2524 break; 2525 } 2526 case KVM_SET_FPU: { 2527 fpu = memdup_user(argp, sizeof(*fpu)); 2528 if (IS_ERR(fpu)) { 2529 r = PTR_ERR(fpu); 2530 fpu = NULL; 2531 goto out; 2532 } 2533 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2534 break; 2535 } 2536 default: 2537 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2538 } 2539 out: 2540 vcpu_put(vcpu); 2541 kfree(fpu); 2542 kfree(kvm_sregs); 2543 return r; 2544 } 2545 2546 #ifdef CONFIG_KVM_COMPAT 2547 static long kvm_vcpu_compat_ioctl(struct file *filp, 2548 unsigned int ioctl, unsigned long arg) 2549 { 2550 struct kvm_vcpu *vcpu = filp->private_data; 2551 void __user *argp = compat_ptr(arg); 2552 int r; 2553 2554 if (vcpu->kvm->mm != current->mm) 2555 return -EIO; 2556 2557 switch (ioctl) { 2558 case KVM_SET_SIGNAL_MASK: { 2559 struct kvm_signal_mask __user *sigmask_arg = argp; 2560 struct kvm_signal_mask kvm_sigmask; 2561 compat_sigset_t csigset; 2562 sigset_t sigset; 2563 2564 if (argp) { 2565 r = -EFAULT; 2566 if (copy_from_user(&kvm_sigmask, argp, 2567 sizeof(kvm_sigmask))) 2568 goto out; 2569 r = -EINVAL; 2570 if (kvm_sigmask.len != sizeof(csigset)) 2571 goto out; 2572 r = -EFAULT; 2573 if (copy_from_user(&csigset, sigmask_arg->sigset, 2574 sizeof(csigset))) 2575 goto out; 2576 sigset_from_compat(&sigset, &csigset); 2577 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2578 } else 2579 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2580 break; 2581 } 2582 default: 2583 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2584 } 2585 2586 out: 2587 return r; 2588 } 2589 #endif 2590 2591 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2592 int (*accessor)(struct kvm_device *dev, 2593 struct kvm_device_attr *attr), 2594 unsigned long arg) 2595 { 2596 struct kvm_device_attr attr; 2597 2598 if (!accessor) 2599 return -EPERM; 2600 2601 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2602 return -EFAULT; 2603 2604 return accessor(dev, &attr); 2605 } 2606 2607 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2608 unsigned long arg) 2609 { 2610 struct kvm_device *dev = filp->private_data; 2611 2612 switch (ioctl) { 2613 case KVM_SET_DEVICE_ATTR: 2614 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2615 case KVM_GET_DEVICE_ATTR: 2616 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2617 case KVM_HAS_DEVICE_ATTR: 2618 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2619 default: 2620 if (dev->ops->ioctl) 2621 return dev->ops->ioctl(dev, ioctl, arg); 2622 2623 return -ENOTTY; 2624 } 2625 } 2626 2627 static int kvm_device_release(struct inode *inode, struct file *filp) 2628 { 2629 struct kvm_device *dev = filp->private_data; 2630 struct kvm *kvm = dev->kvm; 2631 2632 kvm_put_kvm(kvm); 2633 return 0; 2634 } 2635 2636 static const struct file_operations kvm_device_fops = { 2637 .unlocked_ioctl = kvm_device_ioctl, 2638 #ifdef CONFIG_KVM_COMPAT 2639 .compat_ioctl = kvm_device_ioctl, 2640 #endif 2641 .release = kvm_device_release, 2642 }; 2643 2644 struct kvm_device *kvm_device_from_filp(struct file *filp) 2645 { 2646 if (filp->f_op != &kvm_device_fops) 2647 return NULL; 2648 2649 return filp->private_data; 2650 } 2651 2652 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 2653 #ifdef CONFIG_KVM_MPIC 2654 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 2655 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 2656 #endif 2657 2658 #ifdef CONFIG_KVM_XICS 2659 [KVM_DEV_TYPE_XICS] = &kvm_xics_ops, 2660 #endif 2661 }; 2662 2663 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type) 2664 { 2665 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 2666 return -ENOSPC; 2667 2668 if (kvm_device_ops_table[type] != NULL) 2669 return -EEXIST; 2670 2671 kvm_device_ops_table[type] = ops; 2672 return 0; 2673 } 2674 2675 void kvm_unregister_device_ops(u32 type) 2676 { 2677 if (kvm_device_ops_table[type] != NULL) 2678 kvm_device_ops_table[type] = NULL; 2679 } 2680 2681 static int kvm_ioctl_create_device(struct kvm *kvm, 2682 struct kvm_create_device *cd) 2683 { 2684 struct kvm_device_ops *ops = NULL; 2685 struct kvm_device *dev; 2686 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2687 int ret; 2688 2689 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 2690 return -ENODEV; 2691 2692 ops = kvm_device_ops_table[cd->type]; 2693 if (ops == NULL) 2694 return -ENODEV; 2695 2696 if (test) 2697 return 0; 2698 2699 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2700 if (!dev) 2701 return -ENOMEM; 2702 2703 dev->ops = ops; 2704 dev->kvm = kvm; 2705 2706 ret = ops->create(dev, cd->type); 2707 if (ret < 0) { 2708 kfree(dev); 2709 return ret; 2710 } 2711 2712 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2713 if (ret < 0) { 2714 ops->destroy(dev); 2715 return ret; 2716 } 2717 2718 list_add(&dev->vm_node, &kvm->devices); 2719 kvm_get_kvm(kvm); 2720 cd->fd = ret; 2721 return 0; 2722 } 2723 2724 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 2725 { 2726 switch (arg) { 2727 case KVM_CAP_USER_MEMORY: 2728 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2729 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2730 case KVM_CAP_INTERNAL_ERROR_DATA: 2731 #ifdef CONFIG_HAVE_KVM_MSI 2732 case KVM_CAP_SIGNAL_MSI: 2733 #endif 2734 #ifdef CONFIG_HAVE_KVM_IRQFD 2735 case KVM_CAP_IRQFD: 2736 case KVM_CAP_IRQFD_RESAMPLE: 2737 #endif 2738 case KVM_CAP_IOEVENTFD_ANY_LENGTH: 2739 case KVM_CAP_CHECK_EXTENSION_VM: 2740 return 1; 2741 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2742 case KVM_CAP_IRQ_ROUTING: 2743 return KVM_MAX_IRQ_ROUTES; 2744 #endif 2745 #if KVM_ADDRESS_SPACE_NUM > 1 2746 case KVM_CAP_MULTI_ADDRESS_SPACE: 2747 return KVM_ADDRESS_SPACE_NUM; 2748 #endif 2749 default: 2750 break; 2751 } 2752 return kvm_vm_ioctl_check_extension(kvm, arg); 2753 } 2754 2755 static long kvm_vm_ioctl(struct file *filp, 2756 unsigned int ioctl, unsigned long arg) 2757 { 2758 struct kvm *kvm = filp->private_data; 2759 void __user *argp = (void __user *)arg; 2760 int r; 2761 2762 if (kvm->mm != current->mm) 2763 return -EIO; 2764 switch (ioctl) { 2765 case KVM_CREATE_VCPU: 2766 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2767 break; 2768 case KVM_SET_USER_MEMORY_REGION: { 2769 struct kvm_userspace_memory_region kvm_userspace_mem; 2770 2771 r = -EFAULT; 2772 if (copy_from_user(&kvm_userspace_mem, argp, 2773 sizeof(kvm_userspace_mem))) 2774 goto out; 2775 2776 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2777 break; 2778 } 2779 case KVM_GET_DIRTY_LOG: { 2780 struct kvm_dirty_log log; 2781 2782 r = -EFAULT; 2783 if (copy_from_user(&log, argp, sizeof(log))) 2784 goto out; 2785 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2786 break; 2787 } 2788 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2789 case KVM_REGISTER_COALESCED_MMIO: { 2790 struct kvm_coalesced_mmio_zone zone; 2791 2792 r = -EFAULT; 2793 if (copy_from_user(&zone, argp, sizeof(zone))) 2794 goto out; 2795 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2796 break; 2797 } 2798 case KVM_UNREGISTER_COALESCED_MMIO: { 2799 struct kvm_coalesced_mmio_zone zone; 2800 2801 r = -EFAULT; 2802 if (copy_from_user(&zone, argp, sizeof(zone))) 2803 goto out; 2804 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2805 break; 2806 } 2807 #endif 2808 case KVM_IRQFD: { 2809 struct kvm_irqfd data; 2810 2811 r = -EFAULT; 2812 if (copy_from_user(&data, argp, sizeof(data))) 2813 goto out; 2814 r = kvm_irqfd(kvm, &data); 2815 break; 2816 } 2817 case KVM_IOEVENTFD: { 2818 struct kvm_ioeventfd data; 2819 2820 r = -EFAULT; 2821 if (copy_from_user(&data, argp, sizeof(data))) 2822 goto out; 2823 r = kvm_ioeventfd(kvm, &data); 2824 break; 2825 } 2826 #ifdef CONFIG_HAVE_KVM_MSI 2827 case KVM_SIGNAL_MSI: { 2828 struct kvm_msi msi; 2829 2830 r = -EFAULT; 2831 if (copy_from_user(&msi, argp, sizeof(msi))) 2832 goto out; 2833 r = kvm_send_userspace_msi(kvm, &msi); 2834 break; 2835 } 2836 #endif 2837 #ifdef __KVM_HAVE_IRQ_LINE 2838 case KVM_IRQ_LINE_STATUS: 2839 case KVM_IRQ_LINE: { 2840 struct kvm_irq_level irq_event; 2841 2842 r = -EFAULT; 2843 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 2844 goto out; 2845 2846 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 2847 ioctl == KVM_IRQ_LINE_STATUS); 2848 if (r) 2849 goto out; 2850 2851 r = -EFAULT; 2852 if (ioctl == KVM_IRQ_LINE_STATUS) { 2853 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 2854 goto out; 2855 } 2856 2857 r = 0; 2858 break; 2859 } 2860 #endif 2861 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2862 case KVM_SET_GSI_ROUTING: { 2863 struct kvm_irq_routing routing; 2864 struct kvm_irq_routing __user *urouting; 2865 struct kvm_irq_routing_entry *entries; 2866 2867 r = -EFAULT; 2868 if (copy_from_user(&routing, argp, sizeof(routing))) 2869 goto out; 2870 r = -EINVAL; 2871 if (routing.nr >= KVM_MAX_IRQ_ROUTES) 2872 goto out; 2873 if (routing.flags) 2874 goto out; 2875 r = -ENOMEM; 2876 entries = vmalloc(routing.nr * sizeof(*entries)); 2877 if (!entries) 2878 goto out; 2879 r = -EFAULT; 2880 urouting = argp; 2881 if (copy_from_user(entries, urouting->entries, 2882 routing.nr * sizeof(*entries))) 2883 goto out_free_irq_routing; 2884 r = kvm_set_irq_routing(kvm, entries, routing.nr, 2885 routing.flags); 2886 out_free_irq_routing: 2887 vfree(entries); 2888 break; 2889 } 2890 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 2891 case KVM_CREATE_DEVICE: { 2892 struct kvm_create_device cd; 2893 2894 r = -EFAULT; 2895 if (copy_from_user(&cd, argp, sizeof(cd))) 2896 goto out; 2897 2898 r = kvm_ioctl_create_device(kvm, &cd); 2899 if (r) 2900 goto out; 2901 2902 r = -EFAULT; 2903 if (copy_to_user(argp, &cd, sizeof(cd))) 2904 goto out; 2905 2906 r = 0; 2907 break; 2908 } 2909 case KVM_CHECK_EXTENSION: 2910 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 2911 break; 2912 default: 2913 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2914 } 2915 out: 2916 return r; 2917 } 2918 2919 #ifdef CONFIG_KVM_COMPAT 2920 struct compat_kvm_dirty_log { 2921 __u32 slot; 2922 __u32 padding1; 2923 union { 2924 compat_uptr_t dirty_bitmap; /* one bit per page */ 2925 __u64 padding2; 2926 }; 2927 }; 2928 2929 static long kvm_vm_compat_ioctl(struct file *filp, 2930 unsigned int ioctl, unsigned long arg) 2931 { 2932 struct kvm *kvm = filp->private_data; 2933 int r; 2934 2935 if (kvm->mm != current->mm) 2936 return -EIO; 2937 switch (ioctl) { 2938 case KVM_GET_DIRTY_LOG: { 2939 struct compat_kvm_dirty_log compat_log; 2940 struct kvm_dirty_log log; 2941 2942 r = -EFAULT; 2943 if (copy_from_user(&compat_log, (void __user *)arg, 2944 sizeof(compat_log))) 2945 goto out; 2946 log.slot = compat_log.slot; 2947 log.padding1 = compat_log.padding1; 2948 log.padding2 = compat_log.padding2; 2949 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 2950 2951 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2952 break; 2953 } 2954 default: 2955 r = kvm_vm_ioctl(filp, ioctl, arg); 2956 } 2957 2958 out: 2959 return r; 2960 } 2961 #endif 2962 2963 static struct file_operations kvm_vm_fops = { 2964 .release = kvm_vm_release, 2965 .unlocked_ioctl = kvm_vm_ioctl, 2966 #ifdef CONFIG_KVM_COMPAT 2967 .compat_ioctl = kvm_vm_compat_ioctl, 2968 #endif 2969 .llseek = noop_llseek, 2970 }; 2971 2972 static int kvm_dev_ioctl_create_vm(unsigned long type) 2973 { 2974 int r; 2975 struct kvm *kvm; 2976 2977 kvm = kvm_create_vm(type); 2978 if (IS_ERR(kvm)) 2979 return PTR_ERR(kvm); 2980 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2981 r = kvm_coalesced_mmio_init(kvm); 2982 if (r < 0) { 2983 kvm_put_kvm(kvm); 2984 return r; 2985 } 2986 #endif 2987 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC); 2988 if (r < 0) 2989 kvm_put_kvm(kvm); 2990 2991 return r; 2992 } 2993 2994 static long kvm_dev_ioctl(struct file *filp, 2995 unsigned int ioctl, unsigned long arg) 2996 { 2997 long r = -EINVAL; 2998 2999 switch (ioctl) { 3000 case KVM_GET_API_VERSION: 3001 if (arg) 3002 goto out; 3003 r = KVM_API_VERSION; 3004 break; 3005 case KVM_CREATE_VM: 3006 r = kvm_dev_ioctl_create_vm(arg); 3007 break; 3008 case KVM_CHECK_EXTENSION: 3009 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 3010 break; 3011 case KVM_GET_VCPU_MMAP_SIZE: 3012 if (arg) 3013 goto out; 3014 r = PAGE_SIZE; /* struct kvm_run */ 3015 #ifdef CONFIG_X86 3016 r += PAGE_SIZE; /* pio data page */ 3017 #endif 3018 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 3019 r += PAGE_SIZE; /* coalesced mmio ring page */ 3020 #endif 3021 break; 3022 case KVM_TRACE_ENABLE: 3023 case KVM_TRACE_PAUSE: 3024 case KVM_TRACE_DISABLE: 3025 r = -EOPNOTSUPP; 3026 break; 3027 default: 3028 return kvm_arch_dev_ioctl(filp, ioctl, arg); 3029 } 3030 out: 3031 return r; 3032 } 3033 3034 static struct file_operations kvm_chardev_ops = { 3035 .unlocked_ioctl = kvm_dev_ioctl, 3036 .compat_ioctl = kvm_dev_ioctl, 3037 .llseek = noop_llseek, 3038 }; 3039 3040 static struct miscdevice kvm_dev = { 3041 KVM_MINOR, 3042 "kvm", 3043 &kvm_chardev_ops, 3044 }; 3045 3046 static void hardware_enable_nolock(void *junk) 3047 { 3048 int cpu = raw_smp_processor_id(); 3049 int r; 3050 3051 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3052 return; 3053 3054 cpumask_set_cpu(cpu, cpus_hardware_enabled); 3055 3056 r = kvm_arch_hardware_enable(); 3057 3058 if (r) { 3059 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3060 atomic_inc(&hardware_enable_failed); 3061 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu); 3062 } 3063 } 3064 3065 static void hardware_enable(void) 3066 { 3067 raw_spin_lock(&kvm_count_lock); 3068 if (kvm_usage_count) 3069 hardware_enable_nolock(NULL); 3070 raw_spin_unlock(&kvm_count_lock); 3071 } 3072 3073 static void hardware_disable_nolock(void *junk) 3074 { 3075 int cpu = raw_smp_processor_id(); 3076 3077 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3078 return; 3079 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3080 kvm_arch_hardware_disable(); 3081 } 3082 3083 static void hardware_disable(void) 3084 { 3085 raw_spin_lock(&kvm_count_lock); 3086 if (kvm_usage_count) 3087 hardware_disable_nolock(NULL); 3088 raw_spin_unlock(&kvm_count_lock); 3089 } 3090 3091 static void hardware_disable_all_nolock(void) 3092 { 3093 BUG_ON(!kvm_usage_count); 3094 3095 kvm_usage_count--; 3096 if (!kvm_usage_count) 3097 on_each_cpu(hardware_disable_nolock, NULL, 1); 3098 } 3099 3100 static void hardware_disable_all(void) 3101 { 3102 raw_spin_lock(&kvm_count_lock); 3103 hardware_disable_all_nolock(); 3104 raw_spin_unlock(&kvm_count_lock); 3105 } 3106 3107 static int hardware_enable_all(void) 3108 { 3109 int r = 0; 3110 3111 raw_spin_lock(&kvm_count_lock); 3112 3113 kvm_usage_count++; 3114 if (kvm_usage_count == 1) { 3115 atomic_set(&hardware_enable_failed, 0); 3116 on_each_cpu(hardware_enable_nolock, NULL, 1); 3117 3118 if (atomic_read(&hardware_enable_failed)) { 3119 hardware_disable_all_nolock(); 3120 r = -EBUSY; 3121 } 3122 } 3123 3124 raw_spin_unlock(&kvm_count_lock); 3125 3126 return r; 3127 } 3128 3129 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 3130 void *v) 3131 { 3132 val &= ~CPU_TASKS_FROZEN; 3133 switch (val) { 3134 case CPU_DYING: 3135 hardware_disable(); 3136 break; 3137 case CPU_STARTING: 3138 hardware_enable(); 3139 break; 3140 } 3141 return NOTIFY_OK; 3142 } 3143 3144 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 3145 void *v) 3146 { 3147 /* 3148 * Some (well, at least mine) BIOSes hang on reboot if 3149 * in vmx root mode. 3150 * 3151 * And Intel TXT required VMX off for all cpu when system shutdown. 3152 */ 3153 pr_info("kvm: exiting hardware virtualization\n"); 3154 kvm_rebooting = true; 3155 on_each_cpu(hardware_disable_nolock, NULL, 1); 3156 return NOTIFY_OK; 3157 } 3158 3159 static struct notifier_block kvm_reboot_notifier = { 3160 .notifier_call = kvm_reboot, 3161 .priority = 0, 3162 }; 3163 3164 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 3165 { 3166 int i; 3167 3168 for (i = 0; i < bus->dev_count; i++) { 3169 struct kvm_io_device *pos = bus->range[i].dev; 3170 3171 kvm_iodevice_destructor(pos); 3172 } 3173 kfree(bus); 3174 } 3175 3176 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 3177 const struct kvm_io_range *r2) 3178 { 3179 gpa_t addr1 = r1->addr; 3180 gpa_t addr2 = r2->addr; 3181 3182 if (addr1 < addr2) 3183 return -1; 3184 3185 /* If r2->len == 0, match the exact address. If r2->len != 0, 3186 * accept any overlapping write. Any order is acceptable for 3187 * overlapping ranges, because kvm_io_bus_get_first_dev ensures 3188 * we process all of them. 3189 */ 3190 if (r2->len) { 3191 addr1 += r1->len; 3192 addr2 += r2->len; 3193 } 3194 3195 if (addr1 > addr2) 3196 return 1; 3197 3198 return 0; 3199 } 3200 3201 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 3202 { 3203 return kvm_io_bus_cmp(p1, p2); 3204 } 3205 3206 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 3207 gpa_t addr, int len) 3208 { 3209 bus->range[bus->dev_count++] = (struct kvm_io_range) { 3210 .addr = addr, 3211 .len = len, 3212 .dev = dev, 3213 }; 3214 3215 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 3216 kvm_io_bus_sort_cmp, NULL); 3217 3218 return 0; 3219 } 3220 3221 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 3222 gpa_t addr, int len) 3223 { 3224 struct kvm_io_range *range, key; 3225 int off; 3226 3227 key = (struct kvm_io_range) { 3228 .addr = addr, 3229 .len = len, 3230 }; 3231 3232 range = bsearch(&key, bus->range, bus->dev_count, 3233 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 3234 if (range == NULL) 3235 return -ENOENT; 3236 3237 off = range - bus->range; 3238 3239 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 3240 off--; 3241 3242 return off; 3243 } 3244 3245 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3246 struct kvm_io_range *range, const void *val) 3247 { 3248 int idx; 3249 3250 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3251 if (idx < 0) 3252 return -EOPNOTSUPP; 3253 3254 while (idx < bus->dev_count && 3255 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3256 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 3257 range->len, val)) 3258 return idx; 3259 idx++; 3260 } 3261 3262 return -EOPNOTSUPP; 3263 } 3264 3265 /* kvm_io_bus_write - called under kvm->slots_lock */ 3266 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3267 int len, const void *val) 3268 { 3269 struct kvm_io_bus *bus; 3270 struct kvm_io_range range; 3271 int r; 3272 3273 range = (struct kvm_io_range) { 3274 .addr = addr, 3275 .len = len, 3276 }; 3277 3278 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3279 r = __kvm_io_bus_write(vcpu, bus, &range, val); 3280 return r < 0 ? r : 0; 3281 } 3282 3283 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 3284 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 3285 gpa_t addr, int len, const void *val, long cookie) 3286 { 3287 struct kvm_io_bus *bus; 3288 struct kvm_io_range range; 3289 3290 range = (struct kvm_io_range) { 3291 .addr = addr, 3292 .len = len, 3293 }; 3294 3295 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3296 3297 /* First try the device referenced by cookie. */ 3298 if ((cookie >= 0) && (cookie < bus->dev_count) && 3299 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 3300 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 3301 val)) 3302 return cookie; 3303 3304 /* 3305 * cookie contained garbage; fall back to search and return the 3306 * correct cookie value. 3307 */ 3308 return __kvm_io_bus_write(vcpu, bus, &range, val); 3309 } 3310 3311 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3312 struct kvm_io_range *range, void *val) 3313 { 3314 int idx; 3315 3316 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3317 if (idx < 0) 3318 return -EOPNOTSUPP; 3319 3320 while (idx < bus->dev_count && 3321 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3322 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 3323 range->len, val)) 3324 return idx; 3325 idx++; 3326 } 3327 3328 return -EOPNOTSUPP; 3329 } 3330 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3331 3332 /* kvm_io_bus_read - called under kvm->slots_lock */ 3333 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3334 int len, void *val) 3335 { 3336 struct kvm_io_bus *bus; 3337 struct kvm_io_range range; 3338 int r; 3339 3340 range = (struct kvm_io_range) { 3341 .addr = addr, 3342 .len = len, 3343 }; 3344 3345 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3346 r = __kvm_io_bus_read(vcpu, bus, &range, val); 3347 return r < 0 ? r : 0; 3348 } 3349 3350 3351 /* Caller must hold slots_lock. */ 3352 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3353 int len, struct kvm_io_device *dev) 3354 { 3355 struct kvm_io_bus *new_bus, *bus; 3356 3357 bus = kvm->buses[bus_idx]; 3358 /* exclude ioeventfd which is limited by maximum fd */ 3359 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 3360 return -ENOSPC; 3361 3362 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) * 3363 sizeof(struct kvm_io_range)), GFP_KERNEL); 3364 if (!new_bus) 3365 return -ENOMEM; 3366 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 3367 sizeof(struct kvm_io_range))); 3368 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 3369 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3370 synchronize_srcu_expedited(&kvm->srcu); 3371 kfree(bus); 3372 3373 return 0; 3374 } 3375 3376 /* Caller must hold slots_lock. */ 3377 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3378 struct kvm_io_device *dev) 3379 { 3380 int i, r; 3381 struct kvm_io_bus *new_bus, *bus; 3382 3383 bus = kvm->buses[bus_idx]; 3384 r = -ENOENT; 3385 for (i = 0; i < bus->dev_count; i++) 3386 if (bus->range[i].dev == dev) { 3387 r = 0; 3388 break; 3389 } 3390 3391 if (r) 3392 return r; 3393 3394 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) * 3395 sizeof(struct kvm_io_range)), GFP_KERNEL); 3396 if (!new_bus) 3397 return -ENOMEM; 3398 3399 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3400 new_bus->dev_count--; 3401 memcpy(new_bus->range + i, bus->range + i + 1, 3402 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3403 3404 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3405 synchronize_srcu_expedited(&kvm->srcu); 3406 kfree(bus); 3407 return r; 3408 } 3409 3410 static struct notifier_block kvm_cpu_notifier = { 3411 .notifier_call = kvm_cpu_hotplug, 3412 }; 3413 3414 static int vm_stat_get(void *_offset, u64 *val) 3415 { 3416 unsigned offset = (long)_offset; 3417 struct kvm *kvm; 3418 3419 *val = 0; 3420 spin_lock(&kvm_lock); 3421 list_for_each_entry(kvm, &vm_list, vm_list) 3422 *val += *(u32 *)((void *)kvm + offset); 3423 spin_unlock(&kvm_lock); 3424 return 0; 3425 } 3426 3427 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 3428 3429 static int vcpu_stat_get(void *_offset, u64 *val) 3430 { 3431 unsigned offset = (long)_offset; 3432 struct kvm *kvm; 3433 struct kvm_vcpu *vcpu; 3434 int i; 3435 3436 *val = 0; 3437 spin_lock(&kvm_lock); 3438 list_for_each_entry(kvm, &vm_list, vm_list) 3439 kvm_for_each_vcpu(i, vcpu, kvm) 3440 *val += *(u32 *)((void *)vcpu + offset); 3441 3442 spin_unlock(&kvm_lock); 3443 return 0; 3444 } 3445 3446 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 3447 3448 static const struct file_operations *stat_fops[] = { 3449 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3450 [KVM_STAT_VM] = &vm_stat_fops, 3451 }; 3452 3453 static int kvm_init_debug(void) 3454 { 3455 int r = -EEXIST; 3456 struct kvm_stats_debugfs_item *p; 3457 3458 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3459 if (kvm_debugfs_dir == NULL) 3460 goto out; 3461 3462 for (p = debugfs_entries; p->name; ++p) { 3463 if (!debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 3464 (void *)(long)p->offset, 3465 stat_fops[p->kind])) 3466 goto out_dir; 3467 } 3468 3469 return 0; 3470 3471 out_dir: 3472 debugfs_remove_recursive(kvm_debugfs_dir); 3473 out: 3474 return r; 3475 } 3476 3477 static int kvm_suspend(void) 3478 { 3479 if (kvm_usage_count) 3480 hardware_disable_nolock(NULL); 3481 return 0; 3482 } 3483 3484 static void kvm_resume(void) 3485 { 3486 if (kvm_usage_count) { 3487 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3488 hardware_enable_nolock(NULL); 3489 } 3490 } 3491 3492 static struct syscore_ops kvm_syscore_ops = { 3493 .suspend = kvm_suspend, 3494 .resume = kvm_resume, 3495 }; 3496 3497 static inline 3498 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3499 { 3500 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3501 } 3502 3503 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3504 { 3505 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3506 3507 if (vcpu->preempted) 3508 vcpu->preempted = false; 3509 3510 kvm_arch_sched_in(vcpu, cpu); 3511 3512 kvm_arch_vcpu_load(vcpu, cpu); 3513 } 3514 3515 static void kvm_sched_out(struct preempt_notifier *pn, 3516 struct task_struct *next) 3517 { 3518 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3519 3520 if (current->state == TASK_RUNNING) 3521 vcpu->preempted = true; 3522 kvm_arch_vcpu_put(vcpu); 3523 } 3524 3525 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3526 struct module *module) 3527 { 3528 int r; 3529 int cpu; 3530 3531 r = kvm_arch_init(opaque); 3532 if (r) 3533 goto out_fail; 3534 3535 /* 3536 * kvm_arch_init makes sure there's at most one caller 3537 * for architectures that support multiple implementations, 3538 * like intel and amd on x86. 3539 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3540 * conflicts in case kvm is already setup for another implementation. 3541 */ 3542 r = kvm_irqfd_init(); 3543 if (r) 3544 goto out_irqfd; 3545 3546 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 3547 r = -ENOMEM; 3548 goto out_free_0; 3549 } 3550 3551 r = kvm_arch_hardware_setup(); 3552 if (r < 0) 3553 goto out_free_0a; 3554 3555 for_each_online_cpu(cpu) { 3556 smp_call_function_single(cpu, 3557 kvm_arch_check_processor_compat, 3558 &r, 1); 3559 if (r < 0) 3560 goto out_free_1; 3561 } 3562 3563 r = register_cpu_notifier(&kvm_cpu_notifier); 3564 if (r) 3565 goto out_free_2; 3566 register_reboot_notifier(&kvm_reboot_notifier); 3567 3568 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 3569 if (!vcpu_align) 3570 vcpu_align = __alignof__(struct kvm_vcpu); 3571 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 3572 0, NULL); 3573 if (!kvm_vcpu_cache) { 3574 r = -ENOMEM; 3575 goto out_free_3; 3576 } 3577 3578 r = kvm_async_pf_init(); 3579 if (r) 3580 goto out_free; 3581 3582 kvm_chardev_ops.owner = module; 3583 kvm_vm_fops.owner = module; 3584 kvm_vcpu_fops.owner = module; 3585 3586 r = misc_register(&kvm_dev); 3587 if (r) { 3588 pr_err("kvm: misc device register failed\n"); 3589 goto out_unreg; 3590 } 3591 3592 register_syscore_ops(&kvm_syscore_ops); 3593 3594 kvm_preempt_ops.sched_in = kvm_sched_in; 3595 kvm_preempt_ops.sched_out = kvm_sched_out; 3596 3597 r = kvm_init_debug(); 3598 if (r) { 3599 pr_err("kvm: create debugfs files failed\n"); 3600 goto out_undebugfs; 3601 } 3602 3603 r = kvm_vfio_ops_init(); 3604 WARN_ON(r); 3605 3606 return 0; 3607 3608 out_undebugfs: 3609 unregister_syscore_ops(&kvm_syscore_ops); 3610 misc_deregister(&kvm_dev); 3611 out_unreg: 3612 kvm_async_pf_deinit(); 3613 out_free: 3614 kmem_cache_destroy(kvm_vcpu_cache); 3615 out_free_3: 3616 unregister_reboot_notifier(&kvm_reboot_notifier); 3617 unregister_cpu_notifier(&kvm_cpu_notifier); 3618 out_free_2: 3619 out_free_1: 3620 kvm_arch_hardware_unsetup(); 3621 out_free_0a: 3622 free_cpumask_var(cpus_hardware_enabled); 3623 out_free_0: 3624 kvm_irqfd_exit(); 3625 out_irqfd: 3626 kvm_arch_exit(); 3627 out_fail: 3628 return r; 3629 } 3630 EXPORT_SYMBOL_GPL(kvm_init); 3631 3632 void kvm_exit(void) 3633 { 3634 debugfs_remove_recursive(kvm_debugfs_dir); 3635 misc_deregister(&kvm_dev); 3636 kmem_cache_destroy(kvm_vcpu_cache); 3637 kvm_async_pf_deinit(); 3638 unregister_syscore_ops(&kvm_syscore_ops); 3639 unregister_reboot_notifier(&kvm_reboot_notifier); 3640 unregister_cpu_notifier(&kvm_cpu_notifier); 3641 on_each_cpu(hardware_disable_nolock, NULL, 1); 3642 kvm_arch_hardware_unsetup(); 3643 kvm_arch_exit(); 3644 kvm_irqfd_exit(); 3645 free_cpumask_var(cpus_hardware_enabled); 3646 kvm_vfio_ops_exit(); 3647 } 3648 EXPORT_SYMBOL_GPL(kvm_exit); 3649