1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Avi Kivity <avi@qumranet.com> 12 * Yaniv Kamay <yaniv@qumranet.com> 13 * 14 * This work is licensed under the terms of the GNU GPL, version 2. See 15 * the COPYING file in the top-level directory. 16 * 17 */ 18 19 #include <kvm/iodev.h> 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/percpu.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/syscore_ops.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 #include <linux/bitops.h> 45 #include <linux/spinlock.h> 46 #include <linux/compat.h> 47 #include <linux/srcu.h> 48 #include <linux/hugetlb.h> 49 #include <linux/slab.h> 50 #include <linux/sort.h> 51 #include <linux/bsearch.h> 52 53 #include <asm/processor.h> 54 #include <asm/io.h> 55 #include <asm/ioctl.h> 56 #include <asm/uaccess.h> 57 #include <asm/pgtable.h> 58 59 #include "coalesced_mmio.h" 60 #include "async_pf.h" 61 #include "vfio.h" 62 63 #define CREATE_TRACE_POINTS 64 #include <trace/events/kvm.h> 65 66 /* Worst case buffer size needed for holding an integer. */ 67 #define ITOA_MAX_LEN 12 68 69 MODULE_AUTHOR("Qumranet"); 70 MODULE_LICENSE("GPL"); 71 72 /* Architectures should define their poll value according to the halt latency */ 73 static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; 74 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR); 75 76 /* Default doubles per-vcpu halt_poll_ns. */ 77 static unsigned int halt_poll_ns_grow = 2; 78 module_param(halt_poll_ns_grow, uint, S_IRUGO | S_IWUSR); 79 80 /* Default resets per-vcpu halt_poll_ns . */ 81 static unsigned int halt_poll_ns_shrink; 82 module_param(halt_poll_ns_shrink, uint, S_IRUGO | S_IWUSR); 83 84 /* 85 * Ordering of locks: 86 * 87 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock 88 */ 89 90 DEFINE_SPINLOCK(kvm_lock); 91 static DEFINE_RAW_SPINLOCK(kvm_count_lock); 92 LIST_HEAD(vm_list); 93 94 static cpumask_var_t cpus_hardware_enabled; 95 static int kvm_usage_count; 96 static atomic_t hardware_enable_failed; 97 98 struct kmem_cache *kvm_vcpu_cache; 99 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 100 101 static __read_mostly struct preempt_ops kvm_preempt_ops; 102 103 struct dentry *kvm_debugfs_dir; 104 EXPORT_SYMBOL_GPL(kvm_debugfs_dir); 105 106 static int kvm_debugfs_num_entries; 107 static const struct file_operations *stat_fops_per_vm[]; 108 109 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 110 unsigned long arg); 111 #ifdef CONFIG_KVM_COMPAT 112 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl, 113 unsigned long arg); 114 #endif 115 static int hardware_enable_all(void); 116 static void hardware_disable_all(void); 117 118 static void kvm_io_bus_destroy(struct kvm_io_bus *bus); 119 120 static void kvm_release_pfn_dirty(kvm_pfn_t pfn); 121 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn); 122 123 __visible bool kvm_rebooting; 124 EXPORT_SYMBOL_GPL(kvm_rebooting); 125 126 static bool largepages_enabled = true; 127 128 bool kvm_is_reserved_pfn(kvm_pfn_t pfn) 129 { 130 if (pfn_valid(pfn)) 131 return PageReserved(pfn_to_page(pfn)); 132 133 return true; 134 } 135 136 /* 137 * Switches to specified vcpu, until a matching vcpu_put() 138 */ 139 int vcpu_load(struct kvm_vcpu *vcpu) 140 { 141 int cpu; 142 143 if (mutex_lock_killable(&vcpu->mutex)) 144 return -EINTR; 145 cpu = get_cpu(); 146 preempt_notifier_register(&vcpu->preempt_notifier); 147 kvm_arch_vcpu_load(vcpu, cpu); 148 put_cpu(); 149 return 0; 150 } 151 152 void vcpu_put(struct kvm_vcpu *vcpu) 153 { 154 preempt_disable(); 155 kvm_arch_vcpu_put(vcpu); 156 preempt_notifier_unregister(&vcpu->preempt_notifier); 157 preempt_enable(); 158 mutex_unlock(&vcpu->mutex); 159 } 160 161 static void ack_flush(void *_completed) 162 { 163 } 164 165 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) 166 { 167 int i, cpu, me; 168 cpumask_var_t cpus; 169 bool called = true; 170 struct kvm_vcpu *vcpu; 171 172 zalloc_cpumask_var(&cpus, GFP_ATOMIC); 173 174 me = get_cpu(); 175 kvm_for_each_vcpu(i, vcpu, kvm) { 176 kvm_make_request(req, vcpu); 177 cpu = vcpu->cpu; 178 179 /* Set ->requests bit before we read ->mode. */ 180 smp_mb__after_atomic(); 181 182 if (cpus != NULL && cpu != -1 && cpu != me && 183 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE) 184 cpumask_set_cpu(cpu, cpus); 185 } 186 if (unlikely(cpus == NULL)) 187 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1); 188 else if (!cpumask_empty(cpus)) 189 smp_call_function_many(cpus, ack_flush, NULL, 1); 190 else 191 called = false; 192 put_cpu(); 193 free_cpumask_var(cpus); 194 return called; 195 } 196 197 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL 198 void kvm_flush_remote_tlbs(struct kvm *kvm) 199 { 200 /* 201 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in 202 * kvm_make_all_cpus_request. 203 */ 204 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty); 205 206 /* 207 * We want to publish modifications to the page tables before reading 208 * mode. Pairs with a memory barrier in arch-specific code. 209 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest 210 * and smp_mb in walk_shadow_page_lockless_begin/end. 211 * - powerpc: smp_mb in kvmppc_prepare_to_enter. 212 * 213 * There is already an smp_mb__after_atomic() before 214 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that 215 * barrier here. 216 */ 217 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) 218 ++kvm->stat.remote_tlb_flush; 219 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); 220 } 221 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); 222 #endif 223 224 void kvm_reload_remote_mmus(struct kvm *kvm) 225 { 226 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); 227 } 228 229 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 230 { 231 struct page *page; 232 int r; 233 234 mutex_init(&vcpu->mutex); 235 vcpu->cpu = -1; 236 vcpu->kvm = kvm; 237 vcpu->vcpu_id = id; 238 vcpu->pid = NULL; 239 init_swait_queue_head(&vcpu->wq); 240 kvm_async_pf_vcpu_init(vcpu); 241 242 vcpu->pre_pcpu = -1; 243 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list); 244 245 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 246 if (!page) { 247 r = -ENOMEM; 248 goto fail; 249 } 250 vcpu->run = page_address(page); 251 252 kvm_vcpu_set_in_spin_loop(vcpu, false); 253 kvm_vcpu_set_dy_eligible(vcpu, false); 254 vcpu->preempted = false; 255 256 r = kvm_arch_vcpu_init(vcpu); 257 if (r < 0) 258 goto fail_free_run; 259 return 0; 260 261 fail_free_run: 262 free_page((unsigned long)vcpu->run); 263 fail: 264 return r; 265 } 266 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 267 268 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 269 { 270 put_pid(vcpu->pid); 271 kvm_arch_vcpu_uninit(vcpu); 272 free_page((unsigned long)vcpu->run); 273 } 274 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 275 276 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 277 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) 278 { 279 return container_of(mn, struct kvm, mmu_notifier); 280 } 281 282 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn, 283 struct mm_struct *mm, 284 unsigned long address) 285 { 286 struct kvm *kvm = mmu_notifier_to_kvm(mn); 287 int need_tlb_flush, idx; 288 289 /* 290 * When ->invalidate_page runs, the linux pte has been zapped 291 * already but the page is still allocated until 292 * ->invalidate_page returns. So if we increase the sequence 293 * here the kvm page fault will notice if the spte can't be 294 * established because the page is going to be freed. If 295 * instead the kvm page fault establishes the spte before 296 * ->invalidate_page runs, kvm_unmap_hva will release it 297 * before returning. 298 * 299 * The sequence increase only need to be seen at spin_unlock 300 * time, and not at spin_lock time. 301 * 302 * Increasing the sequence after the spin_unlock would be 303 * unsafe because the kvm page fault could then establish the 304 * pte after kvm_unmap_hva returned, without noticing the page 305 * is going to be freed. 306 */ 307 idx = srcu_read_lock(&kvm->srcu); 308 spin_lock(&kvm->mmu_lock); 309 310 kvm->mmu_notifier_seq++; 311 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty; 312 /* we've to flush the tlb before the pages can be freed */ 313 if (need_tlb_flush) 314 kvm_flush_remote_tlbs(kvm); 315 316 spin_unlock(&kvm->mmu_lock); 317 318 kvm_arch_mmu_notifier_invalidate_page(kvm, address); 319 320 srcu_read_unlock(&kvm->srcu, idx); 321 } 322 323 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, 324 struct mm_struct *mm, 325 unsigned long address, 326 pte_t pte) 327 { 328 struct kvm *kvm = mmu_notifier_to_kvm(mn); 329 int idx; 330 331 idx = srcu_read_lock(&kvm->srcu); 332 spin_lock(&kvm->mmu_lock); 333 kvm->mmu_notifier_seq++; 334 kvm_set_spte_hva(kvm, address, pte); 335 spin_unlock(&kvm->mmu_lock); 336 srcu_read_unlock(&kvm->srcu, idx); 337 } 338 339 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, 340 struct mm_struct *mm, 341 unsigned long start, 342 unsigned long end) 343 { 344 struct kvm *kvm = mmu_notifier_to_kvm(mn); 345 int need_tlb_flush = 0, idx; 346 347 idx = srcu_read_lock(&kvm->srcu); 348 spin_lock(&kvm->mmu_lock); 349 /* 350 * The count increase must become visible at unlock time as no 351 * spte can be established without taking the mmu_lock and 352 * count is also read inside the mmu_lock critical section. 353 */ 354 kvm->mmu_notifier_count++; 355 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end); 356 need_tlb_flush |= kvm->tlbs_dirty; 357 /* we've to flush the tlb before the pages can be freed */ 358 if (need_tlb_flush) 359 kvm_flush_remote_tlbs(kvm); 360 361 spin_unlock(&kvm->mmu_lock); 362 srcu_read_unlock(&kvm->srcu, idx); 363 } 364 365 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, 366 struct mm_struct *mm, 367 unsigned long start, 368 unsigned long end) 369 { 370 struct kvm *kvm = mmu_notifier_to_kvm(mn); 371 372 spin_lock(&kvm->mmu_lock); 373 /* 374 * This sequence increase will notify the kvm page fault that 375 * the page that is going to be mapped in the spte could have 376 * been freed. 377 */ 378 kvm->mmu_notifier_seq++; 379 smp_wmb(); 380 /* 381 * The above sequence increase must be visible before the 382 * below count decrease, which is ensured by the smp_wmb above 383 * in conjunction with the smp_rmb in mmu_notifier_retry(). 384 */ 385 kvm->mmu_notifier_count--; 386 spin_unlock(&kvm->mmu_lock); 387 388 BUG_ON(kvm->mmu_notifier_count < 0); 389 } 390 391 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, 392 struct mm_struct *mm, 393 unsigned long start, 394 unsigned long end) 395 { 396 struct kvm *kvm = mmu_notifier_to_kvm(mn); 397 int young, idx; 398 399 idx = srcu_read_lock(&kvm->srcu); 400 spin_lock(&kvm->mmu_lock); 401 402 young = kvm_age_hva(kvm, start, end); 403 if (young) 404 kvm_flush_remote_tlbs(kvm); 405 406 spin_unlock(&kvm->mmu_lock); 407 srcu_read_unlock(&kvm->srcu, idx); 408 409 return young; 410 } 411 412 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, 413 struct mm_struct *mm, 414 unsigned long start, 415 unsigned long end) 416 { 417 struct kvm *kvm = mmu_notifier_to_kvm(mn); 418 int young, idx; 419 420 idx = srcu_read_lock(&kvm->srcu); 421 spin_lock(&kvm->mmu_lock); 422 /* 423 * Even though we do not flush TLB, this will still adversely 424 * affect performance on pre-Haswell Intel EPT, where there is 425 * no EPT Access Bit to clear so that we have to tear down EPT 426 * tables instead. If we find this unacceptable, we can always 427 * add a parameter to kvm_age_hva so that it effectively doesn't 428 * do anything on clear_young. 429 * 430 * Also note that currently we never issue secondary TLB flushes 431 * from clear_young, leaving this job up to the regular system 432 * cadence. If we find this inaccurate, we might come up with a 433 * more sophisticated heuristic later. 434 */ 435 young = kvm_age_hva(kvm, start, end); 436 spin_unlock(&kvm->mmu_lock); 437 srcu_read_unlock(&kvm->srcu, idx); 438 439 return young; 440 } 441 442 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, 443 struct mm_struct *mm, 444 unsigned long address) 445 { 446 struct kvm *kvm = mmu_notifier_to_kvm(mn); 447 int young, idx; 448 449 idx = srcu_read_lock(&kvm->srcu); 450 spin_lock(&kvm->mmu_lock); 451 young = kvm_test_age_hva(kvm, address); 452 spin_unlock(&kvm->mmu_lock); 453 srcu_read_unlock(&kvm->srcu, idx); 454 455 return young; 456 } 457 458 static void kvm_mmu_notifier_release(struct mmu_notifier *mn, 459 struct mm_struct *mm) 460 { 461 struct kvm *kvm = mmu_notifier_to_kvm(mn); 462 int idx; 463 464 idx = srcu_read_lock(&kvm->srcu); 465 kvm_arch_flush_shadow_all(kvm); 466 srcu_read_unlock(&kvm->srcu, idx); 467 } 468 469 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { 470 .invalidate_page = kvm_mmu_notifier_invalidate_page, 471 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, 472 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, 473 .clear_flush_young = kvm_mmu_notifier_clear_flush_young, 474 .clear_young = kvm_mmu_notifier_clear_young, 475 .test_young = kvm_mmu_notifier_test_young, 476 .change_pte = kvm_mmu_notifier_change_pte, 477 .release = kvm_mmu_notifier_release, 478 }; 479 480 static int kvm_init_mmu_notifier(struct kvm *kvm) 481 { 482 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; 483 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); 484 } 485 486 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ 487 488 static int kvm_init_mmu_notifier(struct kvm *kvm) 489 { 490 return 0; 491 } 492 493 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ 494 495 static struct kvm_memslots *kvm_alloc_memslots(void) 496 { 497 int i; 498 struct kvm_memslots *slots; 499 500 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 501 if (!slots) 502 return NULL; 503 504 /* 505 * Init kvm generation close to the maximum to easily test the 506 * code of handling generation number wrap-around. 507 */ 508 slots->generation = -150; 509 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) 510 slots->id_to_index[i] = slots->memslots[i].id = i; 511 512 return slots; 513 } 514 515 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) 516 { 517 if (!memslot->dirty_bitmap) 518 return; 519 520 kvfree(memslot->dirty_bitmap); 521 memslot->dirty_bitmap = NULL; 522 } 523 524 /* 525 * Free any memory in @free but not in @dont. 526 */ 527 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 528 struct kvm_memory_slot *dont) 529 { 530 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 531 kvm_destroy_dirty_bitmap(free); 532 533 kvm_arch_free_memslot(kvm, free, dont); 534 535 free->npages = 0; 536 } 537 538 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) 539 { 540 struct kvm_memory_slot *memslot; 541 542 if (!slots) 543 return; 544 545 kvm_for_each_memslot(memslot, slots) 546 kvm_free_memslot(kvm, memslot, NULL); 547 548 kvfree(slots); 549 } 550 551 static void kvm_destroy_vm_debugfs(struct kvm *kvm) 552 { 553 int i; 554 555 if (!kvm->debugfs_dentry) 556 return; 557 558 debugfs_remove_recursive(kvm->debugfs_dentry); 559 560 for (i = 0; i < kvm_debugfs_num_entries; i++) 561 kfree(kvm->debugfs_stat_data[i]); 562 kfree(kvm->debugfs_stat_data); 563 } 564 565 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) 566 { 567 char dir_name[ITOA_MAX_LEN * 2]; 568 struct kvm_stat_data *stat_data; 569 struct kvm_stats_debugfs_item *p; 570 571 if (!debugfs_initialized()) 572 return 0; 573 574 snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd); 575 kvm->debugfs_dentry = debugfs_create_dir(dir_name, 576 kvm_debugfs_dir); 577 if (!kvm->debugfs_dentry) 578 return -ENOMEM; 579 580 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, 581 sizeof(*kvm->debugfs_stat_data), 582 GFP_KERNEL); 583 if (!kvm->debugfs_stat_data) 584 return -ENOMEM; 585 586 for (p = debugfs_entries; p->name; p++) { 587 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL); 588 if (!stat_data) 589 return -ENOMEM; 590 591 stat_data->kvm = kvm; 592 stat_data->offset = p->offset; 593 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data; 594 if (!debugfs_create_file(p->name, 0444, 595 kvm->debugfs_dentry, 596 stat_data, 597 stat_fops_per_vm[p->kind])) 598 return -ENOMEM; 599 } 600 return 0; 601 } 602 603 static struct kvm *kvm_create_vm(unsigned long type) 604 { 605 int r, i; 606 struct kvm *kvm = kvm_arch_alloc_vm(); 607 608 if (!kvm) 609 return ERR_PTR(-ENOMEM); 610 611 spin_lock_init(&kvm->mmu_lock); 612 atomic_inc(¤t->mm->mm_count); 613 kvm->mm = current->mm; 614 kvm_eventfd_init(kvm); 615 mutex_init(&kvm->lock); 616 mutex_init(&kvm->irq_lock); 617 mutex_init(&kvm->slots_lock); 618 atomic_set(&kvm->users_count, 1); 619 INIT_LIST_HEAD(&kvm->devices); 620 621 r = kvm_arch_init_vm(kvm, type); 622 if (r) 623 goto out_err_no_disable; 624 625 r = hardware_enable_all(); 626 if (r) 627 goto out_err_no_disable; 628 629 #ifdef CONFIG_HAVE_KVM_IRQFD 630 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); 631 #endif 632 633 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); 634 635 r = -ENOMEM; 636 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 637 kvm->memslots[i] = kvm_alloc_memslots(); 638 if (!kvm->memslots[i]) 639 goto out_err_no_srcu; 640 } 641 642 if (init_srcu_struct(&kvm->srcu)) 643 goto out_err_no_srcu; 644 if (init_srcu_struct(&kvm->irq_srcu)) 645 goto out_err_no_irq_srcu; 646 for (i = 0; i < KVM_NR_BUSES; i++) { 647 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus), 648 GFP_KERNEL); 649 if (!kvm->buses[i]) 650 goto out_err; 651 } 652 653 r = kvm_init_mmu_notifier(kvm); 654 if (r) 655 goto out_err; 656 657 spin_lock(&kvm_lock); 658 list_add(&kvm->vm_list, &vm_list); 659 spin_unlock(&kvm_lock); 660 661 preempt_notifier_inc(); 662 663 return kvm; 664 665 out_err: 666 cleanup_srcu_struct(&kvm->irq_srcu); 667 out_err_no_irq_srcu: 668 cleanup_srcu_struct(&kvm->srcu); 669 out_err_no_srcu: 670 hardware_disable_all(); 671 out_err_no_disable: 672 for (i = 0; i < KVM_NR_BUSES; i++) 673 kfree(kvm->buses[i]); 674 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 675 kvm_free_memslots(kvm, kvm->memslots[i]); 676 kvm_arch_free_vm(kvm); 677 mmdrop(current->mm); 678 return ERR_PTR(r); 679 } 680 681 /* 682 * Avoid using vmalloc for a small buffer. 683 * Should not be used when the size is statically known. 684 */ 685 void *kvm_kvzalloc(unsigned long size) 686 { 687 if (size > PAGE_SIZE) 688 return vzalloc(size); 689 else 690 return kzalloc(size, GFP_KERNEL); 691 } 692 693 static void kvm_destroy_devices(struct kvm *kvm) 694 { 695 struct kvm_device *dev, *tmp; 696 697 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { 698 list_del(&dev->vm_node); 699 dev->ops->destroy(dev); 700 } 701 } 702 703 static void kvm_destroy_vm(struct kvm *kvm) 704 { 705 int i; 706 struct mm_struct *mm = kvm->mm; 707 708 kvm_destroy_vm_debugfs(kvm); 709 kvm_arch_sync_events(kvm); 710 spin_lock(&kvm_lock); 711 list_del(&kvm->vm_list); 712 spin_unlock(&kvm_lock); 713 kvm_free_irq_routing(kvm); 714 for (i = 0; i < KVM_NR_BUSES; i++) 715 kvm_io_bus_destroy(kvm->buses[i]); 716 kvm_coalesced_mmio_free(kvm); 717 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 718 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); 719 #else 720 kvm_arch_flush_shadow_all(kvm); 721 #endif 722 kvm_arch_destroy_vm(kvm); 723 kvm_destroy_devices(kvm); 724 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) 725 kvm_free_memslots(kvm, kvm->memslots[i]); 726 cleanup_srcu_struct(&kvm->irq_srcu); 727 cleanup_srcu_struct(&kvm->srcu); 728 kvm_arch_free_vm(kvm); 729 preempt_notifier_dec(); 730 hardware_disable_all(); 731 mmdrop(mm); 732 } 733 734 void kvm_get_kvm(struct kvm *kvm) 735 { 736 atomic_inc(&kvm->users_count); 737 } 738 EXPORT_SYMBOL_GPL(kvm_get_kvm); 739 740 void kvm_put_kvm(struct kvm *kvm) 741 { 742 if (atomic_dec_and_test(&kvm->users_count)) 743 kvm_destroy_vm(kvm); 744 } 745 EXPORT_SYMBOL_GPL(kvm_put_kvm); 746 747 748 static int kvm_vm_release(struct inode *inode, struct file *filp) 749 { 750 struct kvm *kvm = filp->private_data; 751 752 kvm_irqfd_release(kvm); 753 754 kvm_put_kvm(kvm); 755 return 0; 756 } 757 758 /* 759 * Allocation size is twice as large as the actual dirty bitmap size. 760 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. 761 */ 762 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) 763 { 764 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); 765 766 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes); 767 if (!memslot->dirty_bitmap) 768 return -ENOMEM; 769 770 return 0; 771 } 772 773 /* 774 * Insert memslot and re-sort memslots based on their GFN, 775 * so binary search could be used to lookup GFN. 776 * Sorting algorithm takes advantage of having initially 777 * sorted array and known changed memslot position. 778 */ 779 static void update_memslots(struct kvm_memslots *slots, 780 struct kvm_memory_slot *new) 781 { 782 int id = new->id; 783 int i = slots->id_to_index[id]; 784 struct kvm_memory_slot *mslots = slots->memslots; 785 786 WARN_ON(mslots[i].id != id); 787 if (!new->npages) { 788 WARN_ON(!mslots[i].npages); 789 if (mslots[i].npages) 790 slots->used_slots--; 791 } else { 792 if (!mslots[i].npages) 793 slots->used_slots++; 794 } 795 796 while (i < KVM_MEM_SLOTS_NUM - 1 && 797 new->base_gfn <= mslots[i + 1].base_gfn) { 798 if (!mslots[i + 1].npages) 799 break; 800 mslots[i] = mslots[i + 1]; 801 slots->id_to_index[mslots[i].id] = i; 802 i++; 803 } 804 805 /* 806 * The ">=" is needed when creating a slot with base_gfn == 0, 807 * so that it moves before all those with base_gfn == npages == 0. 808 * 809 * On the other hand, if new->npages is zero, the above loop has 810 * already left i pointing to the beginning of the empty part of 811 * mslots, and the ">=" would move the hole backwards in this 812 * case---which is wrong. So skip the loop when deleting a slot. 813 */ 814 if (new->npages) { 815 while (i > 0 && 816 new->base_gfn >= mslots[i - 1].base_gfn) { 817 mslots[i] = mslots[i - 1]; 818 slots->id_to_index[mslots[i].id] = i; 819 i--; 820 } 821 } else 822 WARN_ON_ONCE(i != slots->used_slots); 823 824 mslots[i] = *new; 825 slots->id_to_index[mslots[i].id] = i; 826 } 827 828 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) 829 { 830 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; 831 832 #ifdef __KVM_HAVE_READONLY_MEM 833 valid_flags |= KVM_MEM_READONLY; 834 #endif 835 836 if (mem->flags & ~valid_flags) 837 return -EINVAL; 838 839 return 0; 840 } 841 842 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, 843 int as_id, struct kvm_memslots *slots) 844 { 845 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); 846 847 /* 848 * Set the low bit in the generation, which disables SPTE caching 849 * until the end of synchronize_srcu_expedited. 850 */ 851 WARN_ON(old_memslots->generation & 1); 852 slots->generation = old_memslots->generation + 1; 853 854 rcu_assign_pointer(kvm->memslots[as_id], slots); 855 synchronize_srcu_expedited(&kvm->srcu); 856 857 /* 858 * Increment the new memslot generation a second time. This prevents 859 * vm exits that race with memslot updates from caching a memslot 860 * generation that will (potentially) be valid forever. 861 */ 862 slots->generation++; 863 864 kvm_arch_memslots_updated(kvm, slots); 865 866 return old_memslots; 867 } 868 869 /* 870 * Allocate some memory and give it an address in the guest physical address 871 * space. 872 * 873 * Discontiguous memory is allowed, mostly for framebuffers. 874 * 875 * Must be called holding kvm->slots_lock for write. 876 */ 877 int __kvm_set_memory_region(struct kvm *kvm, 878 const struct kvm_userspace_memory_region *mem) 879 { 880 int r; 881 gfn_t base_gfn; 882 unsigned long npages; 883 struct kvm_memory_slot *slot; 884 struct kvm_memory_slot old, new; 885 struct kvm_memslots *slots = NULL, *old_memslots; 886 int as_id, id; 887 enum kvm_mr_change change; 888 889 r = check_memory_region_flags(mem); 890 if (r) 891 goto out; 892 893 r = -EINVAL; 894 as_id = mem->slot >> 16; 895 id = (u16)mem->slot; 896 897 /* General sanity checks */ 898 if (mem->memory_size & (PAGE_SIZE - 1)) 899 goto out; 900 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 901 goto out; 902 /* We can read the guest memory with __xxx_user() later on. */ 903 if ((id < KVM_USER_MEM_SLOTS) && 904 ((mem->userspace_addr & (PAGE_SIZE - 1)) || 905 !access_ok(VERIFY_WRITE, 906 (void __user *)(unsigned long)mem->userspace_addr, 907 mem->memory_size))) 908 goto out; 909 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) 910 goto out; 911 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 912 goto out; 913 914 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); 915 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 916 npages = mem->memory_size >> PAGE_SHIFT; 917 918 if (npages > KVM_MEM_MAX_NR_PAGES) 919 goto out; 920 921 new = old = *slot; 922 923 new.id = id; 924 new.base_gfn = base_gfn; 925 new.npages = npages; 926 new.flags = mem->flags; 927 928 if (npages) { 929 if (!old.npages) 930 change = KVM_MR_CREATE; 931 else { /* Modify an existing slot. */ 932 if ((mem->userspace_addr != old.userspace_addr) || 933 (npages != old.npages) || 934 ((new.flags ^ old.flags) & KVM_MEM_READONLY)) 935 goto out; 936 937 if (base_gfn != old.base_gfn) 938 change = KVM_MR_MOVE; 939 else if (new.flags != old.flags) 940 change = KVM_MR_FLAGS_ONLY; 941 else { /* Nothing to change. */ 942 r = 0; 943 goto out; 944 } 945 } 946 } else { 947 if (!old.npages) 948 goto out; 949 950 change = KVM_MR_DELETE; 951 new.base_gfn = 0; 952 new.flags = 0; 953 } 954 955 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 956 /* Check for overlaps */ 957 r = -EEXIST; 958 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { 959 if ((slot->id >= KVM_USER_MEM_SLOTS) || 960 (slot->id == id)) 961 continue; 962 if (!((base_gfn + npages <= slot->base_gfn) || 963 (base_gfn >= slot->base_gfn + slot->npages))) 964 goto out; 965 } 966 } 967 968 /* Free page dirty bitmap if unneeded */ 969 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 970 new.dirty_bitmap = NULL; 971 972 r = -ENOMEM; 973 if (change == KVM_MR_CREATE) { 974 new.userspace_addr = mem->userspace_addr; 975 976 if (kvm_arch_create_memslot(kvm, &new, npages)) 977 goto out_free; 978 } 979 980 /* Allocate page dirty bitmap if needed */ 981 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 982 if (kvm_create_dirty_bitmap(&new) < 0) 983 goto out_free; 984 } 985 986 slots = kvm_kvzalloc(sizeof(struct kvm_memslots)); 987 if (!slots) 988 goto out_free; 989 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); 990 991 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { 992 slot = id_to_memslot(slots, id); 993 slot->flags |= KVM_MEMSLOT_INVALID; 994 995 old_memslots = install_new_memslots(kvm, as_id, slots); 996 997 /* slot was deleted or moved, clear iommu mapping */ 998 kvm_iommu_unmap_pages(kvm, &old); 999 /* From this point no new shadow pages pointing to a deleted, 1000 * or moved, memslot will be created. 1001 * 1002 * validation of sp->gfn happens in: 1003 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) 1004 * - kvm_is_visible_gfn (mmu_check_roots) 1005 */ 1006 kvm_arch_flush_shadow_memslot(kvm, slot); 1007 1008 /* 1009 * We can re-use the old_memslots from above, the only difference 1010 * from the currently installed memslots is the invalid flag. This 1011 * will get overwritten by update_memslots anyway. 1012 */ 1013 slots = old_memslots; 1014 } 1015 1016 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); 1017 if (r) 1018 goto out_slots; 1019 1020 /* actual memory is freed via old in kvm_free_memslot below */ 1021 if (change == KVM_MR_DELETE) { 1022 new.dirty_bitmap = NULL; 1023 memset(&new.arch, 0, sizeof(new.arch)); 1024 } 1025 1026 update_memslots(slots, &new); 1027 old_memslots = install_new_memslots(kvm, as_id, slots); 1028 1029 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); 1030 1031 kvm_free_memslot(kvm, &old, &new); 1032 kvfree(old_memslots); 1033 1034 /* 1035 * IOMMU mapping: New slots need to be mapped. Old slots need to be 1036 * un-mapped and re-mapped if their base changes. Since base change 1037 * unmapping is handled above with slot deletion, mapping alone is 1038 * needed here. Anything else the iommu might care about for existing 1039 * slots (size changes, userspace addr changes and read-only flag 1040 * changes) is disallowed above, so any other attribute changes getting 1041 * here can be skipped. 1042 */ 1043 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { 1044 r = kvm_iommu_map_pages(kvm, &new); 1045 return r; 1046 } 1047 1048 return 0; 1049 1050 out_slots: 1051 kvfree(slots); 1052 out_free: 1053 kvm_free_memslot(kvm, &new, &old); 1054 out: 1055 return r; 1056 } 1057 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 1058 1059 int kvm_set_memory_region(struct kvm *kvm, 1060 const struct kvm_userspace_memory_region *mem) 1061 { 1062 int r; 1063 1064 mutex_lock(&kvm->slots_lock); 1065 r = __kvm_set_memory_region(kvm, mem); 1066 mutex_unlock(&kvm->slots_lock); 1067 return r; 1068 } 1069 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 1070 1071 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 1072 struct kvm_userspace_memory_region *mem) 1073 { 1074 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) 1075 return -EINVAL; 1076 1077 return kvm_set_memory_region(kvm, mem); 1078 } 1079 1080 int kvm_get_dirty_log(struct kvm *kvm, 1081 struct kvm_dirty_log *log, int *is_dirty) 1082 { 1083 struct kvm_memslots *slots; 1084 struct kvm_memory_slot *memslot; 1085 int r, i, as_id, id; 1086 unsigned long n; 1087 unsigned long any = 0; 1088 1089 r = -EINVAL; 1090 as_id = log->slot >> 16; 1091 id = (u16)log->slot; 1092 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1093 goto out; 1094 1095 slots = __kvm_memslots(kvm, as_id); 1096 memslot = id_to_memslot(slots, id); 1097 r = -ENOENT; 1098 if (!memslot->dirty_bitmap) 1099 goto out; 1100 1101 n = kvm_dirty_bitmap_bytes(memslot); 1102 1103 for (i = 0; !any && i < n/sizeof(long); ++i) 1104 any = memslot->dirty_bitmap[i]; 1105 1106 r = -EFAULT; 1107 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 1108 goto out; 1109 1110 if (any) 1111 *is_dirty = 1; 1112 1113 r = 0; 1114 out: 1115 return r; 1116 } 1117 EXPORT_SYMBOL_GPL(kvm_get_dirty_log); 1118 1119 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1120 /** 1121 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages 1122 * are dirty write protect them for next write. 1123 * @kvm: pointer to kvm instance 1124 * @log: slot id and address to which we copy the log 1125 * @is_dirty: flag set if any page is dirty 1126 * 1127 * We need to keep it in mind that VCPU threads can write to the bitmap 1128 * concurrently. So, to avoid losing track of dirty pages we keep the 1129 * following order: 1130 * 1131 * 1. Take a snapshot of the bit and clear it if needed. 1132 * 2. Write protect the corresponding page. 1133 * 3. Copy the snapshot to the userspace. 1134 * 4. Upon return caller flushes TLB's if needed. 1135 * 1136 * Between 2 and 4, the guest may write to the page using the remaining TLB 1137 * entry. This is not a problem because the page is reported dirty using 1138 * the snapshot taken before and step 4 ensures that writes done after 1139 * exiting to userspace will be logged for the next call. 1140 * 1141 */ 1142 int kvm_get_dirty_log_protect(struct kvm *kvm, 1143 struct kvm_dirty_log *log, bool *is_dirty) 1144 { 1145 struct kvm_memslots *slots; 1146 struct kvm_memory_slot *memslot; 1147 int r, i, as_id, id; 1148 unsigned long n; 1149 unsigned long *dirty_bitmap; 1150 unsigned long *dirty_bitmap_buffer; 1151 1152 r = -EINVAL; 1153 as_id = log->slot >> 16; 1154 id = (u16)log->slot; 1155 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) 1156 goto out; 1157 1158 slots = __kvm_memslots(kvm, as_id); 1159 memslot = id_to_memslot(slots, id); 1160 1161 dirty_bitmap = memslot->dirty_bitmap; 1162 r = -ENOENT; 1163 if (!dirty_bitmap) 1164 goto out; 1165 1166 n = kvm_dirty_bitmap_bytes(memslot); 1167 1168 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long); 1169 memset(dirty_bitmap_buffer, 0, n); 1170 1171 spin_lock(&kvm->mmu_lock); 1172 *is_dirty = false; 1173 for (i = 0; i < n / sizeof(long); i++) { 1174 unsigned long mask; 1175 gfn_t offset; 1176 1177 if (!dirty_bitmap[i]) 1178 continue; 1179 1180 *is_dirty = true; 1181 1182 mask = xchg(&dirty_bitmap[i], 0); 1183 dirty_bitmap_buffer[i] = mask; 1184 1185 if (mask) { 1186 offset = i * BITS_PER_LONG; 1187 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, 1188 offset, mask); 1189 } 1190 } 1191 1192 spin_unlock(&kvm->mmu_lock); 1193 1194 r = -EFAULT; 1195 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) 1196 goto out; 1197 1198 r = 0; 1199 out: 1200 return r; 1201 } 1202 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect); 1203 #endif 1204 1205 bool kvm_largepages_enabled(void) 1206 { 1207 return largepages_enabled; 1208 } 1209 1210 void kvm_disable_largepages(void) 1211 { 1212 largepages_enabled = false; 1213 } 1214 EXPORT_SYMBOL_GPL(kvm_disable_largepages); 1215 1216 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 1217 { 1218 return __gfn_to_memslot(kvm_memslots(kvm), gfn); 1219 } 1220 EXPORT_SYMBOL_GPL(gfn_to_memslot); 1221 1222 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) 1223 { 1224 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); 1225 } 1226 1227 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 1228 { 1229 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); 1230 1231 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || 1232 memslot->flags & KVM_MEMSLOT_INVALID) 1233 return false; 1234 1235 return true; 1236 } 1237 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 1238 1239 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn) 1240 { 1241 struct vm_area_struct *vma; 1242 unsigned long addr, size; 1243 1244 size = PAGE_SIZE; 1245 1246 addr = gfn_to_hva(kvm, gfn); 1247 if (kvm_is_error_hva(addr)) 1248 return PAGE_SIZE; 1249 1250 down_read(¤t->mm->mmap_sem); 1251 vma = find_vma(current->mm, addr); 1252 if (!vma) 1253 goto out; 1254 1255 size = vma_kernel_pagesize(vma); 1256 1257 out: 1258 up_read(¤t->mm->mmap_sem); 1259 1260 return size; 1261 } 1262 1263 static bool memslot_is_readonly(struct kvm_memory_slot *slot) 1264 { 1265 return slot->flags & KVM_MEM_READONLY; 1266 } 1267 1268 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1269 gfn_t *nr_pages, bool write) 1270 { 1271 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 1272 return KVM_HVA_ERR_BAD; 1273 1274 if (memslot_is_readonly(slot) && write) 1275 return KVM_HVA_ERR_RO_BAD; 1276 1277 if (nr_pages) 1278 *nr_pages = slot->npages - (gfn - slot->base_gfn); 1279 1280 return __gfn_to_hva_memslot(slot, gfn); 1281 } 1282 1283 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn, 1284 gfn_t *nr_pages) 1285 { 1286 return __gfn_to_hva_many(slot, gfn, nr_pages, true); 1287 } 1288 1289 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, 1290 gfn_t gfn) 1291 { 1292 return gfn_to_hva_many(slot, gfn, NULL); 1293 } 1294 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); 1295 1296 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 1297 { 1298 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); 1299 } 1300 EXPORT_SYMBOL_GPL(gfn_to_hva); 1301 1302 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) 1303 { 1304 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); 1305 } 1306 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); 1307 1308 /* 1309 * If writable is set to false, the hva returned by this function is only 1310 * allowed to be read. 1311 */ 1312 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, 1313 gfn_t gfn, bool *writable) 1314 { 1315 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false); 1316 1317 if (!kvm_is_error_hva(hva) && writable) 1318 *writable = !memslot_is_readonly(slot); 1319 1320 return hva; 1321 } 1322 1323 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) 1324 { 1325 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1326 1327 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1328 } 1329 1330 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable) 1331 { 1332 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1333 1334 return gfn_to_hva_memslot_prot(slot, gfn, writable); 1335 } 1336 1337 static int get_user_page_nowait(unsigned long start, int write, 1338 struct page **page) 1339 { 1340 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; 1341 1342 if (write) 1343 flags |= FOLL_WRITE; 1344 1345 return __get_user_pages(current, current->mm, start, 1, flags, page, 1346 NULL, NULL); 1347 } 1348 1349 static inline int check_user_page_hwpoison(unsigned long addr) 1350 { 1351 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; 1352 1353 rc = __get_user_pages(current, current->mm, addr, 1, 1354 flags, NULL, NULL, NULL); 1355 return rc == -EHWPOISON; 1356 } 1357 1358 /* 1359 * The atomic path to get the writable pfn which will be stored in @pfn, 1360 * true indicates success, otherwise false is returned. 1361 */ 1362 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async, 1363 bool write_fault, bool *writable, kvm_pfn_t *pfn) 1364 { 1365 struct page *page[1]; 1366 int npages; 1367 1368 if (!(async || atomic)) 1369 return false; 1370 1371 /* 1372 * Fast pin a writable pfn only if it is a write fault request 1373 * or the caller allows to map a writable pfn for a read fault 1374 * request. 1375 */ 1376 if (!(write_fault || writable)) 1377 return false; 1378 1379 npages = __get_user_pages_fast(addr, 1, 1, page); 1380 if (npages == 1) { 1381 *pfn = page_to_pfn(page[0]); 1382 1383 if (writable) 1384 *writable = true; 1385 return true; 1386 } 1387 1388 return false; 1389 } 1390 1391 /* 1392 * The slow path to get the pfn of the specified host virtual address, 1393 * 1 indicates success, -errno is returned if error is detected. 1394 */ 1395 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault, 1396 bool *writable, kvm_pfn_t *pfn) 1397 { 1398 struct page *page[1]; 1399 int npages = 0; 1400 1401 might_sleep(); 1402 1403 if (writable) 1404 *writable = write_fault; 1405 1406 if (async) { 1407 down_read(¤t->mm->mmap_sem); 1408 npages = get_user_page_nowait(addr, write_fault, page); 1409 up_read(¤t->mm->mmap_sem); 1410 } else 1411 npages = __get_user_pages_unlocked(current, current->mm, addr, 1, 1412 write_fault, 0, page, 1413 FOLL_TOUCH|FOLL_HWPOISON); 1414 if (npages != 1) 1415 return npages; 1416 1417 /* map read fault as writable if possible */ 1418 if (unlikely(!write_fault) && writable) { 1419 struct page *wpage[1]; 1420 1421 npages = __get_user_pages_fast(addr, 1, 1, wpage); 1422 if (npages == 1) { 1423 *writable = true; 1424 put_page(page[0]); 1425 page[0] = wpage[0]; 1426 } 1427 1428 npages = 1; 1429 } 1430 *pfn = page_to_pfn(page[0]); 1431 return npages; 1432 } 1433 1434 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) 1435 { 1436 if (unlikely(!(vma->vm_flags & VM_READ))) 1437 return false; 1438 1439 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE)))) 1440 return false; 1441 1442 return true; 1443 } 1444 1445 /* 1446 * Pin guest page in memory and return its pfn. 1447 * @addr: host virtual address which maps memory to the guest 1448 * @atomic: whether this function can sleep 1449 * @async: whether this function need to wait IO complete if the 1450 * host page is not in the memory 1451 * @write_fault: whether we should get a writable host page 1452 * @writable: whether it allows to map a writable host page for !@write_fault 1453 * 1454 * The function will map a writable host page for these two cases: 1455 * 1): @write_fault = true 1456 * 2): @write_fault = false && @writable, @writable will tell the caller 1457 * whether the mapping is writable. 1458 */ 1459 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, 1460 bool write_fault, bool *writable) 1461 { 1462 struct vm_area_struct *vma; 1463 kvm_pfn_t pfn = 0; 1464 int npages; 1465 1466 /* we can do it either atomically or asynchronously, not both */ 1467 BUG_ON(atomic && async); 1468 1469 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn)) 1470 return pfn; 1471 1472 if (atomic) 1473 return KVM_PFN_ERR_FAULT; 1474 1475 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn); 1476 if (npages == 1) 1477 return pfn; 1478 1479 down_read(¤t->mm->mmap_sem); 1480 if (npages == -EHWPOISON || 1481 (!async && check_user_page_hwpoison(addr))) { 1482 pfn = KVM_PFN_ERR_HWPOISON; 1483 goto exit; 1484 } 1485 1486 vma = find_vma_intersection(current->mm, addr, addr + 1); 1487 1488 if (vma == NULL) 1489 pfn = KVM_PFN_ERR_FAULT; 1490 else if ((vma->vm_flags & VM_PFNMAP)) { 1491 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + 1492 vma->vm_pgoff; 1493 BUG_ON(!kvm_is_reserved_pfn(pfn)); 1494 } else { 1495 if (async && vma_is_valid(vma, write_fault)) 1496 *async = true; 1497 pfn = KVM_PFN_ERR_FAULT; 1498 } 1499 exit: 1500 up_read(¤t->mm->mmap_sem); 1501 return pfn; 1502 } 1503 1504 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, 1505 bool atomic, bool *async, bool write_fault, 1506 bool *writable) 1507 { 1508 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault); 1509 1510 if (addr == KVM_HVA_ERR_RO_BAD) { 1511 if (writable) 1512 *writable = false; 1513 return KVM_PFN_ERR_RO_FAULT; 1514 } 1515 1516 if (kvm_is_error_hva(addr)) { 1517 if (writable) 1518 *writable = false; 1519 return KVM_PFN_NOSLOT; 1520 } 1521 1522 /* Do not map writable pfn in the readonly memslot. */ 1523 if (writable && memslot_is_readonly(slot)) { 1524 *writable = false; 1525 writable = NULL; 1526 } 1527 1528 return hva_to_pfn(addr, atomic, async, write_fault, 1529 writable); 1530 } 1531 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot); 1532 1533 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 1534 bool *writable) 1535 { 1536 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, 1537 write_fault, writable); 1538 } 1539 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot); 1540 1541 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1542 { 1543 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL); 1544 } 1545 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot); 1546 1547 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) 1548 { 1549 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL); 1550 } 1551 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); 1552 1553 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) 1554 { 1555 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); 1556 } 1557 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); 1558 1559 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) 1560 { 1561 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1562 } 1563 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic); 1564 1565 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 1566 { 1567 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); 1568 } 1569 EXPORT_SYMBOL_GPL(gfn_to_pfn); 1570 1571 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 1572 { 1573 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); 1574 } 1575 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn); 1576 1577 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1578 struct page **pages, int nr_pages) 1579 { 1580 unsigned long addr; 1581 gfn_t entry; 1582 1583 addr = gfn_to_hva_many(slot, gfn, &entry); 1584 if (kvm_is_error_hva(addr)) 1585 return -1; 1586 1587 if (entry < nr_pages) 1588 return 0; 1589 1590 return __get_user_pages_fast(addr, nr_pages, 1, pages); 1591 } 1592 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic); 1593 1594 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn) 1595 { 1596 if (is_error_noslot_pfn(pfn)) 1597 return KVM_ERR_PTR_BAD_PAGE; 1598 1599 if (kvm_is_reserved_pfn(pfn)) { 1600 WARN_ON(1); 1601 return KVM_ERR_PTR_BAD_PAGE; 1602 } 1603 1604 return pfn_to_page(pfn); 1605 } 1606 1607 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1608 { 1609 kvm_pfn_t pfn; 1610 1611 pfn = gfn_to_pfn(kvm, gfn); 1612 1613 return kvm_pfn_to_page(pfn); 1614 } 1615 EXPORT_SYMBOL_GPL(gfn_to_page); 1616 1617 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn) 1618 { 1619 kvm_pfn_t pfn; 1620 1621 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn); 1622 1623 return kvm_pfn_to_page(pfn); 1624 } 1625 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page); 1626 1627 void kvm_release_page_clean(struct page *page) 1628 { 1629 WARN_ON(is_error_page(page)); 1630 1631 kvm_release_pfn_clean(page_to_pfn(page)); 1632 } 1633 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 1634 1635 void kvm_release_pfn_clean(kvm_pfn_t pfn) 1636 { 1637 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn)) 1638 put_page(pfn_to_page(pfn)); 1639 } 1640 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 1641 1642 void kvm_release_page_dirty(struct page *page) 1643 { 1644 WARN_ON(is_error_page(page)); 1645 1646 kvm_release_pfn_dirty(page_to_pfn(page)); 1647 } 1648 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 1649 1650 static void kvm_release_pfn_dirty(kvm_pfn_t pfn) 1651 { 1652 kvm_set_pfn_dirty(pfn); 1653 kvm_release_pfn_clean(pfn); 1654 } 1655 1656 void kvm_set_pfn_dirty(kvm_pfn_t pfn) 1657 { 1658 if (!kvm_is_reserved_pfn(pfn)) { 1659 struct page *page = pfn_to_page(pfn); 1660 1661 if (!PageReserved(page)) 1662 SetPageDirty(page); 1663 } 1664 } 1665 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 1666 1667 void kvm_set_pfn_accessed(kvm_pfn_t pfn) 1668 { 1669 if (!kvm_is_reserved_pfn(pfn)) 1670 mark_page_accessed(pfn_to_page(pfn)); 1671 } 1672 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 1673 1674 void kvm_get_pfn(kvm_pfn_t pfn) 1675 { 1676 if (!kvm_is_reserved_pfn(pfn)) 1677 get_page(pfn_to_page(pfn)); 1678 } 1679 EXPORT_SYMBOL_GPL(kvm_get_pfn); 1680 1681 static int next_segment(unsigned long len, int offset) 1682 { 1683 if (len > PAGE_SIZE - offset) 1684 return PAGE_SIZE - offset; 1685 else 1686 return len; 1687 } 1688 1689 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn, 1690 void *data, int offset, int len) 1691 { 1692 int r; 1693 unsigned long addr; 1694 1695 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1696 if (kvm_is_error_hva(addr)) 1697 return -EFAULT; 1698 r = __copy_from_user(data, (void __user *)addr + offset, len); 1699 if (r) 1700 return -EFAULT; 1701 return 0; 1702 } 1703 1704 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1705 int len) 1706 { 1707 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1708 1709 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1710 } 1711 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 1712 1713 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, 1714 int offset, int len) 1715 { 1716 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1717 1718 return __kvm_read_guest_page(slot, gfn, data, offset, len); 1719 } 1720 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); 1721 1722 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 1723 { 1724 gfn_t gfn = gpa >> PAGE_SHIFT; 1725 int seg; 1726 int offset = offset_in_page(gpa); 1727 int ret; 1728 1729 while ((seg = next_segment(len, offset)) != 0) { 1730 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 1731 if (ret < 0) 1732 return ret; 1733 offset = 0; 1734 len -= seg; 1735 data += seg; 1736 ++gfn; 1737 } 1738 return 0; 1739 } 1740 EXPORT_SYMBOL_GPL(kvm_read_guest); 1741 1742 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) 1743 { 1744 gfn_t gfn = gpa >> PAGE_SHIFT; 1745 int seg; 1746 int offset = offset_in_page(gpa); 1747 int ret; 1748 1749 while ((seg = next_segment(len, offset)) != 0) { 1750 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg); 1751 if (ret < 0) 1752 return ret; 1753 offset = 0; 1754 len -= seg; 1755 data += seg; 1756 ++gfn; 1757 } 1758 return 0; 1759 } 1760 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); 1761 1762 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 1763 void *data, int offset, unsigned long len) 1764 { 1765 int r; 1766 unsigned long addr; 1767 1768 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL); 1769 if (kvm_is_error_hva(addr)) 1770 return -EFAULT; 1771 pagefault_disable(); 1772 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 1773 pagefault_enable(); 1774 if (r) 1775 return -EFAULT; 1776 return 0; 1777 } 1778 1779 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 1780 unsigned long len) 1781 { 1782 gfn_t gfn = gpa >> PAGE_SHIFT; 1783 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1784 int offset = offset_in_page(gpa); 1785 1786 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1787 } 1788 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic); 1789 1790 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, 1791 void *data, unsigned long len) 1792 { 1793 gfn_t gfn = gpa >> PAGE_SHIFT; 1794 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1795 int offset = offset_in_page(gpa); 1796 1797 return __kvm_read_guest_atomic(slot, gfn, data, offset, len); 1798 } 1799 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); 1800 1801 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn, 1802 const void *data, int offset, int len) 1803 { 1804 int r; 1805 unsigned long addr; 1806 1807 addr = gfn_to_hva_memslot(memslot, gfn); 1808 if (kvm_is_error_hva(addr)) 1809 return -EFAULT; 1810 r = __copy_to_user((void __user *)addr + offset, data, len); 1811 if (r) 1812 return -EFAULT; 1813 mark_page_dirty_in_slot(memslot, gfn); 1814 return 0; 1815 } 1816 1817 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, 1818 const void *data, int offset, int len) 1819 { 1820 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); 1821 1822 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1823 } 1824 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 1825 1826 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 1827 const void *data, int offset, int len) 1828 { 1829 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 1830 1831 return __kvm_write_guest_page(slot, gfn, data, offset, len); 1832 } 1833 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); 1834 1835 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1836 unsigned long len) 1837 { 1838 gfn_t gfn = gpa >> PAGE_SHIFT; 1839 int seg; 1840 int offset = offset_in_page(gpa); 1841 int ret; 1842 1843 while ((seg = next_segment(len, offset)) != 0) { 1844 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 1845 if (ret < 0) 1846 return ret; 1847 offset = 0; 1848 len -= seg; 1849 data += seg; 1850 ++gfn; 1851 } 1852 return 0; 1853 } 1854 EXPORT_SYMBOL_GPL(kvm_write_guest); 1855 1856 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1857 unsigned long len) 1858 { 1859 gfn_t gfn = gpa >> PAGE_SHIFT; 1860 int seg; 1861 int offset = offset_in_page(gpa); 1862 int ret; 1863 1864 while ((seg = next_segment(len, offset)) != 0) { 1865 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg); 1866 if (ret < 0) 1867 return ret; 1868 offset = 0; 1869 len -= seg; 1870 data += seg; 1871 ++gfn; 1872 } 1873 return 0; 1874 } 1875 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); 1876 1877 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1878 gpa_t gpa, unsigned long len) 1879 { 1880 struct kvm_memslots *slots = kvm_memslots(kvm); 1881 int offset = offset_in_page(gpa); 1882 gfn_t start_gfn = gpa >> PAGE_SHIFT; 1883 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT; 1884 gfn_t nr_pages_needed = end_gfn - start_gfn + 1; 1885 gfn_t nr_pages_avail; 1886 1887 ghc->gpa = gpa; 1888 ghc->generation = slots->generation; 1889 ghc->len = len; 1890 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1891 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL); 1892 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) { 1893 ghc->hva += offset; 1894 } else { 1895 /* 1896 * If the requested region crosses two memslots, we still 1897 * verify that the entire region is valid here. 1898 */ 1899 while (start_gfn <= end_gfn) { 1900 ghc->memslot = gfn_to_memslot(kvm, start_gfn); 1901 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, 1902 &nr_pages_avail); 1903 if (kvm_is_error_hva(ghc->hva)) 1904 return -EFAULT; 1905 start_gfn += nr_pages_avail; 1906 } 1907 /* Use the slow path for cross page reads and writes. */ 1908 ghc->memslot = NULL; 1909 } 1910 return 0; 1911 } 1912 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); 1913 1914 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1915 void *data, unsigned long len) 1916 { 1917 struct kvm_memslots *slots = kvm_memslots(kvm); 1918 int r; 1919 1920 BUG_ON(len > ghc->len); 1921 1922 if (slots->generation != ghc->generation) 1923 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1924 1925 if (unlikely(!ghc->memslot)) 1926 return kvm_write_guest(kvm, ghc->gpa, data, len); 1927 1928 if (kvm_is_error_hva(ghc->hva)) 1929 return -EFAULT; 1930 1931 r = __copy_to_user((void __user *)ghc->hva, data, len); 1932 if (r) 1933 return -EFAULT; 1934 mark_page_dirty_in_slot(ghc->memslot, ghc->gpa >> PAGE_SHIFT); 1935 1936 return 0; 1937 } 1938 EXPORT_SYMBOL_GPL(kvm_write_guest_cached); 1939 1940 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1941 void *data, unsigned long len) 1942 { 1943 struct kvm_memslots *slots = kvm_memslots(kvm); 1944 int r; 1945 1946 BUG_ON(len > ghc->len); 1947 1948 if (slots->generation != ghc->generation) 1949 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len); 1950 1951 if (unlikely(!ghc->memslot)) 1952 return kvm_read_guest(kvm, ghc->gpa, data, len); 1953 1954 if (kvm_is_error_hva(ghc->hva)) 1955 return -EFAULT; 1956 1957 r = __copy_from_user(data, (void __user *)ghc->hva, len); 1958 if (r) 1959 return -EFAULT; 1960 1961 return 0; 1962 } 1963 EXPORT_SYMBOL_GPL(kvm_read_guest_cached); 1964 1965 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 1966 { 1967 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 1968 1969 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); 1970 } 1971 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 1972 1973 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 1974 { 1975 gfn_t gfn = gpa >> PAGE_SHIFT; 1976 int seg; 1977 int offset = offset_in_page(gpa); 1978 int ret; 1979 1980 while ((seg = next_segment(len, offset)) != 0) { 1981 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 1982 if (ret < 0) 1983 return ret; 1984 offset = 0; 1985 len -= seg; 1986 ++gfn; 1987 } 1988 return 0; 1989 } 1990 EXPORT_SYMBOL_GPL(kvm_clear_guest); 1991 1992 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, 1993 gfn_t gfn) 1994 { 1995 if (memslot && memslot->dirty_bitmap) { 1996 unsigned long rel_gfn = gfn - memslot->base_gfn; 1997 1998 set_bit_le(rel_gfn, memslot->dirty_bitmap); 1999 } 2000 } 2001 2002 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 2003 { 2004 struct kvm_memory_slot *memslot; 2005 2006 memslot = gfn_to_memslot(kvm, gfn); 2007 mark_page_dirty_in_slot(memslot, gfn); 2008 } 2009 EXPORT_SYMBOL_GPL(mark_page_dirty); 2010 2011 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) 2012 { 2013 struct kvm_memory_slot *memslot; 2014 2015 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 2016 mark_page_dirty_in_slot(memslot, gfn); 2017 } 2018 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); 2019 2020 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) 2021 { 2022 unsigned int old, val, grow; 2023 2024 old = val = vcpu->halt_poll_ns; 2025 grow = READ_ONCE(halt_poll_ns_grow); 2026 /* 10us base */ 2027 if (val == 0 && grow) 2028 val = 10000; 2029 else 2030 val *= grow; 2031 2032 if (val > halt_poll_ns) 2033 val = halt_poll_ns; 2034 2035 vcpu->halt_poll_ns = val; 2036 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); 2037 } 2038 2039 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu) 2040 { 2041 unsigned int old, val, shrink; 2042 2043 old = val = vcpu->halt_poll_ns; 2044 shrink = READ_ONCE(halt_poll_ns_shrink); 2045 if (shrink == 0) 2046 val = 0; 2047 else 2048 val /= shrink; 2049 2050 vcpu->halt_poll_ns = val; 2051 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old); 2052 } 2053 2054 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) 2055 { 2056 if (kvm_arch_vcpu_runnable(vcpu)) { 2057 kvm_make_request(KVM_REQ_UNHALT, vcpu); 2058 return -EINTR; 2059 } 2060 if (kvm_cpu_has_pending_timer(vcpu)) 2061 return -EINTR; 2062 if (signal_pending(current)) 2063 return -EINTR; 2064 2065 return 0; 2066 } 2067 2068 /* 2069 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 2070 */ 2071 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 2072 { 2073 ktime_t start, cur; 2074 DECLARE_SWAITQUEUE(wait); 2075 bool waited = false; 2076 u64 block_ns; 2077 2078 start = cur = ktime_get(); 2079 if (vcpu->halt_poll_ns) { 2080 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); 2081 2082 ++vcpu->stat.halt_attempted_poll; 2083 do { 2084 /* 2085 * This sets KVM_REQ_UNHALT if an interrupt 2086 * arrives. 2087 */ 2088 if (kvm_vcpu_check_block(vcpu) < 0) { 2089 ++vcpu->stat.halt_successful_poll; 2090 if (!vcpu_valid_wakeup(vcpu)) 2091 ++vcpu->stat.halt_poll_invalid; 2092 goto out; 2093 } 2094 cur = ktime_get(); 2095 } while (single_task_running() && ktime_before(cur, stop)); 2096 } 2097 2098 kvm_arch_vcpu_blocking(vcpu); 2099 2100 for (;;) { 2101 prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 2102 2103 if (kvm_vcpu_check_block(vcpu) < 0) 2104 break; 2105 2106 waited = true; 2107 schedule(); 2108 } 2109 2110 finish_swait(&vcpu->wq, &wait); 2111 cur = ktime_get(); 2112 2113 kvm_arch_vcpu_unblocking(vcpu); 2114 out: 2115 block_ns = ktime_to_ns(cur) - ktime_to_ns(start); 2116 2117 if (!vcpu_valid_wakeup(vcpu)) 2118 shrink_halt_poll_ns(vcpu); 2119 else if (halt_poll_ns) { 2120 if (block_ns <= vcpu->halt_poll_ns) 2121 ; 2122 /* we had a long block, shrink polling */ 2123 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns) 2124 shrink_halt_poll_ns(vcpu); 2125 /* we had a short halt and our poll time is too small */ 2126 else if (vcpu->halt_poll_ns < halt_poll_ns && 2127 block_ns < halt_poll_ns) 2128 grow_halt_poll_ns(vcpu); 2129 } else 2130 vcpu->halt_poll_ns = 0; 2131 2132 trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu)); 2133 kvm_arch_vcpu_block_finish(vcpu); 2134 } 2135 EXPORT_SYMBOL_GPL(kvm_vcpu_block); 2136 2137 #ifndef CONFIG_S390 2138 void kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) 2139 { 2140 struct swait_queue_head *wqp; 2141 2142 wqp = kvm_arch_vcpu_wq(vcpu); 2143 if (swait_active(wqp)) { 2144 swake_up(wqp); 2145 ++vcpu->stat.halt_wakeup; 2146 } 2147 2148 } 2149 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up); 2150 2151 /* 2152 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode. 2153 */ 2154 void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 2155 { 2156 int me; 2157 int cpu = vcpu->cpu; 2158 2159 kvm_vcpu_wake_up(vcpu); 2160 me = get_cpu(); 2161 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) 2162 if (kvm_arch_vcpu_should_kick(vcpu)) 2163 smp_send_reschedule(cpu); 2164 put_cpu(); 2165 } 2166 EXPORT_SYMBOL_GPL(kvm_vcpu_kick); 2167 #endif /* !CONFIG_S390 */ 2168 2169 int kvm_vcpu_yield_to(struct kvm_vcpu *target) 2170 { 2171 struct pid *pid; 2172 struct task_struct *task = NULL; 2173 int ret = 0; 2174 2175 rcu_read_lock(); 2176 pid = rcu_dereference(target->pid); 2177 if (pid) 2178 task = get_pid_task(pid, PIDTYPE_PID); 2179 rcu_read_unlock(); 2180 if (!task) 2181 return ret; 2182 ret = yield_to(task, 1); 2183 put_task_struct(task); 2184 2185 return ret; 2186 } 2187 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); 2188 2189 /* 2190 * Helper that checks whether a VCPU is eligible for directed yield. 2191 * Most eligible candidate to yield is decided by following heuristics: 2192 * 2193 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently 2194 * (preempted lock holder), indicated by @in_spin_loop. 2195 * Set at the beiginning and cleared at the end of interception/PLE handler. 2196 * 2197 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get 2198 * chance last time (mostly it has become eligible now since we have probably 2199 * yielded to lockholder in last iteration. This is done by toggling 2200 * @dy_eligible each time a VCPU checked for eligibility.) 2201 * 2202 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding 2203 * to preempted lock-holder could result in wrong VCPU selection and CPU 2204 * burning. Giving priority for a potential lock-holder increases lock 2205 * progress. 2206 * 2207 * Since algorithm is based on heuristics, accessing another VCPU data without 2208 * locking does not harm. It may result in trying to yield to same VCPU, fail 2209 * and continue with next VCPU and so on. 2210 */ 2211 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 2212 { 2213 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2214 bool eligible; 2215 2216 eligible = !vcpu->spin_loop.in_spin_loop || 2217 vcpu->spin_loop.dy_eligible; 2218 2219 if (vcpu->spin_loop.in_spin_loop) 2220 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible); 2221 2222 return eligible; 2223 #else 2224 return true; 2225 #endif 2226 } 2227 2228 void kvm_vcpu_on_spin(struct kvm_vcpu *me) 2229 { 2230 struct kvm *kvm = me->kvm; 2231 struct kvm_vcpu *vcpu; 2232 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; 2233 int yielded = 0; 2234 int try = 3; 2235 int pass; 2236 int i; 2237 2238 kvm_vcpu_set_in_spin_loop(me, true); 2239 /* 2240 * We boost the priority of a VCPU that is runnable but not 2241 * currently running, because it got preempted by something 2242 * else and called schedule in __vcpu_run. Hopefully that 2243 * VCPU is holding the lock that we need and will release it. 2244 * We approximate round-robin by starting at the last boosted VCPU. 2245 */ 2246 for (pass = 0; pass < 2 && !yielded && try; pass++) { 2247 kvm_for_each_vcpu(i, vcpu, kvm) { 2248 if (!pass && i <= last_boosted_vcpu) { 2249 i = last_boosted_vcpu; 2250 continue; 2251 } else if (pass && i > last_boosted_vcpu) 2252 break; 2253 if (!ACCESS_ONCE(vcpu->preempted)) 2254 continue; 2255 if (vcpu == me) 2256 continue; 2257 if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu)) 2258 continue; 2259 if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) 2260 continue; 2261 2262 yielded = kvm_vcpu_yield_to(vcpu); 2263 if (yielded > 0) { 2264 kvm->last_boosted_vcpu = i; 2265 break; 2266 } else if (yielded < 0) { 2267 try--; 2268 if (!try) 2269 break; 2270 } 2271 } 2272 } 2273 kvm_vcpu_set_in_spin_loop(me, false); 2274 2275 /* Ensure vcpu is not eligible during next spinloop */ 2276 kvm_vcpu_set_dy_eligible(me, false); 2277 } 2278 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); 2279 2280 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2281 { 2282 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 2283 struct page *page; 2284 2285 if (vmf->pgoff == 0) 2286 page = virt_to_page(vcpu->run); 2287 #ifdef CONFIG_X86 2288 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 2289 page = virt_to_page(vcpu->arch.pio_data); 2290 #endif 2291 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2292 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 2293 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 2294 #endif 2295 else 2296 return kvm_arch_vcpu_fault(vcpu, vmf); 2297 get_page(page); 2298 vmf->page = page; 2299 return 0; 2300 } 2301 2302 static const struct vm_operations_struct kvm_vcpu_vm_ops = { 2303 .fault = kvm_vcpu_fault, 2304 }; 2305 2306 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 2307 { 2308 vma->vm_ops = &kvm_vcpu_vm_ops; 2309 return 0; 2310 } 2311 2312 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 2313 { 2314 struct kvm_vcpu *vcpu = filp->private_data; 2315 2316 kvm_put_kvm(vcpu->kvm); 2317 return 0; 2318 } 2319 2320 static struct file_operations kvm_vcpu_fops = { 2321 .release = kvm_vcpu_release, 2322 .unlocked_ioctl = kvm_vcpu_ioctl, 2323 #ifdef CONFIG_KVM_COMPAT 2324 .compat_ioctl = kvm_vcpu_compat_ioctl, 2325 #endif 2326 .mmap = kvm_vcpu_mmap, 2327 .llseek = noop_llseek, 2328 }; 2329 2330 /* 2331 * Allocates an inode for the vcpu. 2332 */ 2333 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 2334 { 2335 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); 2336 } 2337 2338 /* 2339 * Creates some virtual cpus. Good luck creating more than one. 2340 */ 2341 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) 2342 { 2343 int r; 2344 struct kvm_vcpu *vcpu; 2345 2346 if (id >= KVM_MAX_VCPU_ID) 2347 return -EINVAL; 2348 2349 vcpu = kvm_arch_vcpu_create(kvm, id); 2350 if (IS_ERR(vcpu)) 2351 return PTR_ERR(vcpu); 2352 2353 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 2354 2355 r = kvm_arch_vcpu_setup(vcpu); 2356 if (r) 2357 goto vcpu_destroy; 2358 2359 mutex_lock(&kvm->lock); 2360 if (!kvm_vcpu_compatible(vcpu)) { 2361 r = -EINVAL; 2362 goto unlock_vcpu_destroy; 2363 } 2364 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) { 2365 r = -EINVAL; 2366 goto unlock_vcpu_destroy; 2367 } 2368 if (kvm_get_vcpu_by_id(kvm, id)) { 2369 r = -EEXIST; 2370 goto unlock_vcpu_destroy; 2371 } 2372 2373 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]); 2374 2375 /* Now it's all set up, let userspace reach it */ 2376 kvm_get_kvm(kvm); 2377 r = create_vcpu_fd(vcpu); 2378 if (r < 0) { 2379 kvm_put_kvm(kvm); 2380 goto unlock_vcpu_destroy; 2381 } 2382 2383 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu; 2384 2385 /* 2386 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus 2387 * before kvm->online_vcpu's incremented value. 2388 */ 2389 smp_wmb(); 2390 atomic_inc(&kvm->online_vcpus); 2391 2392 mutex_unlock(&kvm->lock); 2393 kvm_arch_vcpu_postcreate(vcpu); 2394 return r; 2395 2396 unlock_vcpu_destroy: 2397 mutex_unlock(&kvm->lock); 2398 vcpu_destroy: 2399 kvm_arch_vcpu_destroy(vcpu); 2400 return r; 2401 } 2402 2403 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 2404 { 2405 if (sigset) { 2406 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2407 vcpu->sigset_active = 1; 2408 vcpu->sigset = *sigset; 2409 } else 2410 vcpu->sigset_active = 0; 2411 return 0; 2412 } 2413 2414 static long kvm_vcpu_ioctl(struct file *filp, 2415 unsigned int ioctl, unsigned long arg) 2416 { 2417 struct kvm_vcpu *vcpu = filp->private_data; 2418 void __user *argp = (void __user *)arg; 2419 int r; 2420 struct kvm_fpu *fpu = NULL; 2421 struct kvm_sregs *kvm_sregs = NULL; 2422 2423 if (vcpu->kvm->mm != current->mm) 2424 return -EIO; 2425 2426 if (unlikely(_IOC_TYPE(ioctl) != KVMIO)) 2427 return -EINVAL; 2428 2429 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS) 2430 /* 2431 * Special cases: vcpu ioctls that are asynchronous to vcpu execution, 2432 * so vcpu_load() would break it. 2433 */ 2434 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT) 2435 return kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2436 #endif 2437 2438 2439 r = vcpu_load(vcpu); 2440 if (r) 2441 return r; 2442 switch (ioctl) { 2443 case KVM_RUN: 2444 r = -EINVAL; 2445 if (arg) 2446 goto out; 2447 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) { 2448 /* The thread running this VCPU changed. */ 2449 struct pid *oldpid = vcpu->pid; 2450 struct pid *newpid = get_task_pid(current, PIDTYPE_PID); 2451 2452 rcu_assign_pointer(vcpu->pid, newpid); 2453 if (oldpid) 2454 synchronize_rcu(); 2455 put_pid(oldpid); 2456 } 2457 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 2458 trace_kvm_userspace_exit(vcpu->run->exit_reason, r); 2459 break; 2460 case KVM_GET_REGS: { 2461 struct kvm_regs *kvm_regs; 2462 2463 r = -ENOMEM; 2464 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 2465 if (!kvm_regs) 2466 goto out; 2467 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 2468 if (r) 2469 goto out_free1; 2470 r = -EFAULT; 2471 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 2472 goto out_free1; 2473 r = 0; 2474 out_free1: 2475 kfree(kvm_regs); 2476 break; 2477 } 2478 case KVM_SET_REGS: { 2479 struct kvm_regs *kvm_regs; 2480 2481 r = -ENOMEM; 2482 kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); 2483 if (IS_ERR(kvm_regs)) { 2484 r = PTR_ERR(kvm_regs); 2485 goto out; 2486 } 2487 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 2488 kfree(kvm_regs); 2489 break; 2490 } 2491 case KVM_GET_SREGS: { 2492 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); 2493 r = -ENOMEM; 2494 if (!kvm_sregs) 2495 goto out; 2496 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs); 2497 if (r) 2498 goto out; 2499 r = -EFAULT; 2500 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs))) 2501 goto out; 2502 r = 0; 2503 break; 2504 } 2505 case KVM_SET_SREGS: { 2506 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs)); 2507 if (IS_ERR(kvm_sregs)) { 2508 r = PTR_ERR(kvm_sregs); 2509 kvm_sregs = NULL; 2510 goto out; 2511 } 2512 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs); 2513 break; 2514 } 2515 case KVM_GET_MP_STATE: { 2516 struct kvm_mp_state mp_state; 2517 2518 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 2519 if (r) 2520 goto out; 2521 r = -EFAULT; 2522 if (copy_to_user(argp, &mp_state, sizeof(mp_state))) 2523 goto out; 2524 r = 0; 2525 break; 2526 } 2527 case KVM_SET_MP_STATE: { 2528 struct kvm_mp_state mp_state; 2529 2530 r = -EFAULT; 2531 if (copy_from_user(&mp_state, argp, sizeof(mp_state))) 2532 goto out; 2533 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 2534 break; 2535 } 2536 case KVM_TRANSLATE: { 2537 struct kvm_translation tr; 2538 2539 r = -EFAULT; 2540 if (copy_from_user(&tr, argp, sizeof(tr))) 2541 goto out; 2542 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 2543 if (r) 2544 goto out; 2545 r = -EFAULT; 2546 if (copy_to_user(argp, &tr, sizeof(tr))) 2547 goto out; 2548 r = 0; 2549 break; 2550 } 2551 case KVM_SET_GUEST_DEBUG: { 2552 struct kvm_guest_debug dbg; 2553 2554 r = -EFAULT; 2555 if (copy_from_user(&dbg, argp, sizeof(dbg))) 2556 goto out; 2557 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg); 2558 break; 2559 } 2560 case KVM_SET_SIGNAL_MASK: { 2561 struct kvm_signal_mask __user *sigmask_arg = argp; 2562 struct kvm_signal_mask kvm_sigmask; 2563 sigset_t sigset, *p; 2564 2565 p = NULL; 2566 if (argp) { 2567 r = -EFAULT; 2568 if (copy_from_user(&kvm_sigmask, argp, 2569 sizeof(kvm_sigmask))) 2570 goto out; 2571 r = -EINVAL; 2572 if (kvm_sigmask.len != sizeof(sigset)) 2573 goto out; 2574 r = -EFAULT; 2575 if (copy_from_user(&sigset, sigmask_arg->sigset, 2576 sizeof(sigset))) 2577 goto out; 2578 p = &sigset; 2579 } 2580 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p); 2581 break; 2582 } 2583 case KVM_GET_FPU: { 2584 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); 2585 r = -ENOMEM; 2586 if (!fpu) 2587 goto out; 2588 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu); 2589 if (r) 2590 goto out; 2591 r = -EFAULT; 2592 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu))) 2593 goto out; 2594 r = 0; 2595 break; 2596 } 2597 case KVM_SET_FPU: { 2598 fpu = memdup_user(argp, sizeof(*fpu)); 2599 if (IS_ERR(fpu)) { 2600 r = PTR_ERR(fpu); 2601 fpu = NULL; 2602 goto out; 2603 } 2604 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); 2605 break; 2606 } 2607 default: 2608 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 2609 } 2610 out: 2611 vcpu_put(vcpu); 2612 kfree(fpu); 2613 kfree(kvm_sregs); 2614 return r; 2615 } 2616 2617 #ifdef CONFIG_KVM_COMPAT 2618 static long kvm_vcpu_compat_ioctl(struct file *filp, 2619 unsigned int ioctl, unsigned long arg) 2620 { 2621 struct kvm_vcpu *vcpu = filp->private_data; 2622 void __user *argp = compat_ptr(arg); 2623 int r; 2624 2625 if (vcpu->kvm->mm != current->mm) 2626 return -EIO; 2627 2628 switch (ioctl) { 2629 case KVM_SET_SIGNAL_MASK: { 2630 struct kvm_signal_mask __user *sigmask_arg = argp; 2631 struct kvm_signal_mask kvm_sigmask; 2632 compat_sigset_t csigset; 2633 sigset_t sigset; 2634 2635 if (argp) { 2636 r = -EFAULT; 2637 if (copy_from_user(&kvm_sigmask, argp, 2638 sizeof(kvm_sigmask))) 2639 goto out; 2640 r = -EINVAL; 2641 if (kvm_sigmask.len != sizeof(csigset)) 2642 goto out; 2643 r = -EFAULT; 2644 if (copy_from_user(&csigset, sigmask_arg->sigset, 2645 sizeof(csigset))) 2646 goto out; 2647 sigset_from_compat(&sigset, &csigset); 2648 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 2649 } else 2650 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL); 2651 break; 2652 } 2653 default: 2654 r = kvm_vcpu_ioctl(filp, ioctl, arg); 2655 } 2656 2657 out: 2658 return r; 2659 } 2660 #endif 2661 2662 static int kvm_device_ioctl_attr(struct kvm_device *dev, 2663 int (*accessor)(struct kvm_device *dev, 2664 struct kvm_device_attr *attr), 2665 unsigned long arg) 2666 { 2667 struct kvm_device_attr attr; 2668 2669 if (!accessor) 2670 return -EPERM; 2671 2672 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2673 return -EFAULT; 2674 2675 return accessor(dev, &attr); 2676 } 2677 2678 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl, 2679 unsigned long arg) 2680 { 2681 struct kvm_device *dev = filp->private_data; 2682 2683 switch (ioctl) { 2684 case KVM_SET_DEVICE_ATTR: 2685 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg); 2686 case KVM_GET_DEVICE_ATTR: 2687 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg); 2688 case KVM_HAS_DEVICE_ATTR: 2689 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg); 2690 default: 2691 if (dev->ops->ioctl) 2692 return dev->ops->ioctl(dev, ioctl, arg); 2693 2694 return -ENOTTY; 2695 } 2696 } 2697 2698 static int kvm_device_release(struct inode *inode, struct file *filp) 2699 { 2700 struct kvm_device *dev = filp->private_data; 2701 struct kvm *kvm = dev->kvm; 2702 2703 kvm_put_kvm(kvm); 2704 return 0; 2705 } 2706 2707 static const struct file_operations kvm_device_fops = { 2708 .unlocked_ioctl = kvm_device_ioctl, 2709 #ifdef CONFIG_KVM_COMPAT 2710 .compat_ioctl = kvm_device_ioctl, 2711 #endif 2712 .release = kvm_device_release, 2713 }; 2714 2715 struct kvm_device *kvm_device_from_filp(struct file *filp) 2716 { 2717 if (filp->f_op != &kvm_device_fops) 2718 return NULL; 2719 2720 return filp->private_data; 2721 } 2722 2723 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = { 2724 #ifdef CONFIG_KVM_MPIC 2725 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops, 2726 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops, 2727 #endif 2728 2729 #ifdef CONFIG_KVM_XICS 2730 [KVM_DEV_TYPE_XICS] = &kvm_xics_ops, 2731 #endif 2732 }; 2733 2734 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type) 2735 { 2736 if (type >= ARRAY_SIZE(kvm_device_ops_table)) 2737 return -ENOSPC; 2738 2739 if (kvm_device_ops_table[type] != NULL) 2740 return -EEXIST; 2741 2742 kvm_device_ops_table[type] = ops; 2743 return 0; 2744 } 2745 2746 void kvm_unregister_device_ops(u32 type) 2747 { 2748 if (kvm_device_ops_table[type] != NULL) 2749 kvm_device_ops_table[type] = NULL; 2750 } 2751 2752 static int kvm_ioctl_create_device(struct kvm *kvm, 2753 struct kvm_create_device *cd) 2754 { 2755 struct kvm_device_ops *ops = NULL; 2756 struct kvm_device *dev; 2757 bool test = cd->flags & KVM_CREATE_DEVICE_TEST; 2758 int ret; 2759 2760 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table)) 2761 return -ENODEV; 2762 2763 ops = kvm_device_ops_table[cd->type]; 2764 if (ops == NULL) 2765 return -ENODEV; 2766 2767 if (test) 2768 return 0; 2769 2770 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2771 if (!dev) 2772 return -ENOMEM; 2773 2774 dev->ops = ops; 2775 dev->kvm = kvm; 2776 2777 ret = ops->create(dev, cd->type); 2778 if (ret < 0) { 2779 kfree(dev); 2780 return ret; 2781 } 2782 2783 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC); 2784 if (ret < 0) { 2785 ops->destroy(dev); 2786 return ret; 2787 } 2788 2789 list_add(&dev->vm_node, &kvm->devices); 2790 kvm_get_kvm(kvm); 2791 cd->fd = ret; 2792 return 0; 2793 } 2794 2795 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) 2796 { 2797 switch (arg) { 2798 case KVM_CAP_USER_MEMORY: 2799 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 2800 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 2801 case KVM_CAP_INTERNAL_ERROR_DATA: 2802 #ifdef CONFIG_HAVE_KVM_MSI 2803 case KVM_CAP_SIGNAL_MSI: 2804 #endif 2805 #ifdef CONFIG_HAVE_KVM_IRQFD 2806 case KVM_CAP_IRQFD: 2807 case KVM_CAP_IRQFD_RESAMPLE: 2808 #endif 2809 case KVM_CAP_IOEVENTFD_ANY_LENGTH: 2810 case KVM_CAP_CHECK_EXTENSION_VM: 2811 return 1; 2812 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2813 case KVM_CAP_IRQ_ROUTING: 2814 return KVM_MAX_IRQ_ROUTES; 2815 #endif 2816 #if KVM_ADDRESS_SPACE_NUM > 1 2817 case KVM_CAP_MULTI_ADDRESS_SPACE: 2818 return KVM_ADDRESS_SPACE_NUM; 2819 #endif 2820 case KVM_CAP_MAX_VCPU_ID: 2821 return KVM_MAX_VCPU_ID; 2822 default: 2823 break; 2824 } 2825 return kvm_vm_ioctl_check_extension(kvm, arg); 2826 } 2827 2828 static long kvm_vm_ioctl(struct file *filp, 2829 unsigned int ioctl, unsigned long arg) 2830 { 2831 struct kvm *kvm = filp->private_data; 2832 void __user *argp = (void __user *)arg; 2833 int r; 2834 2835 if (kvm->mm != current->mm) 2836 return -EIO; 2837 switch (ioctl) { 2838 case KVM_CREATE_VCPU: 2839 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 2840 break; 2841 case KVM_SET_USER_MEMORY_REGION: { 2842 struct kvm_userspace_memory_region kvm_userspace_mem; 2843 2844 r = -EFAULT; 2845 if (copy_from_user(&kvm_userspace_mem, argp, 2846 sizeof(kvm_userspace_mem))) 2847 goto out; 2848 2849 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); 2850 break; 2851 } 2852 case KVM_GET_DIRTY_LOG: { 2853 struct kvm_dirty_log log; 2854 2855 r = -EFAULT; 2856 if (copy_from_user(&log, argp, sizeof(log))) 2857 goto out; 2858 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 2859 break; 2860 } 2861 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 2862 case KVM_REGISTER_COALESCED_MMIO: { 2863 struct kvm_coalesced_mmio_zone zone; 2864 2865 r = -EFAULT; 2866 if (copy_from_user(&zone, argp, sizeof(zone))) 2867 goto out; 2868 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 2869 break; 2870 } 2871 case KVM_UNREGISTER_COALESCED_MMIO: { 2872 struct kvm_coalesced_mmio_zone zone; 2873 2874 r = -EFAULT; 2875 if (copy_from_user(&zone, argp, sizeof(zone))) 2876 goto out; 2877 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 2878 break; 2879 } 2880 #endif 2881 case KVM_IRQFD: { 2882 struct kvm_irqfd data; 2883 2884 r = -EFAULT; 2885 if (copy_from_user(&data, argp, sizeof(data))) 2886 goto out; 2887 r = kvm_irqfd(kvm, &data); 2888 break; 2889 } 2890 case KVM_IOEVENTFD: { 2891 struct kvm_ioeventfd data; 2892 2893 r = -EFAULT; 2894 if (copy_from_user(&data, argp, sizeof(data))) 2895 goto out; 2896 r = kvm_ioeventfd(kvm, &data); 2897 break; 2898 } 2899 #ifdef CONFIG_HAVE_KVM_MSI 2900 case KVM_SIGNAL_MSI: { 2901 struct kvm_msi msi; 2902 2903 r = -EFAULT; 2904 if (copy_from_user(&msi, argp, sizeof(msi))) 2905 goto out; 2906 r = kvm_send_userspace_msi(kvm, &msi); 2907 break; 2908 } 2909 #endif 2910 #ifdef __KVM_HAVE_IRQ_LINE 2911 case KVM_IRQ_LINE_STATUS: 2912 case KVM_IRQ_LINE: { 2913 struct kvm_irq_level irq_event; 2914 2915 r = -EFAULT; 2916 if (copy_from_user(&irq_event, argp, sizeof(irq_event))) 2917 goto out; 2918 2919 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, 2920 ioctl == KVM_IRQ_LINE_STATUS); 2921 if (r) 2922 goto out; 2923 2924 r = -EFAULT; 2925 if (ioctl == KVM_IRQ_LINE_STATUS) { 2926 if (copy_to_user(argp, &irq_event, sizeof(irq_event))) 2927 goto out; 2928 } 2929 2930 r = 0; 2931 break; 2932 } 2933 #endif 2934 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2935 case KVM_SET_GSI_ROUTING: { 2936 struct kvm_irq_routing routing; 2937 struct kvm_irq_routing __user *urouting; 2938 struct kvm_irq_routing_entry *entries = NULL; 2939 2940 r = -EFAULT; 2941 if (copy_from_user(&routing, argp, sizeof(routing))) 2942 goto out; 2943 r = -EINVAL; 2944 if (routing.nr >= KVM_MAX_IRQ_ROUTES) 2945 goto out; 2946 if (routing.flags) 2947 goto out; 2948 if (routing.nr) { 2949 r = -ENOMEM; 2950 entries = vmalloc(routing.nr * sizeof(*entries)); 2951 if (!entries) 2952 goto out; 2953 r = -EFAULT; 2954 urouting = argp; 2955 if (copy_from_user(entries, urouting->entries, 2956 routing.nr * sizeof(*entries))) 2957 goto out_free_irq_routing; 2958 } 2959 r = kvm_set_irq_routing(kvm, entries, routing.nr, 2960 routing.flags); 2961 out_free_irq_routing: 2962 vfree(entries); 2963 break; 2964 } 2965 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ 2966 case KVM_CREATE_DEVICE: { 2967 struct kvm_create_device cd; 2968 2969 r = -EFAULT; 2970 if (copy_from_user(&cd, argp, sizeof(cd))) 2971 goto out; 2972 2973 r = kvm_ioctl_create_device(kvm, &cd); 2974 if (r) 2975 goto out; 2976 2977 r = -EFAULT; 2978 if (copy_to_user(argp, &cd, sizeof(cd))) 2979 goto out; 2980 2981 r = 0; 2982 break; 2983 } 2984 case KVM_CHECK_EXTENSION: 2985 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); 2986 break; 2987 default: 2988 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 2989 } 2990 out: 2991 return r; 2992 } 2993 2994 #ifdef CONFIG_KVM_COMPAT 2995 struct compat_kvm_dirty_log { 2996 __u32 slot; 2997 __u32 padding1; 2998 union { 2999 compat_uptr_t dirty_bitmap; /* one bit per page */ 3000 __u64 padding2; 3001 }; 3002 }; 3003 3004 static long kvm_vm_compat_ioctl(struct file *filp, 3005 unsigned int ioctl, unsigned long arg) 3006 { 3007 struct kvm *kvm = filp->private_data; 3008 int r; 3009 3010 if (kvm->mm != current->mm) 3011 return -EIO; 3012 switch (ioctl) { 3013 case KVM_GET_DIRTY_LOG: { 3014 struct compat_kvm_dirty_log compat_log; 3015 struct kvm_dirty_log log; 3016 3017 r = -EFAULT; 3018 if (copy_from_user(&compat_log, (void __user *)arg, 3019 sizeof(compat_log))) 3020 goto out; 3021 log.slot = compat_log.slot; 3022 log.padding1 = compat_log.padding1; 3023 log.padding2 = compat_log.padding2; 3024 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap); 3025 3026 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 3027 break; 3028 } 3029 default: 3030 r = kvm_vm_ioctl(filp, ioctl, arg); 3031 } 3032 3033 out: 3034 return r; 3035 } 3036 #endif 3037 3038 static struct file_operations kvm_vm_fops = { 3039 .release = kvm_vm_release, 3040 .unlocked_ioctl = kvm_vm_ioctl, 3041 #ifdef CONFIG_KVM_COMPAT 3042 .compat_ioctl = kvm_vm_compat_ioctl, 3043 #endif 3044 .llseek = noop_llseek, 3045 }; 3046 3047 static int kvm_dev_ioctl_create_vm(unsigned long type) 3048 { 3049 int r; 3050 struct kvm *kvm; 3051 3052 kvm = kvm_create_vm(type); 3053 if (IS_ERR(kvm)) 3054 return PTR_ERR(kvm); 3055 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 3056 r = kvm_coalesced_mmio_init(kvm); 3057 if (r < 0) { 3058 kvm_put_kvm(kvm); 3059 return r; 3060 } 3061 #endif 3062 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC); 3063 if (r < 0) { 3064 kvm_put_kvm(kvm); 3065 return r; 3066 } 3067 3068 if (kvm_create_vm_debugfs(kvm, r) < 0) { 3069 kvm_put_kvm(kvm); 3070 return -ENOMEM; 3071 } 3072 3073 return r; 3074 } 3075 3076 static long kvm_dev_ioctl(struct file *filp, 3077 unsigned int ioctl, unsigned long arg) 3078 { 3079 long r = -EINVAL; 3080 3081 switch (ioctl) { 3082 case KVM_GET_API_VERSION: 3083 if (arg) 3084 goto out; 3085 r = KVM_API_VERSION; 3086 break; 3087 case KVM_CREATE_VM: 3088 r = kvm_dev_ioctl_create_vm(arg); 3089 break; 3090 case KVM_CHECK_EXTENSION: 3091 r = kvm_vm_ioctl_check_extension_generic(NULL, arg); 3092 break; 3093 case KVM_GET_VCPU_MMAP_SIZE: 3094 if (arg) 3095 goto out; 3096 r = PAGE_SIZE; /* struct kvm_run */ 3097 #ifdef CONFIG_X86 3098 r += PAGE_SIZE; /* pio data page */ 3099 #endif 3100 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 3101 r += PAGE_SIZE; /* coalesced mmio ring page */ 3102 #endif 3103 break; 3104 case KVM_TRACE_ENABLE: 3105 case KVM_TRACE_PAUSE: 3106 case KVM_TRACE_DISABLE: 3107 r = -EOPNOTSUPP; 3108 break; 3109 default: 3110 return kvm_arch_dev_ioctl(filp, ioctl, arg); 3111 } 3112 out: 3113 return r; 3114 } 3115 3116 static struct file_operations kvm_chardev_ops = { 3117 .unlocked_ioctl = kvm_dev_ioctl, 3118 .compat_ioctl = kvm_dev_ioctl, 3119 .llseek = noop_llseek, 3120 }; 3121 3122 static struct miscdevice kvm_dev = { 3123 KVM_MINOR, 3124 "kvm", 3125 &kvm_chardev_ops, 3126 }; 3127 3128 static void hardware_enable_nolock(void *junk) 3129 { 3130 int cpu = raw_smp_processor_id(); 3131 int r; 3132 3133 if (cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3134 return; 3135 3136 cpumask_set_cpu(cpu, cpus_hardware_enabled); 3137 3138 r = kvm_arch_hardware_enable(); 3139 3140 if (r) { 3141 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3142 atomic_inc(&hardware_enable_failed); 3143 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu); 3144 } 3145 } 3146 3147 static void hardware_enable(void) 3148 { 3149 raw_spin_lock(&kvm_count_lock); 3150 if (kvm_usage_count) 3151 hardware_enable_nolock(NULL); 3152 raw_spin_unlock(&kvm_count_lock); 3153 } 3154 3155 static void hardware_disable_nolock(void *junk) 3156 { 3157 int cpu = raw_smp_processor_id(); 3158 3159 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled)) 3160 return; 3161 cpumask_clear_cpu(cpu, cpus_hardware_enabled); 3162 kvm_arch_hardware_disable(); 3163 } 3164 3165 static void hardware_disable(void) 3166 { 3167 raw_spin_lock(&kvm_count_lock); 3168 if (kvm_usage_count) 3169 hardware_disable_nolock(NULL); 3170 raw_spin_unlock(&kvm_count_lock); 3171 } 3172 3173 static void hardware_disable_all_nolock(void) 3174 { 3175 BUG_ON(!kvm_usage_count); 3176 3177 kvm_usage_count--; 3178 if (!kvm_usage_count) 3179 on_each_cpu(hardware_disable_nolock, NULL, 1); 3180 } 3181 3182 static void hardware_disable_all(void) 3183 { 3184 raw_spin_lock(&kvm_count_lock); 3185 hardware_disable_all_nolock(); 3186 raw_spin_unlock(&kvm_count_lock); 3187 } 3188 3189 static int hardware_enable_all(void) 3190 { 3191 int r = 0; 3192 3193 raw_spin_lock(&kvm_count_lock); 3194 3195 kvm_usage_count++; 3196 if (kvm_usage_count == 1) { 3197 atomic_set(&hardware_enable_failed, 0); 3198 on_each_cpu(hardware_enable_nolock, NULL, 1); 3199 3200 if (atomic_read(&hardware_enable_failed)) { 3201 hardware_disable_all_nolock(); 3202 r = -EBUSY; 3203 } 3204 } 3205 3206 raw_spin_unlock(&kvm_count_lock); 3207 3208 return r; 3209 } 3210 3211 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 3212 void *v) 3213 { 3214 val &= ~CPU_TASKS_FROZEN; 3215 switch (val) { 3216 case CPU_DYING: 3217 hardware_disable(); 3218 break; 3219 case CPU_STARTING: 3220 hardware_enable(); 3221 break; 3222 } 3223 return NOTIFY_OK; 3224 } 3225 3226 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 3227 void *v) 3228 { 3229 /* 3230 * Some (well, at least mine) BIOSes hang on reboot if 3231 * in vmx root mode. 3232 * 3233 * And Intel TXT required VMX off for all cpu when system shutdown. 3234 */ 3235 pr_info("kvm: exiting hardware virtualization\n"); 3236 kvm_rebooting = true; 3237 on_each_cpu(hardware_disable_nolock, NULL, 1); 3238 return NOTIFY_OK; 3239 } 3240 3241 static struct notifier_block kvm_reboot_notifier = { 3242 .notifier_call = kvm_reboot, 3243 .priority = 0, 3244 }; 3245 3246 static void kvm_io_bus_destroy(struct kvm_io_bus *bus) 3247 { 3248 int i; 3249 3250 for (i = 0; i < bus->dev_count; i++) { 3251 struct kvm_io_device *pos = bus->range[i].dev; 3252 3253 kvm_iodevice_destructor(pos); 3254 } 3255 kfree(bus); 3256 } 3257 3258 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1, 3259 const struct kvm_io_range *r2) 3260 { 3261 gpa_t addr1 = r1->addr; 3262 gpa_t addr2 = r2->addr; 3263 3264 if (addr1 < addr2) 3265 return -1; 3266 3267 /* If r2->len == 0, match the exact address. If r2->len != 0, 3268 * accept any overlapping write. Any order is acceptable for 3269 * overlapping ranges, because kvm_io_bus_get_first_dev ensures 3270 * we process all of them. 3271 */ 3272 if (r2->len) { 3273 addr1 += r1->len; 3274 addr2 += r2->len; 3275 } 3276 3277 if (addr1 > addr2) 3278 return 1; 3279 3280 return 0; 3281 } 3282 3283 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2) 3284 { 3285 return kvm_io_bus_cmp(p1, p2); 3286 } 3287 3288 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev, 3289 gpa_t addr, int len) 3290 { 3291 bus->range[bus->dev_count++] = (struct kvm_io_range) { 3292 .addr = addr, 3293 .len = len, 3294 .dev = dev, 3295 }; 3296 3297 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range), 3298 kvm_io_bus_sort_cmp, NULL); 3299 3300 return 0; 3301 } 3302 3303 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus, 3304 gpa_t addr, int len) 3305 { 3306 struct kvm_io_range *range, key; 3307 int off; 3308 3309 key = (struct kvm_io_range) { 3310 .addr = addr, 3311 .len = len, 3312 }; 3313 3314 range = bsearch(&key, bus->range, bus->dev_count, 3315 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp); 3316 if (range == NULL) 3317 return -ENOENT; 3318 3319 off = range - bus->range; 3320 3321 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0) 3322 off--; 3323 3324 return off; 3325 } 3326 3327 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3328 struct kvm_io_range *range, const void *val) 3329 { 3330 int idx; 3331 3332 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3333 if (idx < 0) 3334 return -EOPNOTSUPP; 3335 3336 while (idx < bus->dev_count && 3337 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3338 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr, 3339 range->len, val)) 3340 return idx; 3341 idx++; 3342 } 3343 3344 return -EOPNOTSUPP; 3345 } 3346 3347 /* kvm_io_bus_write - called under kvm->slots_lock */ 3348 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3349 int len, const void *val) 3350 { 3351 struct kvm_io_bus *bus; 3352 struct kvm_io_range range; 3353 int r; 3354 3355 range = (struct kvm_io_range) { 3356 .addr = addr, 3357 .len = len, 3358 }; 3359 3360 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3361 r = __kvm_io_bus_write(vcpu, bus, &range, val); 3362 return r < 0 ? r : 0; 3363 } 3364 3365 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ 3366 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 3367 gpa_t addr, int len, const void *val, long cookie) 3368 { 3369 struct kvm_io_bus *bus; 3370 struct kvm_io_range range; 3371 3372 range = (struct kvm_io_range) { 3373 .addr = addr, 3374 .len = len, 3375 }; 3376 3377 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3378 3379 /* First try the device referenced by cookie. */ 3380 if ((cookie >= 0) && (cookie < bus->dev_count) && 3381 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0)) 3382 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len, 3383 val)) 3384 return cookie; 3385 3386 /* 3387 * cookie contained garbage; fall back to search and return the 3388 * correct cookie value. 3389 */ 3390 return __kvm_io_bus_write(vcpu, bus, &range, val); 3391 } 3392 3393 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, 3394 struct kvm_io_range *range, void *val) 3395 { 3396 int idx; 3397 3398 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len); 3399 if (idx < 0) 3400 return -EOPNOTSUPP; 3401 3402 while (idx < bus->dev_count && 3403 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) { 3404 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr, 3405 range->len, val)) 3406 return idx; 3407 idx++; 3408 } 3409 3410 return -EOPNOTSUPP; 3411 } 3412 EXPORT_SYMBOL_GPL(kvm_io_bus_write); 3413 3414 /* kvm_io_bus_read - called under kvm->slots_lock */ 3415 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 3416 int len, void *val) 3417 { 3418 struct kvm_io_bus *bus; 3419 struct kvm_io_range range; 3420 int r; 3421 3422 range = (struct kvm_io_range) { 3423 .addr = addr, 3424 .len = len, 3425 }; 3426 3427 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); 3428 r = __kvm_io_bus_read(vcpu, bus, &range, val); 3429 return r < 0 ? r : 0; 3430 } 3431 3432 3433 /* Caller must hold slots_lock. */ 3434 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 3435 int len, struct kvm_io_device *dev) 3436 { 3437 struct kvm_io_bus *new_bus, *bus; 3438 3439 bus = kvm->buses[bus_idx]; 3440 /* exclude ioeventfd which is limited by maximum fd */ 3441 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) 3442 return -ENOSPC; 3443 3444 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) * 3445 sizeof(struct kvm_io_range)), GFP_KERNEL); 3446 if (!new_bus) 3447 return -ENOMEM; 3448 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count * 3449 sizeof(struct kvm_io_range))); 3450 kvm_io_bus_insert_dev(new_bus, dev, addr, len); 3451 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3452 synchronize_srcu_expedited(&kvm->srcu); 3453 kfree(bus); 3454 3455 return 0; 3456 } 3457 3458 /* Caller must hold slots_lock. */ 3459 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 3460 struct kvm_io_device *dev) 3461 { 3462 int i, r; 3463 struct kvm_io_bus *new_bus, *bus; 3464 3465 bus = kvm->buses[bus_idx]; 3466 r = -ENOENT; 3467 for (i = 0; i < bus->dev_count; i++) 3468 if (bus->range[i].dev == dev) { 3469 r = 0; 3470 break; 3471 } 3472 3473 if (r) 3474 return r; 3475 3476 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) * 3477 sizeof(struct kvm_io_range)), GFP_KERNEL); 3478 if (!new_bus) 3479 return -ENOMEM; 3480 3481 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range)); 3482 new_bus->dev_count--; 3483 memcpy(new_bus->range + i, bus->range + i + 1, 3484 (new_bus->dev_count - i) * sizeof(struct kvm_io_range)); 3485 3486 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); 3487 synchronize_srcu_expedited(&kvm->srcu); 3488 kfree(bus); 3489 return r; 3490 } 3491 3492 static struct notifier_block kvm_cpu_notifier = { 3493 .notifier_call = kvm_cpu_hotplug, 3494 }; 3495 3496 static int kvm_debugfs_open(struct inode *inode, struct file *file, 3497 int (*get)(void *, u64 *), int (*set)(void *, u64), 3498 const char *fmt) 3499 { 3500 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 3501 inode->i_private; 3502 3503 /* The debugfs files are a reference to the kvm struct which 3504 * is still valid when kvm_destroy_vm is called. 3505 * To avoid the race between open and the removal of the debugfs 3506 * directory we test against the users count. 3507 */ 3508 if (!atomic_add_unless(&stat_data->kvm->users_count, 1, 0)) 3509 return -ENOENT; 3510 3511 if (simple_attr_open(inode, file, get, set, fmt)) { 3512 kvm_put_kvm(stat_data->kvm); 3513 return -ENOMEM; 3514 } 3515 3516 return 0; 3517 } 3518 3519 static int kvm_debugfs_release(struct inode *inode, struct file *file) 3520 { 3521 struct kvm_stat_data *stat_data = (struct kvm_stat_data *) 3522 inode->i_private; 3523 3524 simple_attr_release(inode, file); 3525 kvm_put_kvm(stat_data->kvm); 3526 3527 return 0; 3528 } 3529 3530 static int vm_stat_get_per_vm(void *data, u64 *val) 3531 { 3532 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3533 3534 *val = *(u32 *)((void *)stat_data->kvm + stat_data->offset); 3535 3536 return 0; 3537 } 3538 3539 static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file) 3540 { 3541 __simple_attr_check_format("%llu\n", 0ull); 3542 return kvm_debugfs_open(inode, file, vm_stat_get_per_vm, 3543 NULL, "%llu\n"); 3544 } 3545 3546 static const struct file_operations vm_stat_get_per_vm_fops = { 3547 .owner = THIS_MODULE, 3548 .open = vm_stat_get_per_vm_open, 3549 .release = kvm_debugfs_release, 3550 .read = simple_attr_read, 3551 .write = simple_attr_write, 3552 .llseek = generic_file_llseek, 3553 }; 3554 3555 static int vcpu_stat_get_per_vm(void *data, u64 *val) 3556 { 3557 int i; 3558 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; 3559 struct kvm_vcpu *vcpu; 3560 3561 *val = 0; 3562 3563 kvm_for_each_vcpu(i, vcpu, stat_data->kvm) 3564 *val += *(u32 *)((void *)vcpu + stat_data->offset); 3565 3566 return 0; 3567 } 3568 3569 static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file) 3570 { 3571 __simple_attr_check_format("%llu\n", 0ull); 3572 return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm, 3573 NULL, "%llu\n"); 3574 } 3575 3576 static const struct file_operations vcpu_stat_get_per_vm_fops = { 3577 .owner = THIS_MODULE, 3578 .open = vcpu_stat_get_per_vm_open, 3579 .release = kvm_debugfs_release, 3580 .read = simple_attr_read, 3581 .write = simple_attr_write, 3582 .llseek = generic_file_llseek, 3583 }; 3584 3585 static const struct file_operations *stat_fops_per_vm[] = { 3586 [KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops, 3587 [KVM_STAT_VM] = &vm_stat_get_per_vm_fops, 3588 }; 3589 3590 static int vm_stat_get(void *_offset, u64 *val) 3591 { 3592 unsigned offset = (long)_offset; 3593 struct kvm *kvm; 3594 struct kvm_stat_data stat_tmp = {.offset = offset}; 3595 u64 tmp_val; 3596 3597 *val = 0; 3598 spin_lock(&kvm_lock); 3599 list_for_each_entry(kvm, &vm_list, vm_list) { 3600 stat_tmp.kvm = kvm; 3601 vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val); 3602 *val += tmp_val; 3603 } 3604 spin_unlock(&kvm_lock); 3605 return 0; 3606 } 3607 3608 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 3609 3610 static int vcpu_stat_get(void *_offset, u64 *val) 3611 { 3612 unsigned offset = (long)_offset; 3613 struct kvm *kvm; 3614 struct kvm_stat_data stat_tmp = {.offset = offset}; 3615 u64 tmp_val; 3616 3617 *val = 0; 3618 spin_lock(&kvm_lock); 3619 list_for_each_entry(kvm, &vm_list, vm_list) { 3620 stat_tmp.kvm = kvm; 3621 vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val); 3622 *val += tmp_val; 3623 } 3624 spin_unlock(&kvm_lock); 3625 return 0; 3626 } 3627 3628 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 3629 3630 static const struct file_operations *stat_fops[] = { 3631 [KVM_STAT_VCPU] = &vcpu_stat_fops, 3632 [KVM_STAT_VM] = &vm_stat_fops, 3633 }; 3634 3635 static int kvm_init_debug(void) 3636 { 3637 int r = -EEXIST; 3638 struct kvm_stats_debugfs_item *p; 3639 3640 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 3641 if (kvm_debugfs_dir == NULL) 3642 goto out; 3643 3644 kvm_debugfs_num_entries = 0; 3645 for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) { 3646 if (!debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 3647 (void *)(long)p->offset, 3648 stat_fops[p->kind])) 3649 goto out_dir; 3650 } 3651 3652 return 0; 3653 3654 out_dir: 3655 debugfs_remove_recursive(kvm_debugfs_dir); 3656 out: 3657 return r; 3658 } 3659 3660 static int kvm_suspend(void) 3661 { 3662 if (kvm_usage_count) 3663 hardware_disable_nolock(NULL); 3664 return 0; 3665 } 3666 3667 static void kvm_resume(void) 3668 { 3669 if (kvm_usage_count) { 3670 WARN_ON(raw_spin_is_locked(&kvm_count_lock)); 3671 hardware_enable_nolock(NULL); 3672 } 3673 } 3674 3675 static struct syscore_ops kvm_syscore_ops = { 3676 .suspend = kvm_suspend, 3677 .resume = kvm_resume, 3678 }; 3679 3680 static inline 3681 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 3682 { 3683 return container_of(pn, struct kvm_vcpu, preempt_notifier); 3684 } 3685 3686 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 3687 { 3688 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3689 3690 if (vcpu->preempted) 3691 vcpu->preempted = false; 3692 3693 kvm_arch_sched_in(vcpu, cpu); 3694 3695 kvm_arch_vcpu_load(vcpu, cpu); 3696 } 3697 3698 static void kvm_sched_out(struct preempt_notifier *pn, 3699 struct task_struct *next) 3700 { 3701 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 3702 3703 if (current->state == TASK_RUNNING) 3704 vcpu->preempted = true; 3705 kvm_arch_vcpu_put(vcpu); 3706 } 3707 3708 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 3709 struct module *module) 3710 { 3711 int r; 3712 int cpu; 3713 3714 r = kvm_arch_init(opaque); 3715 if (r) 3716 goto out_fail; 3717 3718 /* 3719 * kvm_arch_init makes sure there's at most one caller 3720 * for architectures that support multiple implementations, 3721 * like intel and amd on x86. 3722 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating 3723 * conflicts in case kvm is already setup for another implementation. 3724 */ 3725 r = kvm_irqfd_init(); 3726 if (r) 3727 goto out_irqfd; 3728 3729 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { 3730 r = -ENOMEM; 3731 goto out_free_0; 3732 } 3733 3734 r = kvm_arch_hardware_setup(); 3735 if (r < 0) 3736 goto out_free_0a; 3737 3738 for_each_online_cpu(cpu) { 3739 smp_call_function_single(cpu, 3740 kvm_arch_check_processor_compat, 3741 &r, 1); 3742 if (r < 0) 3743 goto out_free_1; 3744 } 3745 3746 r = register_cpu_notifier(&kvm_cpu_notifier); 3747 if (r) 3748 goto out_free_2; 3749 register_reboot_notifier(&kvm_reboot_notifier); 3750 3751 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 3752 if (!vcpu_align) 3753 vcpu_align = __alignof__(struct kvm_vcpu); 3754 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align, 3755 0, NULL); 3756 if (!kvm_vcpu_cache) { 3757 r = -ENOMEM; 3758 goto out_free_3; 3759 } 3760 3761 r = kvm_async_pf_init(); 3762 if (r) 3763 goto out_free; 3764 3765 kvm_chardev_ops.owner = module; 3766 kvm_vm_fops.owner = module; 3767 kvm_vcpu_fops.owner = module; 3768 3769 r = misc_register(&kvm_dev); 3770 if (r) { 3771 pr_err("kvm: misc device register failed\n"); 3772 goto out_unreg; 3773 } 3774 3775 register_syscore_ops(&kvm_syscore_ops); 3776 3777 kvm_preempt_ops.sched_in = kvm_sched_in; 3778 kvm_preempt_ops.sched_out = kvm_sched_out; 3779 3780 r = kvm_init_debug(); 3781 if (r) { 3782 pr_err("kvm: create debugfs files failed\n"); 3783 goto out_undebugfs; 3784 } 3785 3786 r = kvm_vfio_ops_init(); 3787 WARN_ON(r); 3788 3789 return 0; 3790 3791 out_undebugfs: 3792 unregister_syscore_ops(&kvm_syscore_ops); 3793 misc_deregister(&kvm_dev); 3794 out_unreg: 3795 kvm_async_pf_deinit(); 3796 out_free: 3797 kmem_cache_destroy(kvm_vcpu_cache); 3798 out_free_3: 3799 unregister_reboot_notifier(&kvm_reboot_notifier); 3800 unregister_cpu_notifier(&kvm_cpu_notifier); 3801 out_free_2: 3802 out_free_1: 3803 kvm_arch_hardware_unsetup(); 3804 out_free_0a: 3805 free_cpumask_var(cpus_hardware_enabled); 3806 out_free_0: 3807 kvm_irqfd_exit(); 3808 out_irqfd: 3809 kvm_arch_exit(); 3810 out_fail: 3811 return r; 3812 } 3813 EXPORT_SYMBOL_GPL(kvm_init); 3814 3815 void kvm_exit(void) 3816 { 3817 debugfs_remove_recursive(kvm_debugfs_dir); 3818 misc_deregister(&kvm_dev); 3819 kmem_cache_destroy(kvm_vcpu_cache); 3820 kvm_async_pf_deinit(); 3821 unregister_syscore_ops(&kvm_syscore_ops); 3822 unregister_reboot_notifier(&kvm_reboot_notifier); 3823 unregister_cpu_notifier(&kvm_cpu_notifier); 3824 on_each_cpu(hardware_disable_nolock, NULL, 1); 3825 kvm_arch_hardware_unsetup(); 3826 kvm_arch_exit(); 3827 kvm_irqfd_exit(); 3828 free_cpumask_var(cpus_hardware_enabled); 3829 kvm_vfio_ops_exit(); 3830 } 3831 EXPORT_SYMBOL_GPL(kvm_exit); 3832