1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * This module enables machines with Intel VT-x extensions to run virtual 5 * machines without emulation or binary translation. 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * 9 * Authors: 10 * Avi Kivity <avi@qumranet.com> 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * 13 * This work is licensed under the terms of the GNU GPL, version 2. See 14 * the COPYING file in the top-level directory. 15 * 16 */ 17 18 #include "iodev.h" 19 20 #include <linux/kvm_host.h> 21 #include <linux/kvm.h> 22 #include <linux/module.h> 23 #include <linux/errno.h> 24 #include <linux/percpu.h> 25 #include <linux/gfp.h> 26 #include <linux/mm.h> 27 #include <linux/miscdevice.h> 28 #include <linux/vmalloc.h> 29 #include <linux/reboot.h> 30 #include <linux/debugfs.h> 31 #include <linux/highmem.h> 32 #include <linux/file.h> 33 #include <linux/sysdev.h> 34 #include <linux/cpu.h> 35 #include <linux/sched.h> 36 #include <linux/cpumask.h> 37 #include <linux/smp.h> 38 #include <linux/anon_inodes.h> 39 #include <linux/profile.h> 40 #include <linux/kvm_para.h> 41 #include <linux/pagemap.h> 42 #include <linux/mman.h> 43 #include <linux/swap.h> 44 45 #include <asm/processor.h> 46 #include <asm/io.h> 47 #include <asm/uaccess.h> 48 #include <asm/pgtable.h> 49 50 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 51 #include "coalesced_mmio.h" 52 #endif 53 54 MODULE_AUTHOR("Qumranet"); 55 MODULE_LICENSE("GPL"); 56 57 DEFINE_SPINLOCK(kvm_lock); 58 LIST_HEAD(vm_list); 59 60 static cpumask_t cpus_hardware_enabled; 61 62 struct kmem_cache *kvm_vcpu_cache; 63 EXPORT_SYMBOL_GPL(kvm_vcpu_cache); 64 65 static __read_mostly struct preempt_ops kvm_preempt_ops; 66 67 struct dentry *kvm_debugfs_dir; 68 69 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 70 unsigned long arg); 71 72 bool kvm_rebooting; 73 74 static inline int valid_vcpu(int n) 75 { 76 return likely(n >= 0 && n < KVM_MAX_VCPUS); 77 } 78 79 /* 80 * Switches to specified vcpu, until a matching vcpu_put() 81 */ 82 void vcpu_load(struct kvm_vcpu *vcpu) 83 { 84 int cpu; 85 86 mutex_lock(&vcpu->mutex); 87 cpu = get_cpu(); 88 preempt_notifier_register(&vcpu->preempt_notifier); 89 kvm_arch_vcpu_load(vcpu, cpu); 90 put_cpu(); 91 } 92 93 void vcpu_put(struct kvm_vcpu *vcpu) 94 { 95 preempt_disable(); 96 kvm_arch_vcpu_put(vcpu); 97 preempt_notifier_unregister(&vcpu->preempt_notifier); 98 preempt_enable(); 99 mutex_unlock(&vcpu->mutex); 100 } 101 102 static void ack_flush(void *_completed) 103 { 104 } 105 106 void kvm_flush_remote_tlbs(struct kvm *kvm) 107 { 108 int i, cpu, me; 109 cpumask_t cpus; 110 struct kvm_vcpu *vcpu; 111 112 me = get_cpu(); 113 cpus_clear(cpus); 114 for (i = 0; i < KVM_MAX_VCPUS; ++i) { 115 vcpu = kvm->vcpus[i]; 116 if (!vcpu) 117 continue; 118 if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) 119 continue; 120 cpu = vcpu->cpu; 121 if (cpu != -1 && cpu != me) 122 cpu_set(cpu, cpus); 123 } 124 if (cpus_empty(cpus)) 125 goto out; 126 ++kvm->stat.remote_tlb_flush; 127 smp_call_function_mask(cpus, ack_flush, NULL, 1); 128 out: 129 put_cpu(); 130 } 131 132 void kvm_reload_remote_mmus(struct kvm *kvm) 133 { 134 int i, cpu, me; 135 cpumask_t cpus; 136 struct kvm_vcpu *vcpu; 137 138 me = get_cpu(); 139 cpus_clear(cpus); 140 for (i = 0; i < KVM_MAX_VCPUS; ++i) { 141 vcpu = kvm->vcpus[i]; 142 if (!vcpu) 143 continue; 144 if (test_and_set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests)) 145 continue; 146 cpu = vcpu->cpu; 147 if (cpu != -1 && cpu != me) 148 cpu_set(cpu, cpus); 149 } 150 if (cpus_empty(cpus)) 151 goto out; 152 smp_call_function_mask(cpus, ack_flush, NULL, 1); 153 out: 154 put_cpu(); 155 } 156 157 158 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) 159 { 160 struct page *page; 161 int r; 162 163 mutex_init(&vcpu->mutex); 164 vcpu->cpu = -1; 165 vcpu->kvm = kvm; 166 vcpu->vcpu_id = id; 167 init_waitqueue_head(&vcpu->wq); 168 169 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 170 if (!page) { 171 r = -ENOMEM; 172 goto fail; 173 } 174 vcpu->run = page_address(page); 175 176 r = kvm_arch_vcpu_init(vcpu); 177 if (r < 0) 178 goto fail_free_run; 179 return 0; 180 181 fail_free_run: 182 free_page((unsigned long)vcpu->run); 183 fail: 184 return r; 185 } 186 EXPORT_SYMBOL_GPL(kvm_vcpu_init); 187 188 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 189 { 190 kvm_arch_vcpu_uninit(vcpu); 191 free_page((unsigned long)vcpu->run); 192 } 193 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 194 195 static struct kvm *kvm_create_vm(void) 196 { 197 struct kvm *kvm = kvm_arch_create_vm(); 198 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 199 struct page *page; 200 #endif 201 202 if (IS_ERR(kvm)) 203 goto out; 204 205 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 206 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 207 if (!page) { 208 kfree(kvm); 209 return ERR_PTR(-ENOMEM); 210 } 211 kvm->coalesced_mmio_ring = 212 (struct kvm_coalesced_mmio_ring *)page_address(page); 213 #endif 214 215 kvm->mm = current->mm; 216 atomic_inc(&kvm->mm->mm_count); 217 spin_lock_init(&kvm->mmu_lock); 218 kvm_io_bus_init(&kvm->pio_bus); 219 mutex_init(&kvm->lock); 220 kvm_io_bus_init(&kvm->mmio_bus); 221 init_rwsem(&kvm->slots_lock); 222 atomic_set(&kvm->users_count, 1); 223 spin_lock(&kvm_lock); 224 list_add(&kvm->vm_list, &vm_list); 225 spin_unlock(&kvm_lock); 226 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 227 kvm_coalesced_mmio_init(kvm); 228 #endif 229 out: 230 return kvm; 231 } 232 233 /* 234 * Free any memory in @free but not in @dont. 235 */ 236 static void kvm_free_physmem_slot(struct kvm_memory_slot *free, 237 struct kvm_memory_slot *dont) 238 { 239 if (!dont || free->rmap != dont->rmap) 240 vfree(free->rmap); 241 242 if (!dont || free->dirty_bitmap != dont->dirty_bitmap) 243 vfree(free->dirty_bitmap); 244 245 if (!dont || free->lpage_info != dont->lpage_info) 246 vfree(free->lpage_info); 247 248 free->npages = 0; 249 free->dirty_bitmap = NULL; 250 free->rmap = NULL; 251 free->lpage_info = NULL; 252 } 253 254 void kvm_free_physmem(struct kvm *kvm) 255 { 256 int i; 257 258 for (i = 0; i < kvm->nmemslots; ++i) 259 kvm_free_physmem_slot(&kvm->memslots[i], NULL); 260 } 261 262 static void kvm_destroy_vm(struct kvm *kvm) 263 { 264 struct mm_struct *mm = kvm->mm; 265 266 spin_lock(&kvm_lock); 267 list_del(&kvm->vm_list); 268 spin_unlock(&kvm_lock); 269 kvm_io_bus_destroy(&kvm->pio_bus); 270 kvm_io_bus_destroy(&kvm->mmio_bus); 271 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 272 if (kvm->coalesced_mmio_ring != NULL) 273 free_page((unsigned long)kvm->coalesced_mmio_ring); 274 #endif 275 kvm_arch_destroy_vm(kvm); 276 mmdrop(mm); 277 } 278 279 void kvm_get_kvm(struct kvm *kvm) 280 { 281 atomic_inc(&kvm->users_count); 282 } 283 EXPORT_SYMBOL_GPL(kvm_get_kvm); 284 285 void kvm_put_kvm(struct kvm *kvm) 286 { 287 if (atomic_dec_and_test(&kvm->users_count)) 288 kvm_destroy_vm(kvm); 289 } 290 EXPORT_SYMBOL_GPL(kvm_put_kvm); 291 292 293 static int kvm_vm_release(struct inode *inode, struct file *filp) 294 { 295 struct kvm *kvm = filp->private_data; 296 297 kvm_put_kvm(kvm); 298 return 0; 299 } 300 301 /* 302 * Allocate some memory and give it an address in the guest physical address 303 * space. 304 * 305 * Discontiguous memory is allowed, mostly for framebuffers. 306 * 307 * Must be called holding mmap_sem for write. 308 */ 309 int __kvm_set_memory_region(struct kvm *kvm, 310 struct kvm_userspace_memory_region *mem, 311 int user_alloc) 312 { 313 int r; 314 gfn_t base_gfn; 315 unsigned long npages; 316 unsigned long i; 317 struct kvm_memory_slot *memslot; 318 struct kvm_memory_slot old, new; 319 320 r = -EINVAL; 321 /* General sanity checks */ 322 if (mem->memory_size & (PAGE_SIZE - 1)) 323 goto out; 324 if (mem->guest_phys_addr & (PAGE_SIZE - 1)) 325 goto out; 326 if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS) 327 goto out; 328 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) 329 goto out; 330 331 memslot = &kvm->memslots[mem->slot]; 332 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; 333 npages = mem->memory_size >> PAGE_SHIFT; 334 335 if (!npages) 336 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; 337 338 new = old = *memslot; 339 340 new.base_gfn = base_gfn; 341 new.npages = npages; 342 new.flags = mem->flags; 343 344 /* Disallow changing a memory slot's size. */ 345 r = -EINVAL; 346 if (npages && old.npages && npages != old.npages) 347 goto out_free; 348 349 /* Check for overlaps */ 350 r = -EEXIST; 351 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { 352 struct kvm_memory_slot *s = &kvm->memslots[i]; 353 354 if (s == memslot) 355 continue; 356 if (!((base_gfn + npages <= s->base_gfn) || 357 (base_gfn >= s->base_gfn + s->npages))) 358 goto out_free; 359 } 360 361 /* Free page dirty bitmap if unneeded */ 362 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) 363 new.dirty_bitmap = NULL; 364 365 r = -ENOMEM; 366 367 /* Allocate if a slot is being created */ 368 #ifndef CONFIG_S390 369 if (npages && !new.rmap) { 370 new.rmap = vmalloc(npages * sizeof(struct page *)); 371 372 if (!new.rmap) 373 goto out_free; 374 375 memset(new.rmap, 0, npages * sizeof(*new.rmap)); 376 377 new.user_alloc = user_alloc; 378 new.userspace_addr = mem->userspace_addr; 379 } 380 if (npages && !new.lpage_info) { 381 int largepages = npages / KVM_PAGES_PER_HPAGE; 382 if (npages % KVM_PAGES_PER_HPAGE) 383 largepages++; 384 if (base_gfn % KVM_PAGES_PER_HPAGE) 385 largepages++; 386 387 new.lpage_info = vmalloc(largepages * sizeof(*new.lpage_info)); 388 389 if (!new.lpage_info) 390 goto out_free; 391 392 memset(new.lpage_info, 0, largepages * sizeof(*new.lpage_info)); 393 394 if (base_gfn % KVM_PAGES_PER_HPAGE) 395 new.lpage_info[0].write_count = 1; 396 if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE) 397 new.lpage_info[largepages-1].write_count = 1; 398 } 399 400 /* Allocate page dirty bitmap if needed */ 401 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { 402 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8; 403 404 new.dirty_bitmap = vmalloc(dirty_bytes); 405 if (!new.dirty_bitmap) 406 goto out_free; 407 memset(new.dirty_bitmap, 0, dirty_bytes); 408 } 409 #endif /* not defined CONFIG_S390 */ 410 411 if (mem->slot >= kvm->nmemslots) 412 kvm->nmemslots = mem->slot + 1; 413 414 if (!npages) 415 kvm_arch_flush_shadow(kvm); 416 417 *memslot = new; 418 419 r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc); 420 if (r) { 421 *memslot = old; 422 goto out_free; 423 } 424 425 kvm_free_physmem_slot(&old, &new); 426 return 0; 427 428 out_free: 429 kvm_free_physmem_slot(&new, &old); 430 out: 431 return r; 432 433 } 434 EXPORT_SYMBOL_GPL(__kvm_set_memory_region); 435 436 int kvm_set_memory_region(struct kvm *kvm, 437 struct kvm_userspace_memory_region *mem, 438 int user_alloc) 439 { 440 int r; 441 442 down_write(&kvm->slots_lock); 443 r = __kvm_set_memory_region(kvm, mem, user_alloc); 444 up_write(&kvm->slots_lock); 445 return r; 446 } 447 EXPORT_SYMBOL_GPL(kvm_set_memory_region); 448 449 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 450 struct 451 kvm_userspace_memory_region *mem, 452 int user_alloc) 453 { 454 if (mem->slot >= KVM_MEMORY_SLOTS) 455 return -EINVAL; 456 return kvm_set_memory_region(kvm, mem, user_alloc); 457 } 458 459 int kvm_get_dirty_log(struct kvm *kvm, 460 struct kvm_dirty_log *log, int *is_dirty) 461 { 462 struct kvm_memory_slot *memslot; 463 int r, i; 464 int n; 465 unsigned long any = 0; 466 467 r = -EINVAL; 468 if (log->slot >= KVM_MEMORY_SLOTS) 469 goto out; 470 471 memslot = &kvm->memslots[log->slot]; 472 r = -ENOENT; 473 if (!memslot->dirty_bitmap) 474 goto out; 475 476 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8; 477 478 for (i = 0; !any && i < n/sizeof(long); ++i) 479 any = memslot->dirty_bitmap[i]; 480 481 r = -EFAULT; 482 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 483 goto out; 484 485 if (any) 486 *is_dirty = 1; 487 488 r = 0; 489 out: 490 return r; 491 } 492 493 int is_error_page(struct page *page) 494 { 495 return page == bad_page; 496 } 497 EXPORT_SYMBOL_GPL(is_error_page); 498 499 int is_error_pfn(pfn_t pfn) 500 { 501 return pfn == bad_pfn; 502 } 503 EXPORT_SYMBOL_GPL(is_error_pfn); 504 505 static inline unsigned long bad_hva(void) 506 { 507 return PAGE_OFFSET; 508 } 509 510 int kvm_is_error_hva(unsigned long addr) 511 { 512 return addr == bad_hva(); 513 } 514 EXPORT_SYMBOL_GPL(kvm_is_error_hva); 515 516 static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 517 { 518 int i; 519 520 for (i = 0; i < kvm->nmemslots; ++i) { 521 struct kvm_memory_slot *memslot = &kvm->memslots[i]; 522 523 if (gfn >= memslot->base_gfn 524 && gfn < memslot->base_gfn + memslot->npages) 525 return memslot; 526 } 527 return NULL; 528 } 529 530 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) 531 { 532 gfn = unalias_gfn(kvm, gfn); 533 return __gfn_to_memslot(kvm, gfn); 534 } 535 536 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) 537 { 538 int i; 539 540 gfn = unalias_gfn(kvm, gfn); 541 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { 542 struct kvm_memory_slot *memslot = &kvm->memslots[i]; 543 544 if (gfn >= memslot->base_gfn 545 && gfn < memslot->base_gfn + memslot->npages) 546 return 1; 547 } 548 return 0; 549 } 550 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); 551 552 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) 553 { 554 struct kvm_memory_slot *slot; 555 556 gfn = unalias_gfn(kvm, gfn); 557 slot = __gfn_to_memslot(kvm, gfn); 558 if (!slot) 559 return bad_hva(); 560 return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE); 561 } 562 EXPORT_SYMBOL_GPL(gfn_to_hva); 563 564 /* 565 * Requires current->mm->mmap_sem to be held 566 */ 567 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) 568 { 569 struct page *page[1]; 570 unsigned long addr; 571 int npages; 572 pfn_t pfn; 573 574 might_sleep(); 575 576 addr = gfn_to_hva(kvm, gfn); 577 if (kvm_is_error_hva(addr)) { 578 get_page(bad_page); 579 return page_to_pfn(bad_page); 580 } 581 582 npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page, 583 NULL); 584 585 if (unlikely(npages != 1)) { 586 struct vm_area_struct *vma; 587 588 vma = find_vma(current->mm, addr); 589 if (vma == NULL || addr < vma->vm_start || 590 !(vma->vm_flags & VM_PFNMAP)) { 591 get_page(bad_page); 592 return page_to_pfn(bad_page); 593 } 594 595 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 596 BUG_ON(pfn_valid(pfn)); 597 } else 598 pfn = page_to_pfn(page[0]); 599 600 return pfn; 601 } 602 603 EXPORT_SYMBOL_GPL(gfn_to_pfn); 604 605 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 606 { 607 pfn_t pfn; 608 609 pfn = gfn_to_pfn(kvm, gfn); 610 if (pfn_valid(pfn)) 611 return pfn_to_page(pfn); 612 613 WARN_ON(!pfn_valid(pfn)); 614 615 get_page(bad_page); 616 return bad_page; 617 } 618 619 EXPORT_SYMBOL_GPL(gfn_to_page); 620 621 void kvm_release_page_clean(struct page *page) 622 { 623 kvm_release_pfn_clean(page_to_pfn(page)); 624 } 625 EXPORT_SYMBOL_GPL(kvm_release_page_clean); 626 627 void kvm_release_pfn_clean(pfn_t pfn) 628 { 629 if (pfn_valid(pfn)) 630 put_page(pfn_to_page(pfn)); 631 } 632 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean); 633 634 void kvm_release_page_dirty(struct page *page) 635 { 636 kvm_release_pfn_dirty(page_to_pfn(page)); 637 } 638 EXPORT_SYMBOL_GPL(kvm_release_page_dirty); 639 640 void kvm_release_pfn_dirty(pfn_t pfn) 641 { 642 kvm_set_pfn_dirty(pfn); 643 kvm_release_pfn_clean(pfn); 644 } 645 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty); 646 647 void kvm_set_page_dirty(struct page *page) 648 { 649 kvm_set_pfn_dirty(page_to_pfn(page)); 650 } 651 EXPORT_SYMBOL_GPL(kvm_set_page_dirty); 652 653 void kvm_set_pfn_dirty(pfn_t pfn) 654 { 655 if (pfn_valid(pfn)) { 656 struct page *page = pfn_to_page(pfn); 657 if (!PageReserved(page)) 658 SetPageDirty(page); 659 } 660 } 661 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); 662 663 void kvm_set_pfn_accessed(pfn_t pfn) 664 { 665 if (pfn_valid(pfn)) 666 mark_page_accessed(pfn_to_page(pfn)); 667 } 668 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed); 669 670 void kvm_get_pfn(pfn_t pfn) 671 { 672 if (pfn_valid(pfn)) 673 get_page(pfn_to_page(pfn)); 674 } 675 EXPORT_SYMBOL_GPL(kvm_get_pfn); 676 677 static int next_segment(unsigned long len, int offset) 678 { 679 if (len > PAGE_SIZE - offset) 680 return PAGE_SIZE - offset; 681 else 682 return len; 683 } 684 685 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 686 int len) 687 { 688 int r; 689 unsigned long addr; 690 691 addr = gfn_to_hva(kvm, gfn); 692 if (kvm_is_error_hva(addr)) 693 return -EFAULT; 694 r = copy_from_user(data, (void __user *)addr + offset, len); 695 if (r) 696 return -EFAULT; 697 return 0; 698 } 699 EXPORT_SYMBOL_GPL(kvm_read_guest_page); 700 701 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) 702 { 703 gfn_t gfn = gpa >> PAGE_SHIFT; 704 int seg; 705 int offset = offset_in_page(gpa); 706 int ret; 707 708 while ((seg = next_segment(len, offset)) != 0) { 709 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); 710 if (ret < 0) 711 return ret; 712 offset = 0; 713 len -= seg; 714 data += seg; 715 ++gfn; 716 } 717 return 0; 718 } 719 EXPORT_SYMBOL_GPL(kvm_read_guest); 720 721 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 722 unsigned long len) 723 { 724 int r; 725 unsigned long addr; 726 gfn_t gfn = gpa >> PAGE_SHIFT; 727 int offset = offset_in_page(gpa); 728 729 addr = gfn_to_hva(kvm, gfn); 730 if (kvm_is_error_hva(addr)) 731 return -EFAULT; 732 pagefault_disable(); 733 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len); 734 pagefault_enable(); 735 if (r) 736 return -EFAULT; 737 return 0; 738 } 739 EXPORT_SYMBOL(kvm_read_guest_atomic); 740 741 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 742 int offset, int len) 743 { 744 int r; 745 unsigned long addr; 746 747 addr = gfn_to_hva(kvm, gfn); 748 if (kvm_is_error_hva(addr)) 749 return -EFAULT; 750 r = copy_to_user((void __user *)addr + offset, data, len); 751 if (r) 752 return -EFAULT; 753 mark_page_dirty(kvm, gfn); 754 return 0; 755 } 756 EXPORT_SYMBOL_GPL(kvm_write_guest_page); 757 758 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 759 unsigned long len) 760 { 761 gfn_t gfn = gpa >> PAGE_SHIFT; 762 int seg; 763 int offset = offset_in_page(gpa); 764 int ret; 765 766 while ((seg = next_segment(len, offset)) != 0) { 767 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); 768 if (ret < 0) 769 return ret; 770 offset = 0; 771 len -= seg; 772 data += seg; 773 ++gfn; 774 } 775 return 0; 776 } 777 778 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) 779 { 780 return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len); 781 } 782 EXPORT_SYMBOL_GPL(kvm_clear_guest_page); 783 784 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) 785 { 786 gfn_t gfn = gpa >> PAGE_SHIFT; 787 int seg; 788 int offset = offset_in_page(gpa); 789 int ret; 790 791 while ((seg = next_segment(len, offset)) != 0) { 792 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); 793 if (ret < 0) 794 return ret; 795 offset = 0; 796 len -= seg; 797 ++gfn; 798 } 799 return 0; 800 } 801 EXPORT_SYMBOL_GPL(kvm_clear_guest); 802 803 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) 804 { 805 struct kvm_memory_slot *memslot; 806 807 gfn = unalias_gfn(kvm, gfn); 808 memslot = __gfn_to_memslot(kvm, gfn); 809 if (memslot && memslot->dirty_bitmap) { 810 unsigned long rel_gfn = gfn - memslot->base_gfn; 811 812 /* avoid RMW */ 813 if (!test_bit(rel_gfn, memslot->dirty_bitmap)) 814 set_bit(rel_gfn, memslot->dirty_bitmap); 815 } 816 } 817 818 /* 819 * The vCPU has executed a HLT instruction with in-kernel mode enabled. 820 */ 821 void kvm_vcpu_block(struct kvm_vcpu *vcpu) 822 { 823 DEFINE_WAIT(wait); 824 825 for (;;) { 826 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); 827 828 if (kvm_cpu_has_interrupt(vcpu)) 829 break; 830 if (kvm_cpu_has_pending_timer(vcpu)) 831 break; 832 if (kvm_arch_vcpu_runnable(vcpu)) 833 break; 834 if (signal_pending(current)) 835 break; 836 837 vcpu_put(vcpu); 838 schedule(); 839 vcpu_load(vcpu); 840 } 841 842 finish_wait(&vcpu->wq, &wait); 843 } 844 845 void kvm_resched(struct kvm_vcpu *vcpu) 846 { 847 if (!need_resched()) 848 return; 849 cond_resched(); 850 } 851 EXPORT_SYMBOL_GPL(kvm_resched); 852 853 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 854 { 855 struct kvm_vcpu *vcpu = vma->vm_file->private_data; 856 struct page *page; 857 858 if (vmf->pgoff == 0) 859 page = virt_to_page(vcpu->run); 860 #ifdef CONFIG_X86 861 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) 862 page = virt_to_page(vcpu->arch.pio_data); 863 #endif 864 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 865 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) 866 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); 867 #endif 868 else 869 return VM_FAULT_SIGBUS; 870 get_page(page); 871 vmf->page = page; 872 return 0; 873 } 874 875 static struct vm_operations_struct kvm_vcpu_vm_ops = { 876 .fault = kvm_vcpu_fault, 877 }; 878 879 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) 880 { 881 vma->vm_ops = &kvm_vcpu_vm_ops; 882 return 0; 883 } 884 885 static int kvm_vcpu_release(struct inode *inode, struct file *filp) 886 { 887 struct kvm_vcpu *vcpu = filp->private_data; 888 889 kvm_put_kvm(vcpu->kvm); 890 return 0; 891 } 892 893 static const struct file_operations kvm_vcpu_fops = { 894 .release = kvm_vcpu_release, 895 .unlocked_ioctl = kvm_vcpu_ioctl, 896 .compat_ioctl = kvm_vcpu_ioctl, 897 .mmap = kvm_vcpu_mmap, 898 }; 899 900 /* 901 * Allocates an inode for the vcpu. 902 */ 903 static int create_vcpu_fd(struct kvm_vcpu *vcpu) 904 { 905 int fd = anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, 0); 906 if (fd < 0) 907 kvm_put_kvm(vcpu->kvm); 908 return fd; 909 } 910 911 /* 912 * Creates some virtual cpus. Good luck creating more than one. 913 */ 914 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n) 915 { 916 int r; 917 struct kvm_vcpu *vcpu; 918 919 if (!valid_vcpu(n)) 920 return -EINVAL; 921 922 vcpu = kvm_arch_vcpu_create(kvm, n); 923 if (IS_ERR(vcpu)) 924 return PTR_ERR(vcpu); 925 926 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 927 928 r = kvm_arch_vcpu_setup(vcpu); 929 if (r) 930 goto vcpu_destroy; 931 932 mutex_lock(&kvm->lock); 933 if (kvm->vcpus[n]) { 934 r = -EEXIST; 935 mutex_unlock(&kvm->lock); 936 goto vcpu_destroy; 937 } 938 kvm->vcpus[n] = vcpu; 939 mutex_unlock(&kvm->lock); 940 941 /* Now it's all set up, let userspace reach it */ 942 kvm_get_kvm(kvm); 943 r = create_vcpu_fd(vcpu); 944 if (r < 0) 945 goto unlink; 946 return r; 947 948 unlink: 949 mutex_lock(&kvm->lock); 950 kvm->vcpus[n] = NULL; 951 mutex_unlock(&kvm->lock); 952 vcpu_destroy: 953 kvm_arch_vcpu_destroy(vcpu); 954 return r; 955 } 956 957 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) 958 { 959 if (sigset) { 960 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); 961 vcpu->sigset_active = 1; 962 vcpu->sigset = *sigset; 963 } else 964 vcpu->sigset_active = 0; 965 return 0; 966 } 967 968 static long kvm_vcpu_ioctl(struct file *filp, 969 unsigned int ioctl, unsigned long arg) 970 { 971 struct kvm_vcpu *vcpu = filp->private_data; 972 void __user *argp = (void __user *)arg; 973 int r; 974 975 if (vcpu->kvm->mm != current->mm) 976 return -EIO; 977 switch (ioctl) { 978 case KVM_RUN: 979 r = -EINVAL; 980 if (arg) 981 goto out; 982 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); 983 break; 984 case KVM_GET_REGS: { 985 struct kvm_regs *kvm_regs; 986 987 r = -ENOMEM; 988 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 989 if (!kvm_regs) 990 goto out; 991 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); 992 if (r) 993 goto out_free1; 994 r = -EFAULT; 995 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs))) 996 goto out_free1; 997 r = 0; 998 out_free1: 999 kfree(kvm_regs); 1000 break; 1001 } 1002 case KVM_SET_REGS: { 1003 struct kvm_regs *kvm_regs; 1004 1005 r = -ENOMEM; 1006 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); 1007 if (!kvm_regs) 1008 goto out; 1009 r = -EFAULT; 1010 if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs))) 1011 goto out_free2; 1012 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs); 1013 if (r) 1014 goto out_free2; 1015 r = 0; 1016 out_free2: 1017 kfree(kvm_regs); 1018 break; 1019 } 1020 case KVM_GET_SREGS: { 1021 struct kvm_sregs kvm_sregs; 1022 1023 memset(&kvm_sregs, 0, sizeof kvm_sregs); 1024 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs); 1025 if (r) 1026 goto out; 1027 r = -EFAULT; 1028 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs)) 1029 goto out; 1030 r = 0; 1031 break; 1032 } 1033 case KVM_SET_SREGS: { 1034 struct kvm_sregs kvm_sregs; 1035 1036 r = -EFAULT; 1037 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs)) 1038 goto out; 1039 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs); 1040 if (r) 1041 goto out; 1042 r = 0; 1043 break; 1044 } 1045 case KVM_GET_MP_STATE: { 1046 struct kvm_mp_state mp_state; 1047 1048 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state); 1049 if (r) 1050 goto out; 1051 r = -EFAULT; 1052 if (copy_to_user(argp, &mp_state, sizeof mp_state)) 1053 goto out; 1054 r = 0; 1055 break; 1056 } 1057 case KVM_SET_MP_STATE: { 1058 struct kvm_mp_state mp_state; 1059 1060 r = -EFAULT; 1061 if (copy_from_user(&mp_state, argp, sizeof mp_state)) 1062 goto out; 1063 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state); 1064 if (r) 1065 goto out; 1066 r = 0; 1067 break; 1068 } 1069 case KVM_TRANSLATE: { 1070 struct kvm_translation tr; 1071 1072 r = -EFAULT; 1073 if (copy_from_user(&tr, argp, sizeof tr)) 1074 goto out; 1075 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); 1076 if (r) 1077 goto out; 1078 r = -EFAULT; 1079 if (copy_to_user(argp, &tr, sizeof tr)) 1080 goto out; 1081 r = 0; 1082 break; 1083 } 1084 case KVM_DEBUG_GUEST: { 1085 struct kvm_debug_guest dbg; 1086 1087 r = -EFAULT; 1088 if (copy_from_user(&dbg, argp, sizeof dbg)) 1089 goto out; 1090 r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg); 1091 if (r) 1092 goto out; 1093 r = 0; 1094 break; 1095 } 1096 case KVM_SET_SIGNAL_MASK: { 1097 struct kvm_signal_mask __user *sigmask_arg = argp; 1098 struct kvm_signal_mask kvm_sigmask; 1099 sigset_t sigset, *p; 1100 1101 p = NULL; 1102 if (argp) { 1103 r = -EFAULT; 1104 if (copy_from_user(&kvm_sigmask, argp, 1105 sizeof kvm_sigmask)) 1106 goto out; 1107 r = -EINVAL; 1108 if (kvm_sigmask.len != sizeof sigset) 1109 goto out; 1110 r = -EFAULT; 1111 if (copy_from_user(&sigset, sigmask_arg->sigset, 1112 sizeof sigset)) 1113 goto out; 1114 p = &sigset; 1115 } 1116 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); 1117 break; 1118 } 1119 case KVM_GET_FPU: { 1120 struct kvm_fpu fpu; 1121 1122 memset(&fpu, 0, sizeof fpu); 1123 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu); 1124 if (r) 1125 goto out; 1126 r = -EFAULT; 1127 if (copy_to_user(argp, &fpu, sizeof fpu)) 1128 goto out; 1129 r = 0; 1130 break; 1131 } 1132 case KVM_SET_FPU: { 1133 struct kvm_fpu fpu; 1134 1135 r = -EFAULT; 1136 if (copy_from_user(&fpu, argp, sizeof fpu)) 1137 goto out; 1138 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu); 1139 if (r) 1140 goto out; 1141 r = 0; 1142 break; 1143 } 1144 default: 1145 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); 1146 } 1147 out: 1148 return r; 1149 } 1150 1151 static long kvm_vm_ioctl(struct file *filp, 1152 unsigned int ioctl, unsigned long arg) 1153 { 1154 struct kvm *kvm = filp->private_data; 1155 void __user *argp = (void __user *)arg; 1156 int r; 1157 1158 if (kvm->mm != current->mm) 1159 return -EIO; 1160 switch (ioctl) { 1161 case KVM_CREATE_VCPU: 1162 r = kvm_vm_ioctl_create_vcpu(kvm, arg); 1163 if (r < 0) 1164 goto out; 1165 break; 1166 case KVM_SET_USER_MEMORY_REGION: { 1167 struct kvm_userspace_memory_region kvm_userspace_mem; 1168 1169 r = -EFAULT; 1170 if (copy_from_user(&kvm_userspace_mem, argp, 1171 sizeof kvm_userspace_mem)) 1172 goto out; 1173 1174 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1); 1175 if (r) 1176 goto out; 1177 break; 1178 } 1179 case KVM_GET_DIRTY_LOG: { 1180 struct kvm_dirty_log log; 1181 1182 r = -EFAULT; 1183 if (copy_from_user(&log, argp, sizeof log)) 1184 goto out; 1185 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); 1186 if (r) 1187 goto out; 1188 break; 1189 } 1190 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 1191 case KVM_REGISTER_COALESCED_MMIO: { 1192 struct kvm_coalesced_mmio_zone zone; 1193 r = -EFAULT; 1194 if (copy_from_user(&zone, argp, sizeof zone)) 1195 goto out; 1196 r = -ENXIO; 1197 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); 1198 if (r) 1199 goto out; 1200 r = 0; 1201 break; 1202 } 1203 case KVM_UNREGISTER_COALESCED_MMIO: { 1204 struct kvm_coalesced_mmio_zone zone; 1205 r = -EFAULT; 1206 if (copy_from_user(&zone, argp, sizeof zone)) 1207 goto out; 1208 r = -ENXIO; 1209 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); 1210 if (r) 1211 goto out; 1212 r = 0; 1213 break; 1214 } 1215 #endif 1216 default: 1217 r = kvm_arch_vm_ioctl(filp, ioctl, arg); 1218 } 1219 out: 1220 return r; 1221 } 1222 1223 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1224 { 1225 struct kvm *kvm = vma->vm_file->private_data; 1226 struct page *page; 1227 1228 if (!kvm_is_visible_gfn(kvm, vmf->pgoff)) 1229 return VM_FAULT_SIGBUS; 1230 page = gfn_to_page(kvm, vmf->pgoff); 1231 if (is_error_page(page)) { 1232 kvm_release_page_clean(page); 1233 return VM_FAULT_SIGBUS; 1234 } 1235 vmf->page = page; 1236 return 0; 1237 } 1238 1239 static struct vm_operations_struct kvm_vm_vm_ops = { 1240 .fault = kvm_vm_fault, 1241 }; 1242 1243 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma) 1244 { 1245 vma->vm_ops = &kvm_vm_vm_ops; 1246 return 0; 1247 } 1248 1249 static const struct file_operations kvm_vm_fops = { 1250 .release = kvm_vm_release, 1251 .unlocked_ioctl = kvm_vm_ioctl, 1252 .compat_ioctl = kvm_vm_ioctl, 1253 .mmap = kvm_vm_mmap, 1254 }; 1255 1256 static int kvm_dev_ioctl_create_vm(void) 1257 { 1258 int fd; 1259 struct kvm *kvm; 1260 1261 kvm = kvm_create_vm(); 1262 if (IS_ERR(kvm)) 1263 return PTR_ERR(kvm); 1264 fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, 0); 1265 if (fd < 0) 1266 kvm_put_kvm(kvm); 1267 1268 return fd; 1269 } 1270 1271 static long kvm_dev_ioctl(struct file *filp, 1272 unsigned int ioctl, unsigned long arg) 1273 { 1274 long r = -EINVAL; 1275 1276 switch (ioctl) { 1277 case KVM_GET_API_VERSION: 1278 r = -EINVAL; 1279 if (arg) 1280 goto out; 1281 r = KVM_API_VERSION; 1282 break; 1283 case KVM_CREATE_VM: 1284 r = -EINVAL; 1285 if (arg) 1286 goto out; 1287 r = kvm_dev_ioctl_create_vm(); 1288 break; 1289 case KVM_CHECK_EXTENSION: 1290 r = kvm_dev_ioctl_check_extension(arg); 1291 break; 1292 case KVM_GET_VCPU_MMAP_SIZE: 1293 r = -EINVAL; 1294 if (arg) 1295 goto out; 1296 r = PAGE_SIZE; /* struct kvm_run */ 1297 #ifdef CONFIG_X86 1298 r += PAGE_SIZE; /* pio data page */ 1299 #endif 1300 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 1301 r += PAGE_SIZE; /* coalesced mmio ring page */ 1302 #endif 1303 break; 1304 case KVM_TRACE_ENABLE: 1305 case KVM_TRACE_PAUSE: 1306 case KVM_TRACE_DISABLE: 1307 r = kvm_trace_ioctl(ioctl, arg); 1308 break; 1309 default: 1310 return kvm_arch_dev_ioctl(filp, ioctl, arg); 1311 } 1312 out: 1313 return r; 1314 } 1315 1316 static struct file_operations kvm_chardev_ops = { 1317 .unlocked_ioctl = kvm_dev_ioctl, 1318 .compat_ioctl = kvm_dev_ioctl, 1319 }; 1320 1321 static struct miscdevice kvm_dev = { 1322 KVM_MINOR, 1323 "kvm", 1324 &kvm_chardev_ops, 1325 }; 1326 1327 static void hardware_enable(void *junk) 1328 { 1329 int cpu = raw_smp_processor_id(); 1330 1331 if (cpu_isset(cpu, cpus_hardware_enabled)) 1332 return; 1333 cpu_set(cpu, cpus_hardware_enabled); 1334 kvm_arch_hardware_enable(NULL); 1335 } 1336 1337 static void hardware_disable(void *junk) 1338 { 1339 int cpu = raw_smp_processor_id(); 1340 1341 if (!cpu_isset(cpu, cpus_hardware_enabled)) 1342 return; 1343 cpu_clear(cpu, cpus_hardware_enabled); 1344 kvm_arch_hardware_disable(NULL); 1345 } 1346 1347 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 1348 void *v) 1349 { 1350 int cpu = (long)v; 1351 1352 val &= ~CPU_TASKS_FROZEN; 1353 switch (val) { 1354 case CPU_DYING: 1355 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", 1356 cpu); 1357 hardware_disable(NULL); 1358 break; 1359 case CPU_UP_CANCELED: 1360 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", 1361 cpu); 1362 smp_call_function_single(cpu, hardware_disable, NULL, 1); 1363 break; 1364 case CPU_ONLINE: 1365 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", 1366 cpu); 1367 smp_call_function_single(cpu, hardware_enable, NULL, 1); 1368 break; 1369 } 1370 return NOTIFY_OK; 1371 } 1372 1373 1374 asmlinkage void kvm_handle_fault_on_reboot(void) 1375 { 1376 if (kvm_rebooting) 1377 /* spin while reset goes on */ 1378 while (true) 1379 ; 1380 /* Fault while not rebooting. We want the trace. */ 1381 BUG(); 1382 } 1383 EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot); 1384 1385 static int kvm_reboot(struct notifier_block *notifier, unsigned long val, 1386 void *v) 1387 { 1388 if (val == SYS_RESTART) { 1389 /* 1390 * Some (well, at least mine) BIOSes hang on reboot if 1391 * in vmx root mode. 1392 */ 1393 printk(KERN_INFO "kvm: exiting hardware virtualization\n"); 1394 kvm_rebooting = true; 1395 on_each_cpu(hardware_disable, NULL, 1); 1396 } 1397 return NOTIFY_OK; 1398 } 1399 1400 static struct notifier_block kvm_reboot_notifier = { 1401 .notifier_call = kvm_reboot, 1402 .priority = 0, 1403 }; 1404 1405 void kvm_io_bus_init(struct kvm_io_bus *bus) 1406 { 1407 memset(bus, 0, sizeof(*bus)); 1408 } 1409 1410 void kvm_io_bus_destroy(struct kvm_io_bus *bus) 1411 { 1412 int i; 1413 1414 for (i = 0; i < bus->dev_count; i++) { 1415 struct kvm_io_device *pos = bus->devs[i]; 1416 1417 kvm_iodevice_destructor(pos); 1418 } 1419 } 1420 1421 struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, 1422 gpa_t addr, int len, int is_write) 1423 { 1424 int i; 1425 1426 for (i = 0; i < bus->dev_count; i++) { 1427 struct kvm_io_device *pos = bus->devs[i]; 1428 1429 if (pos->in_range(pos, addr, len, is_write)) 1430 return pos; 1431 } 1432 1433 return NULL; 1434 } 1435 1436 void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev) 1437 { 1438 BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1)); 1439 1440 bus->devs[bus->dev_count++] = dev; 1441 } 1442 1443 static struct notifier_block kvm_cpu_notifier = { 1444 .notifier_call = kvm_cpu_hotplug, 1445 .priority = 20, /* must be > scheduler priority */ 1446 }; 1447 1448 static int vm_stat_get(void *_offset, u64 *val) 1449 { 1450 unsigned offset = (long)_offset; 1451 struct kvm *kvm; 1452 1453 *val = 0; 1454 spin_lock(&kvm_lock); 1455 list_for_each_entry(kvm, &vm_list, vm_list) 1456 *val += *(u32 *)((void *)kvm + offset); 1457 spin_unlock(&kvm_lock); 1458 return 0; 1459 } 1460 1461 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); 1462 1463 static int vcpu_stat_get(void *_offset, u64 *val) 1464 { 1465 unsigned offset = (long)_offset; 1466 struct kvm *kvm; 1467 struct kvm_vcpu *vcpu; 1468 int i; 1469 1470 *val = 0; 1471 spin_lock(&kvm_lock); 1472 list_for_each_entry(kvm, &vm_list, vm_list) 1473 for (i = 0; i < KVM_MAX_VCPUS; ++i) { 1474 vcpu = kvm->vcpus[i]; 1475 if (vcpu) 1476 *val += *(u32 *)((void *)vcpu + offset); 1477 } 1478 spin_unlock(&kvm_lock); 1479 return 0; 1480 } 1481 1482 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); 1483 1484 static struct file_operations *stat_fops[] = { 1485 [KVM_STAT_VCPU] = &vcpu_stat_fops, 1486 [KVM_STAT_VM] = &vm_stat_fops, 1487 }; 1488 1489 static void kvm_init_debug(void) 1490 { 1491 struct kvm_stats_debugfs_item *p; 1492 1493 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); 1494 for (p = debugfs_entries; p->name; ++p) 1495 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir, 1496 (void *)(long)p->offset, 1497 stat_fops[p->kind]); 1498 } 1499 1500 static void kvm_exit_debug(void) 1501 { 1502 struct kvm_stats_debugfs_item *p; 1503 1504 for (p = debugfs_entries; p->name; ++p) 1505 debugfs_remove(p->dentry); 1506 debugfs_remove(kvm_debugfs_dir); 1507 } 1508 1509 static int kvm_suspend(struct sys_device *dev, pm_message_t state) 1510 { 1511 hardware_disable(NULL); 1512 return 0; 1513 } 1514 1515 static int kvm_resume(struct sys_device *dev) 1516 { 1517 hardware_enable(NULL); 1518 return 0; 1519 } 1520 1521 static struct sysdev_class kvm_sysdev_class = { 1522 .name = "kvm", 1523 .suspend = kvm_suspend, 1524 .resume = kvm_resume, 1525 }; 1526 1527 static struct sys_device kvm_sysdev = { 1528 .id = 0, 1529 .cls = &kvm_sysdev_class, 1530 }; 1531 1532 struct page *bad_page; 1533 pfn_t bad_pfn; 1534 1535 static inline 1536 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) 1537 { 1538 return container_of(pn, struct kvm_vcpu, preempt_notifier); 1539 } 1540 1541 static void kvm_sched_in(struct preempt_notifier *pn, int cpu) 1542 { 1543 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 1544 1545 kvm_arch_vcpu_load(vcpu, cpu); 1546 } 1547 1548 static void kvm_sched_out(struct preempt_notifier *pn, 1549 struct task_struct *next) 1550 { 1551 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 1552 1553 kvm_arch_vcpu_put(vcpu); 1554 } 1555 1556 int kvm_init(void *opaque, unsigned int vcpu_size, 1557 struct module *module) 1558 { 1559 int r; 1560 int cpu; 1561 1562 kvm_init_debug(); 1563 1564 r = kvm_arch_init(opaque); 1565 if (r) 1566 goto out_fail; 1567 1568 bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1569 1570 if (bad_page == NULL) { 1571 r = -ENOMEM; 1572 goto out; 1573 } 1574 1575 bad_pfn = page_to_pfn(bad_page); 1576 1577 r = kvm_arch_hardware_setup(); 1578 if (r < 0) 1579 goto out_free_0; 1580 1581 for_each_online_cpu(cpu) { 1582 smp_call_function_single(cpu, 1583 kvm_arch_check_processor_compat, 1584 &r, 1); 1585 if (r < 0) 1586 goto out_free_1; 1587 } 1588 1589 on_each_cpu(hardware_enable, NULL, 1); 1590 r = register_cpu_notifier(&kvm_cpu_notifier); 1591 if (r) 1592 goto out_free_2; 1593 register_reboot_notifier(&kvm_reboot_notifier); 1594 1595 r = sysdev_class_register(&kvm_sysdev_class); 1596 if (r) 1597 goto out_free_3; 1598 1599 r = sysdev_register(&kvm_sysdev); 1600 if (r) 1601 goto out_free_4; 1602 1603 /* A kmem cache lets us meet the alignment requirements of fx_save. */ 1604 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, 1605 __alignof__(struct kvm_vcpu), 1606 0, NULL); 1607 if (!kvm_vcpu_cache) { 1608 r = -ENOMEM; 1609 goto out_free_5; 1610 } 1611 1612 kvm_chardev_ops.owner = module; 1613 1614 r = misc_register(&kvm_dev); 1615 if (r) { 1616 printk(KERN_ERR "kvm: misc device register failed\n"); 1617 goto out_free; 1618 } 1619 1620 kvm_preempt_ops.sched_in = kvm_sched_in; 1621 kvm_preempt_ops.sched_out = kvm_sched_out; 1622 1623 return 0; 1624 1625 out_free: 1626 kmem_cache_destroy(kvm_vcpu_cache); 1627 out_free_5: 1628 sysdev_unregister(&kvm_sysdev); 1629 out_free_4: 1630 sysdev_class_unregister(&kvm_sysdev_class); 1631 out_free_3: 1632 unregister_reboot_notifier(&kvm_reboot_notifier); 1633 unregister_cpu_notifier(&kvm_cpu_notifier); 1634 out_free_2: 1635 on_each_cpu(hardware_disable, NULL, 1); 1636 out_free_1: 1637 kvm_arch_hardware_unsetup(); 1638 out_free_0: 1639 __free_page(bad_page); 1640 out: 1641 kvm_arch_exit(); 1642 kvm_exit_debug(); 1643 out_fail: 1644 return r; 1645 } 1646 EXPORT_SYMBOL_GPL(kvm_init); 1647 1648 void kvm_exit(void) 1649 { 1650 kvm_trace_cleanup(); 1651 misc_deregister(&kvm_dev); 1652 kmem_cache_destroy(kvm_vcpu_cache); 1653 sysdev_unregister(&kvm_sysdev); 1654 sysdev_class_unregister(&kvm_sysdev_class); 1655 unregister_reboot_notifier(&kvm_reboot_notifier); 1656 unregister_cpu_notifier(&kvm_cpu_notifier); 1657 on_each_cpu(hardware_disable, NULL, 1); 1658 kvm_arch_hardware_unsetup(); 1659 kvm_arch_exit(); 1660 kvm_exit_debug(); 1661 __free_page(bad_page); 1662 } 1663 EXPORT_SYMBOL_GPL(kvm_exit); 1664