1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Kernel-based Virtual Machine (KVM) Hypervisor
4 *
5 * Copyright (C) 2006 Qumranet, Inc.
6 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
7 *
8 * Authors:
9 * Avi Kivity <avi@qumranet.com>
10 * Yaniv Kamay <yaniv@qumranet.com>
11 */
12
13 #include <kvm/iodev.h>
14
15 #include <linux/kvm_host.h>
16 #include <linux/kvm.h>
17 #include <linux/module.h>
18 #include <linux/errno.h>
19 #include <linux/percpu.h>
20 #include <linux/mm.h>
21 #include <linux/miscdevice.h>
22 #include <linux/vmalloc.h>
23 #include <linux/reboot.h>
24 #include <linux/debugfs.h>
25 #include <linux/highmem.h>
26 #include <linux/file.h>
27 #include <linux/syscore_ops.h>
28 #include <linux/cpu.h>
29 #include <linux/sched/signal.h>
30 #include <linux/sched/mm.h>
31 #include <linux/sched/stat.h>
32 #include <linux/cpumask.h>
33 #include <linux/smp.h>
34 #include <linux/anon_inodes.h>
35 #include <linux/profile.h>
36 #include <linux/kvm_para.h>
37 #include <linux/pagemap.h>
38 #include <linux/mman.h>
39 #include <linux/swap.h>
40 #include <linux/bitops.h>
41 #include <linux/spinlock.h>
42 #include <linux/compat.h>
43 #include <linux/srcu.h>
44 #include <linux/hugetlb.h>
45 #include <linux/slab.h>
46 #include <linux/sort.h>
47 #include <linux/bsearch.h>
48 #include <linux/io.h>
49 #include <linux/lockdep.h>
50 #include <linux/kthread.h>
51 #include <linux/suspend.h>
52
53 #include <asm/processor.h>
54 #include <asm/ioctl.h>
55 #include <linux/uaccess.h>
56
57 #include "coalesced_mmio.h"
58 #include "async_pf.h"
59 #include "kvm_mm.h"
60 #include "vfio.h"
61
62 #include <trace/events/ipi.h>
63
64 #define CREATE_TRACE_POINTS
65 #include <trace/events/kvm.h>
66
67 #include <linux/kvm_dirty_ring.h>
68
69
70 /* Worst case buffer size needed for holding an integer. */
71 #define ITOA_MAX_LEN 12
72
73 MODULE_AUTHOR("Qumranet");
74 MODULE_DESCRIPTION("Kernel-based Virtual Machine (KVM) Hypervisor");
75 MODULE_LICENSE("GPL");
76
77 /* Architectures should define their poll value according to the halt latency */
78 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
79 module_param(halt_poll_ns, uint, 0644);
80 EXPORT_SYMBOL_GPL(halt_poll_ns);
81
82 /* Default doubles per-vcpu halt_poll_ns. */
83 unsigned int halt_poll_ns_grow = 2;
84 module_param(halt_poll_ns_grow, uint, 0644);
85 EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
86
87 /* The start value to grow halt_poll_ns from */
88 unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
89 module_param(halt_poll_ns_grow_start, uint, 0644);
90 EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);
91
92 /* Default halves per-vcpu halt_poll_ns. */
93 unsigned int halt_poll_ns_shrink = 2;
94 module_param(halt_poll_ns_shrink, uint, 0644);
95 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
96
97 /*
98 * Allow direct access (from KVM or the CPU) without MMU notifier protection
99 * to unpinned pages.
100 */
101 static bool allow_unsafe_mappings;
102 module_param(allow_unsafe_mappings, bool, 0444);
103
104 /*
105 * Ordering of locks:
106 *
107 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
108 */
109
110 DEFINE_MUTEX(kvm_lock);
111 LIST_HEAD(vm_list);
112
113 static struct kmem_cache *kvm_vcpu_cache;
114
115 static __read_mostly struct preempt_ops kvm_preempt_ops;
116 static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu);
117
118 static struct dentry *kvm_debugfs_dir;
119
120 static const struct file_operations stat_fops_per_vm;
121
122 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
123 unsigned long arg);
124 #ifdef CONFIG_KVM_COMPAT
125 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
126 unsigned long arg);
127 #define KVM_COMPAT(c) .compat_ioctl = (c)
128 #else
129 /*
130 * For architectures that don't implement a compat infrastructure,
131 * adopt a double line of defense:
132 * - Prevent a compat task from opening /dev/kvm
133 * - If the open has been done by a 64bit task, and the KVM fd
134 * passed to a compat task, let the ioctls fail.
135 */
kvm_no_compat_ioctl(struct file * file,unsigned int ioctl,unsigned long arg)136 static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
137 unsigned long arg) { return -EINVAL; }
138
kvm_no_compat_open(struct inode * inode,struct file * file)139 static int kvm_no_compat_open(struct inode *inode, struct file *file)
140 {
141 return is_compat_task() ? -ENODEV : 0;
142 }
143 #define KVM_COMPAT(c) .compat_ioctl = kvm_no_compat_ioctl, \
144 .open = kvm_no_compat_open
145 #endif
146 static int kvm_enable_virtualization(void);
147 static void kvm_disable_virtualization(void);
148
149 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
150
151 #define KVM_EVENT_CREATE_VM 0
152 #define KVM_EVENT_DESTROY_VM 1
153 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
154 static unsigned long long kvm_createvm_count;
155 static unsigned long long kvm_active_vms;
156
157 static DEFINE_PER_CPU(cpumask_var_t, cpu_kick_mask);
158
kvm_arch_guest_memory_reclaimed(struct kvm * kvm)159 __weak void kvm_arch_guest_memory_reclaimed(struct kvm *kvm)
160 {
161 }
162
163 /*
164 * Switches to specified vcpu, until a matching vcpu_put()
165 */
vcpu_load(struct kvm_vcpu * vcpu)166 void vcpu_load(struct kvm_vcpu *vcpu)
167 {
168 int cpu = get_cpu();
169
170 __this_cpu_write(kvm_running_vcpu, vcpu);
171 preempt_notifier_register(&vcpu->preempt_notifier);
172 kvm_arch_vcpu_load(vcpu, cpu);
173 put_cpu();
174 }
175 EXPORT_SYMBOL_GPL(vcpu_load);
176
vcpu_put(struct kvm_vcpu * vcpu)177 void vcpu_put(struct kvm_vcpu *vcpu)
178 {
179 preempt_disable();
180 kvm_arch_vcpu_put(vcpu);
181 preempt_notifier_unregister(&vcpu->preempt_notifier);
182 __this_cpu_write(kvm_running_vcpu, NULL);
183 preempt_enable();
184 }
185 EXPORT_SYMBOL_GPL(vcpu_put);
186
187 /* TODO: merge with kvm_arch_vcpu_should_kick */
kvm_request_needs_ipi(struct kvm_vcpu * vcpu,unsigned req)188 static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
189 {
190 int mode = kvm_vcpu_exiting_guest_mode(vcpu);
191
192 /*
193 * We need to wait for the VCPU to reenable interrupts and get out of
194 * READING_SHADOW_PAGE_TABLES mode.
195 */
196 if (req & KVM_REQUEST_WAIT)
197 return mode != OUTSIDE_GUEST_MODE;
198
199 /*
200 * Need to kick a running VCPU, but otherwise there is nothing to do.
201 */
202 return mode == IN_GUEST_MODE;
203 }
204
ack_kick(void * _completed)205 static void ack_kick(void *_completed)
206 {
207 }
208
kvm_kick_many_cpus(struct cpumask * cpus,bool wait)209 static inline bool kvm_kick_many_cpus(struct cpumask *cpus, bool wait)
210 {
211 if (cpumask_empty(cpus))
212 return false;
213
214 smp_call_function_many(cpus, ack_kick, NULL, wait);
215 return true;
216 }
217
kvm_make_vcpu_request(struct kvm_vcpu * vcpu,unsigned int req,struct cpumask * tmp,int current_cpu)218 static void kvm_make_vcpu_request(struct kvm_vcpu *vcpu, unsigned int req,
219 struct cpumask *tmp, int current_cpu)
220 {
221 int cpu;
222
223 if (likely(!(req & KVM_REQUEST_NO_ACTION)))
224 __kvm_make_request(req, vcpu);
225
226 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
227 return;
228
229 /*
230 * Note, the vCPU could get migrated to a different pCPU at any point
231 * after kvm_request_needs_ipi(), which could result in sending an IPI
232 * to the previous pCPU. But, that's OK because the purpose of the IPI
233 * is to ensure the vCPU returns to OUTSIDE_GUEST_MODE, which is
234 * satisfied if the vCPU migrates. Entering READING_SHADOW_PAGE_TABLES
235 * after this point is also OK, as the requirement is only that KVM wait
236 * for vCPUs that were reading SPTEs _before_ any changes were
237 * finalized. See kvm_vcpu_kick() for more details on handling requests.
238 */
239 if (kvm_request_needs_ipi(vcpu, req)) {
240 cpu = READ_ONCE(vcpu->cpu);
241 if (cpu != -1 && cpu != current_cpu)
242 __cpumask_set_cpu(cpu, tmp);
243 }
244 }
245
kvm_make_vcpus_request_mask(struct kvm * kvm,unsigned int req,unsigned long * vcpu_bitmap)246 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
247 unsigned long *vcpu_bitmap)
248 {
249 struct kvm_vcpu *vcpu;
250 struct cpumask *cpus;
251 int i, me;
252 bool called;
253
254 me = get_cpu();
255
256 cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask);
257 cpumask_clear(cpus);
258
259 for_each_set_bit(i, vcpu_bitmap, KVM_MAX_VCPUS) {
260 vcpu = kvm_get_vcpu(kvm, i);
261 if (!vcpu)
262 continue;
263 kvm_make_vcpu_request(vcpu, req, cpus, me);
264 }
265
266 called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
267 put_cpu();
268
269 return called;
270 }
271
kvm_make_all_cpus_request(struct kvm * kvm,unsigned int req)272 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
273 {
274 struct kvm_vcpu *vcpu;
275 struct cpumask *cpus;
276 unsigned long i;
277 bool called;
278 int me;
279
280 me = get_cpu();
281
282 cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask);
283 cpumask_clear(cpus);
284
285 kvm_for_each_vcpu(i, vcpu, kvm)
286 kvm_make_vcpu_request(vcpu, req, cpus, me);
287
288 called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
289 put_cpu();
290
291 return called;
292 }
293 EXPORT_SYMBOL_GPL(kvm_make_all_cpus_request);
294
kvm_flush_remote_tlbs(struct kvm * kvm)295 void kvm_flush_remote_tlbs(struct kvm *kvm)
296 {
297 ++kvm->stat.generic.remote_tlb_flush_requests;
298
299 /*
300 * We want to publish modifications to the page tables before reading
301 * mode. Pairs with a memory barrier in arch-specific code.
302 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
303 * and smp_mb in walk_shadow_page_lockless_begin/end.
304 * - powerpc: smp_mb in kvmppc_prepare_to_enter.
305 *
306 * There is already an smp_mb__after_atomic() before
307 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
308 * barrier here.
309 */
310 if (!kvm_arch_flush_remote_tlbs(kvm)
311 || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
312 ++kvm->stat.generic.remote_tlb_flush;
313 }
314 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
315
kvm_flush_remote_tlbs_range(struct kvm * kvm,gfn_t gfn,u64 nr_pages)316 void kvm_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages)
317 {
318 if (!kvm_arch_flush_remote_tlbs_range(kvm, gfn, nr_pages))
319 return;
320
321 /*
322 * Fall back to a flushing entire TLBs if the architecture range-based
323 * TLB invalidation is unsupported or can't be performed for whatever
324 * reason.
325 */
326 kvm_flush_remote_tlbs(kvm);
327 }
328
kvm_flush_remote_tlbs_memslot(struct kvm * kvm,const struct kvm_memory_slot * memslot)329 void kvm_flush_remote_tlbs_memslot(struct kvm *kvm,
330 const struct kvm_memory_slot *memslot)
331 {
332 /*
333 * All current use cases for flushing the TLBs for a specific memslot
334 * are related to dirty logging, and many do the TLB flush out of
335 * mmu_lock. The interaction between the various operations on memslot
336 * must be serialized by slots_locks to ensure the TLB flush from one
337 * operation is observed by any other operation on the same memslot.
338 */
339 lockdep_assert_held(&kvm->slots_lock);
340 kvm_flush_remote_tlbs_range(kvm, memslot->base_gfn, memslot->npages);
341 }
342
kvm_flush_shadow_all(struct kvm * kvm)343 static void kvm_flush_shadow_all(struct kvm *kvm)
344 {
345 kvm_arch_flush_shadow_all(kvm);
346 kvm_arch_guest_memory_reclaimed(kvm);
347 }
348
349 #ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE
mmu_memory_cache_alloc_obj(struct kvm_mmu_memory_cache * mc,gfp_t gfp_flags)350 static inline void *mmu_memory_cache_alloc_obj(struct kvm_mmu_memory_cache *mc,
351 gfp_t gfp_flags)
352 {
353 void *page;
354
355 gfp_flags |= mc->gfp_zero;
356
357 if (mc->kmem_cache)
358 return kmem_cache_alloc(mc->kmem_cache, gfp_flags);
359
360 page = (void *)__get_free_page(gfp_flags);
361 if (page && mc->init_value)
362 memset64(page, mc->init_value, PAGE_SIZE / sizeof(u64));
363 return page;
364 }
365
__kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache * mc,int capacity,int min)366 int __kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int capacity, int min)
367 {
368 gfp_t gfp = mc->gfp_custom ? mc->gfp_custom : GFP_KERNEL_ACCOUNT;
369 void *obj;
370
371 if (mc->nobjs >= min)
372 return 0;
373
374 if (unlikely(!mc->objects)) {
375 if (WARN_ON_ONCE(!capacity))
376 return -EIO;
377
378 /*
379 * Custom init values can be used only for page allocations,
380 * and obviously conflict with __GFP_ZERO.
381 */
382 if (WARN_ON_ONCE(mc->init_value && (mc->kmem_cache || mc->gfp_zero)))
383 return -EIO;
384
385 mc->objects = kvmalloc_array(capacity, sizeof(void *), gfp);
386 if (!mc->objects)
387 return -ENOMEM;
388
389 mc->capacity = capacity;
390 }
391
392 /* It is illegal to request a different capacity across topups. */
393 if (WARN_ON_ONCE(mc->capacity != capacity))
394 return -EIO;
395
396 while (mc->nobjs < mc->capacity) {
397 obj = mmu_memory_cache_alloc_obj(mc, gfp);
398 if (!obj)
399 return mc->nobjs >= min ? 0 : -ENOMEM;
400 mc->objects[mc->nobjs++] = obj;
401 }
402 return 0;
403 }
404
kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache * mc,int min)405 int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min)
406 {
407 return __kvm_mmu_topup_memory_cache(mc, KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE, min);
408 }
409
kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache * mc)410 int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc)
411 {
412 return mc->nobjs;
413 }
414
kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache * mc)415 void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
416 {
417 while (mc->nobjs) {
418 if (mc->kmem_cache)
419 kmem_cache_free(mc->kmem_cache, mc->objects[--mc->nobjs]);
420 else
421 free_page((unsigned long)mc->objects[--mc->nobjs]);
422 }
423
424 kvfree(mc->objects);
425
426 mc->objects = NULL;
427 mc->capacity = 0;
428 }
429
kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache * mc)430 void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
431 {
432 void *p;
433
434 if (WARN_ON(!mc->nobjs))
435 p = mmu_memory_cache_alloc_obj(mc, GFP_ATOMIC | __GFP_ACCOUNT);
436 else
437 p = mc->objects[--mc->nobjs];
438 BUG_ON(!p);
439 return p;
440 }
441 #endif
442
kvm_vcpu_init(struct kvm_vcpu * vcpu,struct kvm * kvm,unsigned id)443 static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
444 {
445 mutex_init(&vcpu->mutex);
446 vcpu->cpu = -1;
447 vcpu->kvm = kvm;
448 vcpu->vcpu_id = id;
449 vcpu->pid = NULL;
450 rwlock_init(&vcpu->pid_lock);
451 #ifndef __KVM_HAVE_ARCH_WQP
452 rcuwait_init(&vcpu->wait);
453 #endif
454 kvm_async_pf_vcpu_init(vcpu);
455
456 kvm_vcpu_set_in_spin_loop(vcpu, false);
457 kvm_vcpu_set_dy_eligible(vcpu, false);
458 vcpu->preempted = false;
459 vcpu->ready = false;
460 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
461 vcpu->last_used_slot = NULL;
462
463 /* Fill the stats id string for the vcpu */
464 snprintf(vcpu->stats_id, sizeof(vcpu->stats_id), "kvm-%d/vcpu-%d",
465 task_pid_nr(current), id);
466 }
467
kvm_vcpu_destroy(struct kvm_vcpu * vcpu)468 static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
469 {
470 kvm_arch_vcpu_destroy(vcpu);
471 kvm_dirty_ring_free(&vcpu->dirty_ring);
472
473 /*
474 * No need for rcu_read_lock as VCPU_RUN is the only place that changes
475 * the vcpu->pid pointer, and at destruction time all file descriptors
476 * are already gone.
477 */
478 put_pid(vcpu->pid);
479
480 free_page((unsigned long)vcpu->run);
481 kmem_cache_free(kvm_vcpu_cache, vcpu);
482 }
483
kvm_destroy_vcpus(struct kvm * kvm)484 void kvm_destroy_vcpus(struct kvm *kvm)
485 {
486 unsigned long i;
487 struct kvm_vcpu *vcpu;
488
489 kvm_for_each_vcpu(i, vcpu, kvm) {
490 kvm_vcpu_destroy(vcpu);
491 xa_erase(&kvm->vcpu_array, i);
492
493 /*
494 * Assert that the vCPU isn't visible in any way, to ensure KVM
495 * doesn't trigger a use-after-free if destroying vCPUs results
496 * in VM-wide request, e.g. to flush remote TLBs when tearing
497 * down MMUs, or to mark the VM dead if a KVM_BUG_ON() fires.
498 */
499 WARN_ON_ONCE(xa_load(&kvm->vcpu_array, i) || kvm_get_vcpu(kvm, i));
500 }
501
502 atomic_set(&kvm->online_vcpus, 0);
503 }
504 EXPORT_SYMBOL_GPL(kvm_destroy_vcpus);
505
506 #ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
mmu_notifier_to_kvm(struct mmu_notifier * mn)507 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
508 {
509 return container_of(mn, struct kvm, mmu_notifier);
510 }
511
512 typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
513
514 typedef void (*on_lock_fn_t)(struct kvm *kvm);
515
516 struct kvm_mmu_notifier_range {
517 /*
518 * 64-bit addresses, as KVM notifiers can operate on host virtual
519 * addresses (unsigned long) and guest physical addresses (64-bit).
520 */
521 u64 start;
522 u64 end;
523 union kvm_mmu_notifier_arg arg;
524 gfn_handler_t handler;
525 on_lock_fn_t on_lock;
526 bool flush_on_ret;
527 bool may_block;
528 bool lockless;
529 };
530
531 /*
532 * The inner-most helper returns a tuple containing the return value from the
533 * arch- and action-specific handler, plus a flag indicating whether or not at
534 * least one memslot was found, i.e. if the handler found guest memory.
535 *
536 * Note, most notifiers are averse to booleans, so even though KVM tracks the
537 * return from arch code as a bool, outer helpers will cast it to an int. :-(
538 */
539 typedef struct kvm_mmu_notifier_return {
540 bool ret;
541 bool found_memslot;
542 } kvm_mn_ret_t;
543
544 /*
545 * Use a dedicated stub instead of NULL to indicate that there is no callback
546 * function/handler. The compiler technically can't guarantee that a real
547 * function will have a non-zero address, and so it will generate code to
548 * check for !NULL, whereas comparing against a stub will be elided at compile
549 * time (unless the compiler is getting long in the tooth, e.g. gcc 4.9).
550 */
kvm_null_fn(void)551 static void kvm_null_fn(void)
552 {
553
554 }
555 #define IS_KVM_NULL_FN(fn) ((fn) == (void *)kvm_null_fn)
556
557 /* Iterate over each memslot intersecting [start, last] (inclusive) range */
558 #define kvm_for_each_memslot_in_hva_range(node, slots, start, last) \
559 for (node = interval_tree_iter_first(&slots->hva_tree, start, last); \
560 node; \
561 node = interval_tree_iter_next(node, start, last)) \
562
kvm_handle_hva_range(struct kvm * kvm,const struct kvm_mmu_notifier_range * range)563 static __always_inline kvm_mn_ret_t kvm_handle_hva_range(struct kvm *kvm,
564 const struct kvm_mmu_notifier_range *range)
565 {
566 struct kvm_mmu_notifier_return r = {
567 .ret = false,
568 .found_memslot = false,
569 };
570 struct kvm_gfn_range gfn_range;
571 struct kvm_memory_slot *slot;
572 struct kvm_memslots *slots;
573 int i, idx;
574
575 if (WARN_ON_ONCE(range->end <= range->start))
576 return r;
577
578 /* A null handler is allowed if and only if on_lock() is provided. */
579 if (WARN_ON_ONCE(IS_KVM_NULL_FN(range->on_lock) &&
580 IS_KVM_NULL_FN(range->handler)))
581 return r;
582
583 /* on_lock will never be called for lockless walks */
584 if (WARN_ON_ONCE(range->lockless && !IS_KVM_NULL_FN(range->on_lock)))
585 return r;
586
587 idx = srcu_read_lock(&kvm->srcu);
588
589 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
590 struct interval_tree_node *node;
591
592 slots = __kvm_memslots(kvm, i);
593 kvm_for_each_memslot_in_hva_range(node, slots,
594 range->start, range->end - 1) {
595 unsigned long hva_start, hva_end;
596
597 slot = container_of(node, struct kvm_memory_slot, hva_node[slots->node_idx]);
598 hva_start = max_t(unsigned long, range->start, slot->userspace_addr);
599 hva_end = min_t(unsigned long, range->end,
600 slot->userspace_addr + (slot->npages << PAGE_SHIFT));
601
602 /*
603 * To optimize for the likely case where the address
604 * range is covered by zero or one memslots, don't
605 * bother making these conditional (to avoid writes on
606 * the second or later invocation of the handler).
607 */
608 gfn_range.arg = range->arg;
609 gfn_range.may_block = range->may_block;
610 /*
611 * HVA-based notifications aren't relevant to private
612 * mappings as they don't have a userspace mapping.
613 */
614 gfn_range.attr_filter = KVM_FILTER_SHARED;
615
616 /*
617 * {gfn(page) | page intersects with [hva_start, hva_end)} =
618 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
619 */
620 gfn_range.start = hva_to_gfn_memslot(hva_start, slot);
621 gfn_range.end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, slot);
622 gfn_range.slot = slot;
623 gfn_range.lockless = range->lockless;
624
625 if (!r.found_memslot) {
626 r.found_memslot = true;
627 if (!range->lockless) {
628 KVM_MMU_LOCK(kvm);
629 if (!IS_KVM_NULL_FN(range->on_lock))
630 range->on_lock(kvm);
631
632 if (IS_KVM_NULL_FN(range->handler))
633 goto mmu_unlock;
634 }
635 }
636 r.ret |= range->handler(kvm, &gfn_range);
637 }
638 }
639
640 if (range->flush_on_ret && r.ret)
641 kvm_flush_remote_tlbs(kvm);
642
643 mmu_unlock:
644 if (r.found_memslot && !range->lockless)
645 KVM_MMU_UNLOCK(kvm);
646
647 srcu_read_unlock(&kvm->srcu, idx);
648
649 return r;
650 }
651
kvm_age_hva_range(struct mmu_notifier * mn,unsigned long start,unsigned long end,gfn_handler_t handler,bool flush_on_ret)652 static __always_inline int kvm_age_hva_range(struct mmu_notifier *mn,
653 unsigned long start,
654 unsigned long end,
655 gfn_handler_t handler,
656 bool flush_on_ret)
657 {
658 struct kvm *kvm = mmu_notifier_to_kvm(mn);
659 const struct kvm_mmu_notifier_range range = {
660 .start = start,
661 .end = end,
662 .handler = handler,
663 .on_lock = (void *)kvm_null_fn,
664 .flush_on_ret = flush_on_ret,
665 .may_block = false,
666 .lockless = IS_ENABLED(CONFIG_KVM_MMU_LOCKLESS_AGING),
667 };
668
669 return kvm_handle_hva_range(kvm, &range).ret;
670 }
671
kvm_age_hva_range_no_flush(struct mmu_notifier * mn,unsigned long start,unsigned long end,gfn_handler_t handler)672 static __always_inline int kvm_age_hva_range_no_flush(struct mmu_notifier *mn,
673 unsigned long start,
674 unsigned long end,
675 gfn_handler_t handler)
676 {
677 return kvm_age_hva_range(mn, start, end, handler, false);
678 }
679
kvm_mmu_invalidate_begin(struct kvm * kvm)680 void kvm_mmu_invalidate_begin(struct kvm *kvm)
681 {
682 lockdep_assert_held_write(&kvm->mmu_lock);
683 /*
684 * The count increase must become visible at unlock time as no
685 * spte can be established without taking the mmu_lock and
686 * count is also read inside the mmu_lock critical section.
687 */
688 kvm->mmu_invalidate_in_progress++;
689
690 if (likely(kvm->mmu_invalidate_in_progress == 1)) {
691 kvm->mmu_invalidate_range_start = INVALID_GPA;
692 kvm->mmu_invalidate_range_end = INVALID_GPA;
693 }
694 }
695
kvm_mmu_invalidate_range_add(struct kvm * kvm,gfn_t start,gfn_t end)696 void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end)
697 {
698 lockdep_assert_held_write(&kvm->mmu_lock);
699
700 WARN_ON_ONCE(!kvm->mmu_invalidate_in_progress);
701
702 if (likely(kvm->mmu_invalidate_range_start == INVALID_GPA)) {
703 kvm->mmu_invalidate_range_start = start;
704 kvm->mmu_invalidate_range_end = end;
705 } else {
706 /*
707 * Fully tracking multiple concurrent ranges has diminishing
708 * returns. Keep things simple and just find the minimal range
709 * which includes the current and new ranges. As there won't be
710 * enough information to subtract a range after its invalidate
711 * completes, any ranges invalidated concurrently will
712 * accumulate and persist until all outstanding invalidates
713 * complete.
714 */
715 kvm->mmu_invalidate_range_start =
716 min(kvm->mmu_invalidate_range_start, start);
717 kvm->mmu_invalidate_range_end =
718 max(kvm->mmu_invalidate_range_end, end);
719 }
720 }
721
kvm_mmu_unmap_gfn_range(struct kvm * kvm,struct kvm_gfn_range * range)722 bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
723 {
724 kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
725 return kvm_unmap_gfn_range(kvm, range);
726 }
727
kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier * mn,const struct mmu_notifier_range * range)728 static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
729 const struct mmu_notifier_range *range)
730 {
731 struct kvm *kvm = mmu_notifier_to_kvm(mn);
732 const struct kvm_mmu_notifier_range hva_range = {
733 .start = range->start,
734 .end = range->end,
735 .handler = kvm_mmu_unmap_gfn_range,
736 .on_lock = kvm_mmu_invalidate_begin,
737 .flush_on_ret = true,
738 .may_block = mmu_notifier_range_blockable(range),
739 };
740
741 trace_kvm_unmap_hva_range(range->start, range->end);
742
743 /*
744 * Prevent memslot modification between range_start() and range_end()
745 * so that conditionally locking provides the same result in both
746 * functions. Without that guarantee, the mmu_invalidate_in_progress
747 * adjustments will be imbalanced.
748 *
749 * Pairs with the decrement in range_end().
750 */
751 spin_lock(&kvm->mn_invalidate_lock);
752 kvm->mn_active_invalidate_count++;
753 spin_unlock(&kvm->mn_invalidate_lock);
754
755 /*
756 * Invalidate pfn caches _before_ invalidating the secondary MMUs, i.e.
757 * before acquiring mmu_lock, to avoid holding mmu_lock while acquiring
758 * each cache's lock. There are relatively few caches in existence at
759 * any given time, and the caches themselves can check for hva overlap,
760 * i.e. don't need to rely on memslot overlap checks for performance.
761 * Because this runs without holding mmu_lock, the pfn caches must use
762 * mn_active_invalidate_count (see above) instead of
763 * mmu_invalidate_in_progress.
764 */
765 gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end);
766
767 /*
768 * If one or more memslots were found and thus zapped, notify arch code
769 * that guest memory has been reclaimed. This needs to be done *after*
770 * dropping mmu_lock, as x86's reclaim path is slooooow.
771 */
772 if (kvm_handle_hva_range(kvm, &hva_range).found_memslot)
773 kvm_arch_guest_memory_reclaimed(kvm);
774
775 return 0;
776 }
777
kvm_mmu_invalidate_end(struct kvm * kvm)778 void kvm_mmu_invalidate_end(struct kvm *kvm)
779 {
780 lockdep_assert_held_write(&kvm->mmu_lock);
781
782 /*
783 * This sequence increase will notify the kvm page fault that
784 * the page that is going to be mapped in the spte could have
785 * been freed.
786 */
787 kvm->mmu_invalidate_seq++;
788 smp_wmb();
789 /*
790 * The above sequence increase must be visible before the
791 * below count decrease, which is ensured by the smp_wmb above
792 * in conjunction with the smp_rmb in mmu_invalidate_retry().
793 */
794 kvm->mmu_invalidate_in_progress--;
795 KVM_BUG_ON(kvm->mmu_invalidate_in_progress < 0, kvm);
796
797 /*
798 * Assert that at least one range was added between start() and end().
799 * Not adding a range isn't fatal, but it is a KVM bug.
800 */
801 WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA);
802 }
803
kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier * mn,const struct mmu_notifier_range * range)804 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
805 const struct mmu_notifier_range *range)
806 {
807 struct kvm *kvm = mmu_notifier_to_kvm(mn);
808 const struct kvm_mmu_notifier_range hva_range = {
809 .start = range->start,
810 .end = range->end,
811 .handler = (void *)kvm_null_fn,
812 .on_lock = kvm_mmu_invalidate_end,
813 .flush_on_ret = false,
814 .may_block = mmu_notifier_range_blockable(range),
815 };
816 bool wake;
817
818 kvm_handle_hva_range(kvm, &hva_range);
819
820 /* Pairs with the increment in range_start(). */
821 spin_lock(&kvm->mn_invalidate_lock);
822 if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count))
823 --kvm->mn_active_invalidate_count;
824 wake = !kvm->mn_active_invalidate_count;
825 spin_unlock(&kvm->mn_invalidate_lock);
826
827 /*
828 * There can only be one waiter, since the wait happens under
829 * slots_lock.
830 */
831 if (wake)
832 rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);
833 }
834
kvm_mmu_notifier_clear_flush_young(struct mmu_notifier * mn,struct mm_struct * mm,unsigned long start,unsigned long end)835 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
836 struct mm_struct *mm,
837 unsigned long start,
838 unsigned long end)
839 {
840 trace_kvm_age_hva(start, end);
841
842 return kvm_age_hva_range(mn, start, end, kvm_age_gfn,
843 !IS_ENABLED(CONFIG_KVM_ELIDE_TLB_FLUSH_IF_YOUNG));
844 }
845
kvm_mmu_notifier_clear_young(struct mmu_notifier * mn,struct mm_struct * mm,unsigned long start,unsigned long end)846 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
847 struct mm_struct *mm,
848 unsigned long start,
849 unsigned long end)
850 {
851 trace_kvm_age_hva(start, end);
852
853 /*
854 * Even though we do not flush TLB, this will still adversely
855 * affect performance on pre-Haswell Intel EPT, where there is
856 * no EPT Access Bit to clear so that we have to tear down EPT
857 * tables instead. If we find this unacceptable, we can always
858 * add a parameter to kvm_age_hva so that it effectively doesn't
859 * do anything on clear_young.
860 *
861 * Also note that currently we never issue secondary TLB flushes
862 * from clear_young, leaving this job up to the regular system
863 * cadence. If we find this inaccurate, we might come up with a
864 * more sophisticated heuristic later.
865 */
866 return kvm_age_hva_range_no_flush(mn, start, end, kvm_age_gfn);
867 }
868
kvm_mmu_notifier_test_young(struct mmu_notifier * mn,struct mm_struct * mm,unsigned long address)869 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
870 struct mm_struct *mm,
871 unsigned long address)
872 {
873 trace_kvm_test_age_hva(address);
874
875 return kvm_age_hva_range_no_flush(mn, address, address + 1,
876 kvm_test_age_gfn);
877 }
878
kvm_mmu_notifier_release(struct mmu_notifier * mn,struct mm_struct * mm)879 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
880 struct mm_struct *mm)
881 {
882 struct kvm *kvm = mmu_notifier_to_kvm(mn);
883 int idx;
884
885 idx = srcu_read_lock(&kvm->srcu);
886 kvm_flush_shadow_all(kvm);
887 srcu_read_unlock(&kvm->srcu, idx);
888 }
889
890 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
891 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
892 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
893 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
894 .clear_young = kvm_mmu_notifier_clear_young,
895 .test_young = kvm_mmu_notifier_test_young,
896 .release = kvm_mmu_notifier_release,
897 };
898
kvm_init_mmu_notifier(struct kvm * kvm)899 static int kvm_init_mmu_notifier(struct kvm *kvm)
900 {
901 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
902 return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
903 }
904
905 #else /* !CONFIG_KVM_GENERIC_MMU_NOTIFIER */
906
kvm_init_mmu_notifier(struct kvm * kvm)907 static int kvm_init_mmu_notifier(struct kvm *kvm)
908 {
909 return 0;
910 }
911
912 #endif /* CONFIG_KVM_GENERIC_MMU_NOTIFIER */
913
914 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
kvm_pm_notifier_call(struct notifier_block * bl,unsigned long state,void * unused)915 static int kvm_pm_notifier_call(struct notifier_block *bl,
916 unsigned long state,
917 void *unused)
918 {
919 struct kvm *kvm = container_of(bl, struct kvm, pm_notifier);
920
921 return kvm_arch_pm_notifier(kvm, state);
922 }
923
kvm_init_pm_notifier(struct kvm * kvm)924 static void kvm_init_pm_notifier(struct kvm *kvm)
925 {
926 kvm->pm_notifier.notifier_call = kvm_pm_notifier_call;
927 /* Suspend KVM before we suspend ftrace, RCU, etc. */
928 kvm->pm_notifier.priority = INT_MAX;
929 register_pm_notifier(&kvm->pm_notifier);
930 }
931
kvm_destroy_pm_notifier(struct kvm * kvm)932 static void kvm_destroy_pm_notifier(struct kvm *kvm)
933 {
934 unregister_pm_notifier(&kvm->pm_notifier);
935 }
936 #else /* !CONFIG_HAVE_KVM_PM_NOTIFIER */
kvm_init_pm_notifier(struct kvm * kvm)937 static void kvm_init_pm_notifier(struct kvm *kvm)
938 {
939 }
940
kvm_destroy_pm_notifier(struct kvm * kvm)941 static void kvm_destroy_pm_notifier(struct kvm *kvm)
942 {
943 }
944 #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */
945
kvm_destroy_dirty_bitmap(struct kvm_memory_slot * memslot)946 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
947 {
948 if (!memslot->dirty_bitmap)
949 return;
950
951 vfree(memslot->dirty_bitmap);
952 memslot->dirty_bitmap = NULL;
953 }
954
955 /* This does not remove the slot from struct kvm_memslots data structures */
kvm_free_memslot(struct kvm * kvm,struct kvm_memory_slot * slot)956 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
957 {
958 if (slot->flags & KVM_MEM_GUEST_MEMFD)
959 kvm_gmem_unbind(slot);
960
961 kvm_destroy_dirty_bitmap(slot);
962
963 kvm_arch_free_memslot(kvm, slot);
964
965 kfree(slot);
966 }
967
kvm_free_memslots(struct kvm * kvm,struct kvm_memslots * slots)968 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
969 {
970 struct hlist_node *idnode;
971 struct kvm_memory_slot *memslot;
972 int bkt;
973
974 /*
975 * The same memslot objects live in both active and inactive sets,
976 * arbitrarily free using index '1' so the second invocation of this
977 * function isn't operating over a structure with dangling pointers
978 * (even though this function isn't actually touching them).
979 */
980 if (!slots->node_idx)
981 return;
982
983 hash_for_each_safe(slots->id_hash, bkt, idnode, memslot, id_node[1])
984 kvm_free_memslot(kvm, memslot);
985 }
986
kvm_stats_debugfs_mode(const struct _kvm_stats_desc * pdesc)987 static umode_t kvm_stats_debugfs_mode(const struct _kvm_stats_desc *pdesc)
988 {
989 switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) {
990 case KVM_STATS_TYPE_INSTANT:
991 return 0444;
992 case KVM_STATS_TYPE_CUMULATIVE:
993 case KVM_STATS_TYPE_PEAK:
994 default:
995 return 0644;
996 }
997 }
998
999
kvm_destroy_vm_debugfs(struct kvm * kvm)1000 static void kvm_destroy_vm_debugfs(struct kvm *kvm)
1001 {
1002 int i;
1003 int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc +
1004 kvm_vcpu_stats_header.num_desc;
1005
1006 if (IS_ERR(kvm->debugfs_dentry))
1007 return;
1008
1009 debugfs_remove_recursive(kvm->debugfs_dentry);
1010
1011 if (kvm->debugfs_stat_data) {
1012 for (i = 0; i < kvm_debugfs_num_entries; i++)
1013 kfree(kvm->debugfs_stat_data[i]);
1014 kfree(kvm->debugfs_stat_data);
1015 }
1016 }
1017
kvm_create_vm_debugfs(struct kvm * kvm,const char * fdname)1018 static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname)
1019 {
1020 static DEFINE_MUTEX(kvm_debugfs_lock);
1021 struct dentry *dent;
1022 char dir_name[ITOA_MAX_LEN * 2];
1023 struct kvm_stat_data *stat_data;
1024 const struct _kvm_stats_desc *pdesc;
1025 int i, ret = -ENOMEM;
1026 int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc +
1027 kvm_vcpu_stats_header.num_desc;
1028
1029 if (!debugfs_initialized())
1030 return 0;
1031
1032 snprintf(dir_name, sizeof(dir_name), "%d-%s", task_pid_nr(current), fdname);
1033 mutex_lock(&kvm_debugfs_lock);
1034 dent = debugfs_lookup(dir_name, kvm_debugfs_dir);
1035 if (dent) {
1036 pr_warn_ratelimited("KVM: debugfs: duplicate directory %s\n", dir_name);
1037 dput(dent);
1038 mutex_unlock(&kvm_debugfs_lock);
1039 return 0;
1040 }
1041 dent = debugfs_create_dir(dir_name, kvm_debugfs_dir);
1042 mutex_unlock(&kvm_debugfs_lock);
1043 if (IS_ERR(dent))
1044 return 0;
1045
1046 kvm->debugfs_dentry = dent;
1047 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
1048 sizeof(*kvm->debugfs_stat_data),
1049 GFP_KERNEL_ACCOUNT);
1050 if (!kvm->debugfs_stat_data)
1051 goto out_err;
1052
1053 for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) {
1054 pdesc = &kvm_vm_stats_desc[i];
1055 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
1056 if (!stat_data)
1057 goto out_err;
1058
1059 stat_data->kvm = kvm;
1060 stat_data->desc = pdesc;
1061 stat_data->kind = KVM_STAT_VM;
1062 kvm->debugfs_stat_data[i] = stat_data;
1063 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
1064 kvm->debugfs_dentry, stat_data,
1065 &stat_fops_per_vm);
1066 }
1067
1068 for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) {
1069 pdesc = &kvm_vcpu_stats_desc[i];
1070 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
1071 if (!stat_data)
1072 goto out_err;
1073
1074 stat_data->kvm = kvm;
1075 stat_data->desc = pdesc;
1076 stat_data->kind = KVM_STAT_VCPU;
1077 kvm->debugfs_stat_data[i + kvm_vm_stats_header.num_desc] = stat_data;
1078 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
1079 kvm->debugfs_dentry, stat_data,
1080 &stat_fops_per_vm);
1081 }
1082
1083 kvm_arch_create_vm_debugfs(kvm);
1084 return 0;
1085 out_err:
1086 kvm_destroy_vm_debugfs(kvm);
1087 return ret;
1088 }
1089
1090 /*
1091 * Called just after removing the VM from the vm_list, but before doing any
1092 * other destruction.
1093 */
kvm_arch_pre_destroy_vm(struct kvm * kvm)1094 void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm)
1095 {
1096 }
1097
1098 /*
1099 * Called after per-vm debugfs created. When called kvm->debugfs_dentry should
1100 * be setup already, so we can create arch-specific debugfs entries under it.
1101 * Cleanup should be automatic done in kvm_destroy_vm_debugfs() recursively, so
1102 * a per-arch destroy interface is not needed.
1103 */
kvm_arch_create_vm_debugfs(struct kvm * kvm)1104 void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
1105 {
1106 }
1107
kvm_create_vm(unsigned long type,const char * fdname)1108 static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
1109 {
1110 struct kvm *kvm = kvm_arch_alloc_vm();
1111 struct kvm_memslots *slots;
1112 int r, i, j;
1113
1114 if (!kvm)
1115 return ERR_PTR(-ENOMEM);
1116
1117 KVM_MMU_LOCK_INIT(kvm);
1118 mmgrab(current->mm);
1119 kvm->mm = current->mm;
1120 kvm_eventfd_init(kvm);
1121 mutex_init(&kvm->lock);
1122 mutex_init(&kvm->irq_lock);
1123 mutex_init(&kvm->slots_lock);
1124 mutex_init(&kvm->slots_arch_lock);
1125 spin_lock_init(&kvm->mn_invalidate_lock);
1126 rcuwait_init(&kvm->mn_memslots_update_rcuwait);
1127 xa_init(&kvm->vcpu_array);
1128 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
1129 xa_init(&kvm->mem_attr_array);
1130 #endif
1131
1132 INIT_LIST_HEAD(&kvm->gpc_list);
1133 spin_lock_init(&kvm->gpc_lock);
1134
1135 INIT_LIST_HEAD(&kvm->devices);
1136 kvm->max_vcpus = KVM_MAX_VCPUS;
1137
1138 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
1139
1140 /*
1141 * Force subsequent debugfs file creations to fail if the VM directory
1142 * is not created (by kvm_create_vm_debugfs()).
1143 */
1144 kvm->debugfs_dentry = ERR_PTR(-ENOENT);
1145
1146 snprintf(kvm->stats_id, sizeof(kvm->stats_id), "kvm-%d",
1147 task_pid_nr(current));
1148
1149 r = -ENOMEM;
1150 if (init_srcu_struct(&kvm->srcu))
1151 goto out_err_no_srcu;
1152 if (init_srcu_struct(&kvm->irq_srcu))
1153 goto out_err_no_irq_srcu;
1154
1155 r = kvm_init_irq_routing(kvm);
1156 if (r)
1157 goto out_err_no_irq_routing;
1158
1159 refcount_set(&kvm->users_count, 1);
1160
1161 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
1162 for (j = 0; j < 2; j++) {
1163 slots = &kvm->__memslots[i][j];
1164
1165 atomic_long_set(&slots->last_used_slot, (unsigned long)NULL);
1166 slots->hva_tree = RB_ROOT_CACHED;
1167 slots->gfn_tree = RB_ROOT;
1168 hash_init(slots->id_hash);
1169 slots->node_idx = j;
1170
1171 /* Generations must be different for each address space. */
1172 slots->generation = i;
1173 }
1174
1175 rcu_assign_pointer(kvm->memslots[i], &kvm->__memslots[i][0]);
1176 }
1177
1178 r = -ENOMEM;
1179 for (i = 0; i < KVM_NR_BUSES; i++) {
1180 rcu_assign_pointer(kvm->buses[i],
1181 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
1182 if (!kvm->buses[i])
1183 goto out_err_no_arch_destroy_vm;
1184 }
1185
1186 r = kvm_arch_init_vm(kvm, type);
1187 if (r)
1188 goto out_err_no_arch_destroy_vm;
1189
1190 r = kvm_enable_virtualization();
1191 if (r)
1192 goto out_err_no_disable;
1193
1194 #ifdef CONFIG_HAVE_KVM_IRQCHIP
1195 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
1196 #endif
1197
1198 r = kvm_init_mmu_notifier(kvm);
1199 if (r)
1200 goto out_err_no_mmu_notifier;
1201
1202 r = kvm_coalesced_mmio_init(kvm);
1203 if (r < 0)
1204 goto out_no_coalesced_mmio;
1205
1206 r = kvm_create_vm_debugfs(kvm, fdname);
1207 if (r)
1208 goto out_err_no_debugfs;
1209
1210 mutex_lock(&kvm_lock);
1211 list_add(&kvm->vm_list, &vm_list);
1212 mutex_unlock(&kvm_lock);
1213
1214 preempt_notifier_inc();
1215 kvm_init_pm_notifier(kvm);
1216
1217 return kvm;
1218
1219 out_err_no_debugfs:
1220 kvm_coalesced_mmio_free(kvm);
1221 out_no_coalesced_mmio:
1222 #ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
1223 if (kvm->mmu_notifier.ops)
1224 mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
1225 #endif
1226 out_err_no_mmu_notifier:
1227 kvm_disable_virtualization();
1228 out_err_no_disable:
1229 kvm_arch_destroy_vm(kvm);
1230 out_err_no_arch_destroy_vm:
1231 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
1232 for (i = 0; i < KVM_NR_BUSES; i++)
1233 kfree(kvm_get_bus(kvm, i));
1234 kvm_free_irq_routing(kvm);
1235 out_err_no_irq_routing:
1236 cleanup_srcu_struct(&kvm->irq_srcu);
1237 out_err_no_irq_srcu:
1238 cleanup_srcu_struct(&kvm->srcu);
1239 out_err_no_srcu:
1240 kvm_arch_free_vm(kvm);
1241 mmdrop(current->mm);
1242 return ERR_PTR(r);
1243 }
1244
kvm_destroy_devices(struct kvm * kvm)1245 static void kvm_destroy_devices(struct kvm *kvm)
1246 {
1247 struct kvm_device *dev, *tmp;
1248
1249 /*
1250 * We do not need to take the kvm->lock here, because nobody else
1251 * has a reference to the struct kvm at this point and therefore
1252 * cannot access the devices list anyhow.
1253 *
1254 * The device list is generally managed as an rculist, but list_del()
1255 * is used intentionally here. If a bug in KVM introduced a reader that
1256 * was not backed by a reference on the kvm struct, the hope is that
1257 * it'd consume the poisoned forward pointer instead of suffering a
1258 * use-after-free, even though this cannot be guaranteed.
1259 */
1260 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
1261 list_del(&dev->vm_node);
1262 dev->ops->destroy(dev);
1263 }
1264 }
1265
kvm_destroy_vm(struct kvm * kvm)1266 static void kvm_destroy_vm(struct kvm *kvm)
1267 {
1268 int i;
1269 struct mm_struct *mm = kvm->mm;
1270
1271 kvm_destroy_pm_notifier(kvm);
1272 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
1273 kvm_destroy_vm_debugfs(kvm);
1274 mutex_lock(&kvm_lock);
1275 list_del(&kvm->vm_list);
1276 mutex_unlock(&kvm_lock);
1277 kvm_arch_pre_destroy_vm(kvm);
1278
1279 kvm_free_irq_routing(kvm);
1280 for (i = 0; i < KVM_NR_BUSES; i++) {
1281 struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
1282
1283 if (bus)
1284 kvm_io_bus_destroy(bus);
1285 kvm->buses[i] = NULL;
1286 }
1287 kvm_coalesced_mmio_free(kvm);
1288 #ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
1289 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
1290 /*
1291 * At this point, pending calls to invalidate_range_start()
1292 * have completed but no more MMU notifiers will run, so
1293 * mn_active_invalidate_count may remain unbalanced.
1294 * No threads can be waiting in kvm_swap_active_memslots() as the
1295 * last reference on KVM has been dropped, but freeing
1296 * memslots would deadlock without this manual intervention.
1297 *
1298 * If the count isn't unbalanced, i.e. KVM did NOT unregister its MMU
1299 * notifier between a start() and end(), then there shouldn't be any
1300 * in-progress invalidations.
1301 */
1302 WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait));
1303 if (kvm->mn_active_invalidate_count)
1304 kvm->mn_active_invalidate_count = 0;
1305 else
1306 WARN_ON(kvm->mmu_invalidate_in_progress);
1307 #else
1308 kvm_flush_shadow_all(kvm);
1309 #endif
1310 kvm_arch_destroy_vm(kvm);
1311 kvm_destroy_devices(kvm);
1312 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
1313 kvm_free_memslots(kvm, &kvm->__memslots[i][0]);
1314 kvm_free_memslots(kvm, &kvm->__memslots[i][1]);
1315 }
1316 cleanup_srcu_struct(&kvm->irq_srcu);
1317 cleanup_srcu_struct(&kvm->srcu);
1318 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
1319 xa_destroy(&kvm->mem_attr_array);
1320 #endif
1321 kvm_arch_free_vm(kvm);
1322 preempt_notifier_dec();
1323 kvm_disable_virtualization();
1324 mmdrop(mm);
1325 }
1326
kvm_get_kvm(struct kvm * kvm)1327 void kvm_get_kvm(struct kvm *kvm)
1328 {
1329 refcount_inc(&kvm->users_count);
1330 }
1331 EXPORT_SYMBOL_GPL(kvm_get_kvm);
1332
1333 /*
1334 * Make sure the vm is not during destruction, which is a safe version of
1335 * kvm_get_kvm(). Return true if kvm referenced successfully, false otherwise.
1336 */
kvm_get_kvm_safe(struct kvm * kvm)1337 bool kvm_get_kvm_safe(struct kvm *kvm)
1338 {
1339 return refcount_inc_not_zero(&kvm->users_count);
1340 }
1341 EXPORT_SYMBOL_GPL(kvm_get_kvm_safe);
1342
kvm_put_kvm(struct kvm * kvm)1343 void kvm_put_kvm(struct kvm *kvm)
1344 {
1345 if (refcount_dec_and_test(&kvm->users_count))
1346 kvm_destroy_vm(kvm);
1347 }
1348 EXPORT_SYMBOL_GPL(kvm_put_kvm);
1349
1350 /*
1351 * Used to put a reference that was taken on behalf of an object associated
1352 * with a user-visible file descriptor, e.g. a vcpu or device, if installation
1353 * of the new file descriptor fails and the reference cannot be transferred to
1354 * its final owner. In such cases, the caller is still actively using @kvm and
1355 * will fail miserably if the refcount unexpectedly hits zero.
1356 */
kvm_put_kvm_no_destroy(struct kvm * kvm)1357 void kvm_put_kvm_no_destroy(struct kvm *kvm)
1358 {
1359 WARN_ON(refcount_dec_and_test(&kvm->users_count));
1360 }
1361 EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy);
1362
kvm_vm_release(struct inode * inode,struct file * filp)1363 static int kvm_vm_release(struct inode *inode, struct file *filp)
1364 {
1365 struct kvm *kvm = filp->private_data;
1366
1367 kvm_irqfd_release(kvm);
1368
1369 kvm_put_kvm(kvm);
1370 return 0;
1371 }
1372
1373 /*
1374 * Allocation size is twice as large as the actual dirty bitmap size.
1375 * See kvm_vm_ioctl_get_dirty_log() why this is needed.
1376 */
kvm_alloc_dirty_bitmap(struct kvm_memory_slot * memslot)1377 static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot)
1378 {
1379 unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(memslot);
1380
1381 memslot->dirty_bitmap = __vcalloc(2, dirty_bytes, GFP_KERNEL_ACCOUNT);
1382 if (!memslot->dirty_bitmap)
1383 return -ENOMEM;
1384
1385 return 0;
1386 }
1387
kvm_get_inactive_memslots(struct kvm * kvm,int as_id)1388 static struct kvm_memslots *kvm_get_inactive_memslots(struct kvm *kvm, int as_id)
1389 {
1390 struct kvm_memslots *active = __kvm_memslots(kvm, as_id);
1391 int node_idx_inactive = active->node_idx ^ 1;
1392
1393 return &kvm->__memslots[as_id][node_idx_inactive];
1394 }
1395
1396 /*
1397 * Helper to get the address space ID when one of memslot pointers may be NULL.
1398 * This also serves as a sanity that at least one of the pointers is non-NULL,
1399 * and that their address space IDs don't diverge.
1400 */
kvm_memslots_get_as_id(struct kvm_memory_slot * a,struct kvm_memory_slot * b)1401 static int kvm_memslots_get_as_id(struct kvm_memory_slot *a,
1402 struct kvm_memory_slot *b)
1403 {
1404 if (WARN_ON_ONCE(!a && !b))
1405 return 0;
1406
1407 if (!a)
1408 return b->as_id;
1409 if (!b)
1410 return a->as_id;
1411
1412 WARN_ON_ONCE(a->as_id != b->as_id);
1413 return a->as_id;
1414 }
1415
kvm_insert_gfn_node(struct kvm_memslots * slots,struct kvm_memory_slot * slot)1416 static void kvm_insert_gfn_node(struct kvm_memslots *slots,
1417 struct kvm_memory_slot *slot)
1418 {
1419 struct rb_root *gfn_tree = &slots->gfn_tree;
1420 struct rb_node **node, *parent;
1421 int idx = slots->node_idx;
1422
1423 parent = NULL;
1424 for (node = &gfn_tree->rb_node; *node; ) {
1425 struct kvm_memory_slot *tmp;
1426
1427 tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]);
1428 parent = *node;
1429 if (slot->base_gfn < tmp->base_gfn)
1430 node = &(*node)->rb_left;
1431 else if (slot->base_gfn > tmp->base_gfn)
1432 node = &(*node)->rb_right;
1433 else
1434 BUG();
1435 }
1436
1437 rb_link_node(&slot->gfn_node[idx], parent, node);
1438 rb_insert_color(&slot->gfn_node[idx], gfn_tree);
1439 }
1440
kvm_erase_gfn_node(struct kvm_memslots * slots,struct kvm_memory_slot * slot)1441 static void kvm_erase_gfn_node(struct kvm_memslots *slots,
1442 struct kvm_memory_slot *slot)
1443 {
1444 rb_erase(&slot->gfn_node[slots->node_idx], &slots->gfn_tree);
1445 }
1446
kvm_replace_gfn_node(struct kvm_memslots * slots,struct kvm_memory_slot * old,struct kvm_memory_slot * new)1447 static void kvm_replace_gfn_node(struct kvm_memslots *slots,
1448 struct kvm_memory_slot *old,
1449 struct kvm_memory_slot *new)
1450 {
1451 int idx = slots->node_idx;
1452
1453 WARN_ON_ONCE(old->base_gfn != new->base_gfn);
1454
1455 rb_replace_node(&old->gfn_node[idx], &new->gfn_node[idx],
1456 &slots->gfn_tree);
1457 }
1458
1459 /*
1460 * Replace @old with @new in the inactive memslots.
1461 *
1462 * With NULL @old this simply adds @new.
1463 * With NULL @new this simply removes @old.
1464 *
1465 * If @new is non-NULL its hva_node[slots_idx] range has to be set
1466 * appropriately.
1467 */
kvm_replace_memslot(struct kvm * kvm,struct kvm_memory_slot * old,struct kvm_memory_slot * new)1468 static void kvm_replace_memslot(struct kvm *kvm,
1469 struct kvm_memory_slot *old,
1470 struct kvm_memory_slot *new)
1471 {
1472 int as_id = kvm_memslots_get_as_id(old, new);
1473 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
1474 int idx = slots->node_idx;
1475
1476 if (old) {
1477 hash_del(&old->id_node[idx]);
1478 interval_tree_remove(&old->hva_node[idx], &slots->hva_tree);
1479
1480 if ((long)old == atomic_long_read(&slots->last_used_slot))
1481 atomic_long_set(&slots->last_used_slot, (long)new);
1482
1483 if (!new) {
1484 kvm_erase_gfn_node(slots, old);
1485 return;
1486 }
1487 }
1488
1489 /*
1490 * Initialize @new's hva range. Do this even when replacing an @old
1491 * slot, kvm_copy_memslot() deliberately does not touch node data.
1492 */
1493 new->hva_node[idx].start = new->userspace_addr;
1494 new->hva_node[idx].last = new->userspace_addr +
1495 (new->npages << PAGE_SHIFT) - 1;
1496
1497 /*
1498 * (Re)Add the new memslot. There is no O(1) interval_tree_replace(),
1499 * hva_node needs to be swapped with remove+insert even though hva can't
1500 * change when replacing an existing slot.
1501 */
1502 hash_add(slots->id_hash, &new->id_node[idx], new->id);
1503 interval_tree_insert(&new->hva_node[idx], &slots->hva_tree);
1504
1505 /*
1506 * If the memslot gfn is unchanged, rb_replace_node() can be used to
1507 * switch the node in the gfn tree instead of removing the old and
1508 * inserting the new as two separate operations. Replacement is a
1509 * single O(1) operation versus two O(log(n)) operations for
1510 * remove+insert.
1511 */
1512 if (old && old->base_gfn == new->base_gfn) {
1513 kvm_replace_gfn_node(slots, old, new);
1514 } else {
1515 if (old)
1516 kvm_erase_gfn_node(slots, old);
1517 kvm_insert_gfn_node(slots, new);
1518 }
1519 }
1520
1521 /*
1522 * Flags that do not access any of the extra space of struct
1523 * kvm_userspace_memory_region2. KVM_SET_USER_MEMORY_REGION_V1_FLAGS
1524 * only allows these.
1525 */
1526 #define KVM_SET_USER_MEMORY_REGION_V1_FLAGS \
1527 (KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY)
1528
check_memory_region_flags(struct kvm * kvm,const struct kvm_userspace_memory_region2 * mem)1529 static int check_memory_region_flags(struct kvm *kvm,
1530 const struct kvm_userspace_memory_region2 *mem)
1531 {
1532 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
1533
1534 if (kvm_arch_has_private_mem(kvm))
1535 valid_flags |= KVM_MEM_GUEST_MEMFD;
1536
1537 /* Dirty logging private memory is not currently supported. */
1538 if (mem->flags & KVM_MEM_GUEST_MEMFD)
1539 valid_flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
1540
1541 /*
1542 * GUEST_MEMFD is incompatible with read-only memslots, as writes to
1543 * read-only memslots have emulated MMIO, not page fault, semantics,
1544 * and KVM doesn't allow emulated MMIO for private memory.
1545 */
1546 if (kvm_arch_has_readonly_mem(kvm) &&
1547 !(mem->flags & KVM_MEM_GUEST_MEMFD))
1548 valid_flags |= KVM_MEM_READONLY;
1549
1550 if (mem->flags & ~valid_flags)
1551 return -EINVAL;
1552
1553 return 0;
1554 }
1555
kvm_swap_active_memslots(struct kvm * kvm,int as_id)1556 static void kvm_swap_active_memslots(struct kvm *kvm, int as_id)
1557 {
1558 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
1559
1560 /* Grab the generation from the activate memslots. */
1561 u64 gen = __kvm_memslots(kvm, as_id)->generation;
1562
1563 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
1564 slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
1565
1566 /*
1567 * Do not store the new memslots while there are invalidations in
1568 * progress, otherwise the locking in invalidate_range_start and
1569 * invalidate_range_end will be unbalanced.
1570 */
1571 spin_lock(&kvm->mn_invalidate_lock);
1572 prepare_to_rcuwait(&kvm->mn_memslots_update_rcuwait);
1573 while (kvm->mn_active_invalidate_count) {
1574 set_current_state(TASK_UNINTERRUPTIBLE);
1575 spin_unlock(&kvm->mn_invalidate_lock);
1576 schedule();
1577 spin_lock(&kvm->mn_invalidate_lock);
1578 }
1579 finish_rcuwait(&kvm->mn_memslots_update_rcuwait);
1580 rcu_assign_pointer(kvm->memslots[as_id], slots);
1581 spin_unlock(&kvm->mn_invalidate_lock);
1582
1583 /*
1584 * Acquired in kvm_set_memslot. Must be released before synchronize
1585 * SRCU below in order to avoid deadlock with another thread
1586 * acquiring the slots_arch_lock in an srcu critical section.
1587 */
1588 mutex_unlock(&kvm->slots_arch_lock);
1589
1590 synchronize_srcu_expedited(&kvm->srcu);
1591
1592 /*
1593 * Increment the new memslot generation a second time, dropping the
1594 * update in-progress flag and incrementing the generation based on
1595 * the number of address spaces. This provides a unique and easily
1596 * identifiable generation number while the memslots are in flux.
1597 */
1598 gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
1599
1600 /*
1601 * Generations must be unique even across address spaces. We do not need
1602 * a global counter for that, instead the generation space is evenly split
1603 * across address spaces. For example, with two address spaces, address
1604 * space 0 will use generations 0, 2, 4, ... while address space 1 will
1605 * use generations 1, 3, 5, ...
1606 */
1607 gen += kvm_arch_nr_memslot_as_ids(kvm);
1608
1609 kvm_arch_memslots_updated(kvm, gen);
1610
1611 slots->generation = gen;
1612 }
1613
kvm_prepare_memory_region(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)1614 static int kvm_prepare_memory_region(struct kvm *kvm,
1615 const struct kvm_memory_slot *old,
1616 struct kvm_memory_slot *new,
1617 enum kvm_mr_change change)
1618 {
1619 int r;
1620
1621 /*
1622 * If dirty logging is disabled, nullify the bitmap; the old bitmap
1623 * will be freed on "commit". If logging is enabled in both old and
1624 * new, reuse the existing bitmap. If logging is enabled only in the
1625 * new and KVM isn't using a ring buffer, allocate and initialize a
1626 * new bitmap.
1627 */
1628 if (change != KVM_MR_DELETE) {
1629 if (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
1630 new->dirty_bitmap = NULL;
1631 else if (old && old->dirty_bitmap)
1632 new->dirty_bitmap = old->dirty_bitmap;
1633 else if (kvm_use_dirty_bitmap(kvm)) {
1634 r = kvm_alloc_dirty_bitmap(new);
1635 if (r)
1636 return r;
1637
1638 if (kvm_dirty_log_manual_protect_and_init_set(kvm))
1639 bitmap_set(new->dirty_bitmap, 0, new->npages);
1640 }
1641 }
1642
1643 r = kvm_arch_prepare_memory_region(kvm, old, new, change);
1644
1645 /* Free the bitmap on failure if it was allocated above. */
1646 if (r && new && new->dirty_bitmap && (!old || !old->dirty_bitmap))
1647 kvm_destroy_dirty_bitmap(new);
1648
1649 return r;
1650 }
1651
kvm_commit_memory_region(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)1652 static void kvm_commit_memory_region(struct kvm *kvm,
1653 struct kvm_memory_slot *old,
1654 const struct kvm_memory_slot *new,
1655 enum kvm_mr_change change)
1656 {
1657 int old_flags = old ? old->flags : 0;
1658 int new_flags = new ? new->flags : 0;
1659 /*
1660 * Update the total number of memslot pages before calling the arch
1661 * hook so that architectures can consume the result directly.
1662 */
1663 if (change == KVM_MR_DELETE)
1664 kvm->nr_memslot_pages -= old->npages;
1665 else if (change == KVM_MR_CREATE)
1666 kvm->nr_memslot_pages += new->npages;
1667
1668 if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES) {
1669 int change = (new_flags & KVM_MEM_LOG_DIRTY_PAGES) ? 1 : -1;
1670 atomic_set(&kvm->nr_memslots_dirty_logging,
1671 atomic_read(&kvm->nr_memslots_dirty_logging) + change);
1672 }
1673
1674 kvm_arch_commit_memory_region(kvm, old, new, change);
1675
1676 switch (change) {
1677 case KVM_MR_CREATE:
1678 /* Nothing more to do. */
1679 break;
1680 case KVM_MR_DELETE:
1681 /* Free the old memslot and all its metadata. */
1682 kvm_free_memslot(kvm, old);
1683 break;
1684 case KVM_MR_MOVE:
1685 case KVM_MR_FLAGS_ONLY:
1686 /*
1687 * Free the dirty bitmap as needed; the below check encompasses
1688 * both the flags and whether a ring buffer is being used)
1689 */
1690 if (old->dirty_bitmap && !new->dirty_bitmap)
1691 kvm_destroy_dirty_bitmap(old);
1692
1693 /*
1694 * The final quirk. Free the detached, old slot, but only its
1695 * memory, not any metadata. Metadata, including arch specific
1696 * data, may be reused by @new.
1697 */
1698 kfree(old);
1699 break;
1700 default:
1701 BUG();
1702 }
1703 }
1704
1705 /*
1706 * Activate @new, which must be installed in the inactive slots by the caller,
1707 * by swapping the active slots and then propagating @new to @old once @old is
1708 * unreachable and can be safely modified.
1709 *
1710 * With NULL @old this simply adds @new to @active (while swapping the sets).
1711 * With NULL @new this simply removes @old from @active and frees it
1712 * (while also swapping the sets).
1713 */
kvm_activate_memslot(struct kvm * kvm,struct kvm_memory_slot * old,struct kvm_memory_slot * new)1714 static void kvm_activate_memslot(struct kvm *kvm,
1715 struct kvm_memory_slot *old,
1716 struct kvm_memory_slot *new)
1717 {
1718 int as_id = kvm_memslots_get_as_id(old, new);
1719
1720 kvm_swap_active_memslots(kvm, as_id);
1721
1722 /* Propagate the new memslot to the now inactive memslots. */
1723 kvm_replace_memslot(kvm, old, new);
1724 }
1725
kvm_copy_memslot(struct kvm_memory_slot * dest,const struct kvm_memory_slot * src)1726 static void kvm_copy_memslot(struct kvm_memory_slot *dest,
1727 const struct kvm_memory_slot *src)
1728 {
1729 dest->base_gfn = src->base_gfn;
1730 dest->npages = src->npages;
1731 dest->dirty_bitmap = src->dirty_bitmap;
1732 dest->arch = src->arch;
1733 dest->userspace_addr = src->userspace_addr;
1734 dest->flags = src->flags;
1735 dest->id = src->id;
1736 dest->as_id = src->as_id;
1737 }
1738
kvm_invalidate_memslot(struct kvm * kvm,struct kvm_memory_slot * old,struct kvm_memory_slot * invalid_slot)1739 static void kvm_invalidate_memslot(struct kvm *kvm,
1740 struct kvm_memory_slot *old,
1741 struct kvm_memory_slot *invalid_slot)
1742 {
1743 /*
1744 * Mark the current slot INVALID. As with all memslot modifications,
1745 * this must be done on an unreachable slot to avoid modifying the
1746 * current slot in the active tree.
1747 */
1748 kvm_copy_memslot(invalid_slot, old);
1749 invalid_slot->flags |= KVM_MEMSLOT_INVALID;
1750 kvm_replace_memslot(kvm, old, invalid_slot);
1751
1752 /*
1753 * Activate the slot that is now marked INVALID, but don't propagate
1754 * the slot to the now inactive slots. The slot is either going to be
1755 * deleted or recreated as a new slot.
1756 */
1757 kvm_swap_active_memslots(kvm, old->as_id);
1758
1759 /*
1760 * From this point no new shadow pages pointing to a deleted, or moved,
1761 * memslot will be created. Validation of sp->gfn happens in:
1762 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1763 * - kvm_is_visible_gfn (mmu_check_root)
1764 */
1765 kvm_arch_flush_shadow_memslot(kvm, old);
1766 kvm_arch_guest_memory_reclaimed(kvm);
1767
1768 /* Was released by kvm_swap_active_memslots(), reacquire. */
1769 mutex_lock(&kvm->slots_arch_lock);
1770
1771 /*
1772 * Copy the arch-specific field of the newly-installed slot back to the
1773 * old slot as the arch data could have changed between releasing
1774 * slots_arch_lock in kvm_swap_active_memslots() and re-acquiring the lock
1775 * above. Writers are required to retrieve memslots *after* acquiring
1776 * slots_arch_lock, thus the active slot's data is guaranteed to be fresh.
1777 */
1778 old->arch = invalid_slot->arch;
1779 }
1780
kvm_create_memslot(struct kvm * kvm,struct kvm_memory_slot * new)1781 static void kvm_create_memslot(struct kvm *kvm,
1782 struct kvm_memory_slot *new)
1783 {
1784 /* Add the new memslot to the inactive set and activate. */
1785 kvm_replace_memslot(kvm, NULL, new);
1786 kvm_activate_memslot(kvm, NULL, new);
1787 }
1788
kvm_delete_memslot(struct kvm * kvm,struct kvm_memory_slot * old,struct kvm_memory_slot * invalid_slot)1789 static void kvm_delete_memslot(struct kvm *kvm,
1790 struct kvm_memory_slot *old,
1791 struct kvm_memory_slot *invalid_slot)
1792 {
1793 /*
1794 * Remove the old memslot (in the inactive memslots) by passing NULL as
1795 * the "new" slot, and for the invalid version in the active slots.
1796 */
1797 kvm_replace_memslot(kvm, old, NULL);
1798 kvm_activate_memslot(kvm, invalid_slot, NULL);
1799 }
1800
kvm_move_memslot(struct kvm * kvm,struct kvm_memory_slot * old,struct kvm_memory_slot * new,struct kvm_memory_slot * invalid_slot)1801 static void kvm_move_memslot(struct kvm *kvm,
1802 struct kvm_memory_slot *old,
1803 struct kvm_memory_slot *new,
1804 struct kvm_memory_slot *invalid_slot)
1805 {
1806 /*
1807 * Replace the old memslot in the inactive slots, and then swap slots
1808 * and replace the current INVALID with the new as well.
1809 */
1810 kvm_replace_memslot(kvm, old, new);
1811 kvm_activate_memslot(kvm, invalid_slot, new);
1812 }
1813
kvm_update_flags_memslot(struct kvm * kvm,struct kvm_memory_slot * old,struct kvm_memory_slot * new)1814 static void kvm_update_flags_memslot(struct kvm *kvm,
1815 struct kvm_memory_slot *old,
1816 struct kvm_memory_slot *new)
1817 {
1818 /*
1819 * Similar to the MOVE case, but the slot doesn't need to be zapped as
1820 * an intermediate step. Instead, the old memslot is simply replaced
1821 * with a new, updated copy in both memslot sets.
1822 */
1823 kvm_replace_memslot(kvm, old, new);
1824 kvm_activate_memslot(kvm, old, new);
1825 }
1826
kvm_set_memslot(struct kvm * kvm,struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)1827 static int kvm_set_memslot(struct kvm *kvm,
1828 struct kvm_memory_slot *old,
1829 struct kvm_memory_slot *new,
1830 enum kvm_mr_change change)
1831 {
1832 struct kvm_memory_slot *invalid_slot;
1833 int r;
1834
1835 /*
1836 * Released in kvm_swap_active_memslots().
1837 *
1838 * Must be held from before the current memslots are copied until after
1839 * the new memslots are installed with rcu_assign_pointer, then
1840 * released before the synchronize srcu in kvm_swap_active_memslots().
1841 *
1842 * When modifying memslots outside of the slots_lock, must be held
1843 * before reading the pointer to the current memslots until after all
1844 * changes to those memslots are complete.
1845 *
1846 * These rules ensure that installing new memslots does not lose
1847 * changes made to the previous memslots.
1848 */
1849 mutex_lock(&kvm->slots_arch_lock);
1850
1851 /*
1852 * Invalidate the old slot if it's being deleted or moved. This is
1853 * done prior to actually deleting/moving the memslot to allow vCPUs to
1854 * continue running by ensuring there are no mappings or shadow pages
1855 * for the memslot when it is deleted/moved. Without pre-invalidation
1856 * (and without a lock), a window would exist between effecting the
1857 * delete/move and committing the changes in arch code where KVM or a
1858 * guest could access a non-existent memslot.
1859 *
1860 * Modifications are done on a temporary, unreachable slot. The old
1861 * slot needs to be preserved in case a later step fails and the
1862 * invalidation needs to be reverted.
1863 */
1864 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
1865 invalid_slot = kzalloc(sizeof(*invalid_slot), GFP_KERNEL_ACCOUNT);
1866 if (!invalid_slot) {
1867 mutex_unlock(&kvm->slots_arch_lock);
1868 return -ENOMEM;
1869 }
1870 kvm_invalidate_memslot(kvm, old, invalid_slot);
1871 }
1872
1873 r = kvm_prepare_memory_region(kvm, old, new, change);
1874 if (r) {
1875 /*
1876 * For DELETE/MOVE, revert the above INVALID change. No
1877 * modifications required since the original slot was preserved
1878 * in the inactive slots. Changing the active memslots also
1879 * release slots_arch_lock.
1880 */
1881 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
1882 kvm_activate_memslot(kvm, invalid_slot, old);
1883 kfree(invalid_slot);
1884 } else {
1885 mutex_unlock(&kvm->slots_arch_lock);
1886 }
1887 return r;
1888 }
1889
1890 /*
1891 * For DELETE and MOVE, the working slot is now active as the INVALID
1892 * version of the old slot. MOVE is particularly special as it reuses
1893 * the old slot and returns a copy of the old slot (in working_slot).
1894 * For CREATE, there is no old slot. For DELETE and FLAGS_ONLY, the
1895 * old slot is detached but otherwise preserved.
1896 */
1897 if (change == KVM_MR_CREATE)
1898 kvm_create_memslot(kvm, new);
1899 else if (change == KVM_MR_DELETE)
1900 kvm_delete_memslot(kvm, old, invalid_slot);
1901 else if (change == KVM_MR_MOVE)
1902 kvm_move_memslot(kvm, old, new, invalid_slot);
1903 else if (change == KVM_MR_FLAGS_ONLY)
1904 kvm_update_flags_memslot(kvm, old, new);
1905 else
1906 BUG();
1907
1908 /* Free the temporary INVALID slot used for DELETE and MOVE. */
1909 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
1910 kfree(invalid_slot);
1911
1912 /*
1913 * No need to refresh new->arch, changes after dropping slots_arch_lock
1914 * will directly hit the final, active memslot. Architectures are
1915 * responsible for knowing that new->arch may be stale.
1916 */
1917 kvm_commit_memory_region(kvm, old, new, change);
1918
1919 return 0;
1920 }
1921
kvm_check_memslot_overlap(struct kvm_memslots * slots,int id,gfn_t start,gfn_t end)1922 static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id,
1923 gfn_t start, gfn_t end)
1924 {
1925 struct kvm_memslot_iter iter;
1926
1927 kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) {
1928 if (iter.slot->id != id)
1929 return true;
1930 }
1931
1932 return false;
1933 }
1934
kvm_set_memory_region(struct kvm * kvm,const struct kvm_userspace_memory_region2 * mem)1935 static int kvm_set_memory_region(struct kvm *kvm,
1936 const struct kvm_userspace_memory_region2 *mem)
1937 {
1938 struct kvm_memory_slot *old, *new;
1939 struct kvm_memslots *slots;
1940 enum kvm_mr_change change;
1941 unsigned long npages;
1942 gfn_t base_gfn;
1943 int as_id, id;
1944 int r;
1945
1946 lockdep_assert_held(&kvm->slots_lock);
1947
1948 r = check_memory_region_flags(kvm, mem);
1949 if (r)
1950 return r;
1951
1952 as_id = mem->slot >> 16;
1953 id = (u16)mem->slot;
1954
1955 /* General sanity checks */
1956 if ((mem->memory_size & (PAGE_SIZE - 1)) ||
1957 (mem->memory_size != (unsigned long)mem->memory_size))
1958 return -EINVAL;
1959 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
1960 return -EINVAL;
1961 /* We can read the guest memory with __xxx_user() later on. */
1962 if ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
1963 (mem->userspace_addr != untagged_addr(mem->userspace_addr)) ||
1964 !access_ok((void __user *)(unsigned long)mem->userspace_addr,
1965 mem->memory_size))
1966 return -EINVAL;
1967 if (mem->flags & KVM_MEM_GUEST_MEMFD &&
1968 (mem->guest_memfd_offset & (PAGE_SIZE - 1) ||
1969 mem->guest_memfd_offset + mem->memory_size < mem->guest_memfd_offset))
1970 return -EINVAL;
1971 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_MEM_SLOTS_NUM)
1972 return -EINVAL;
1973 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
1974 return -EINVAL;
1975
1976 /*
1977 * The size of userspace-defined memory regions is restricted in order
1978 * to play nice with dirty bitmap operations, which are indexed with an
1979 * "unsigned int". KVM's internal memory regions don't support dirty
1980 * logging, and so are exempt.
1981 */
1982 if (id < KVM_USER_MEM_SLOTS &&
1983 (mem->memory_size >> PAGE_SHIFT) > KVM_MEM_MAX_NR_PAGES)
1984 return -EINVAL;
1985
1986 slots = __kvm_memslots(kvm, as_id);
1987
1988 /*
1989 * Note, the old memslot (and the pointer itself!) may be invalidated
1990 * and/or destroyed by kvm_set_memslot().
1991 */
1992 old = id_to_memslot(slots, id);
1993
1994 if (!mem->memory_size) {
1995 if (!old || !old->npages)
1996 return -EINVAL;
1997
1998 if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages))
1999 return -EIO;
2000
2001 return kvm_set_memslot(kvm, old, NULL, KVM_MR_DELETE);
2002 }
2003
2004 base_gfn = (mem->guest_phys_addr >> PAGE_SHIFT);
2005 npages = (mem->memory_size >> PAGE_SHIFT);
2006
2007 if (!old || !old->npages) {
2008 change = KVM_MR_CREATE;
2009
2010 /*
2011 * To simplify KVM internals, the total number of pages across
2012 * all memslots must fit in an unsigned long.
2013 */
2014 if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages)
2015 return -EINVAL;
2016 } else { /* Modify an existing slot. */
2017 /* Private memslots are immutable, they can only be deleted. */
2018 if (mem->flags & KVM_MEM_GUEST_MEMFD)
2019 return -EINVAL;
2020 if ((mem->userspace_addr != old->userspace_addr) ||
2021 (npages != old->npages) ||
2022 ((mem->flags ^ old->flags) & KVM_MEM_READONLY))
2023 return -EINVAL;
2024
2025 if (base_gfn != old->base_gfn)
2026 change = KVM_MR_MOVE;
2027 else if (mem->flags != old->flags)
2028 change = KVM_MR_FLAGS_ONLY;
2029 else /* Nothing to change. */
2030 return 0;
2031 }
2032
2033 if ((change == KVM_MR_CREATE || change == KVM_MR_MOVE) &&
2034 kvm_check_memslot_overlap(slots, id, base_gfn, base_gfn + npages))
2035 return -EEXIST;
2036
2037 /* Allocate a slot that will persist in the memslot. */
2038 new = kzalloc(sizeof(*new), GFP_KERNEL_ACCOUNT);
2039 if (!new)
2040 return -ENOMEM;
2041
2042 new->as_id = as_id;
2043 new->id = id;
2044 new->base_gfn = base_gfn;
2045 new->npages = npages;
2046 new->flags = mem->flags;
2047 new->userspace_addr = mem->userspace_addr;
2048 if (mem->flags & KVM_MEM_GUEST_MEMFD) {
2049 r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset);
2050 if (r)
2051 goto out;
2052 }
2053
2054 r = kvm_set_memslot(kvm, old, new, change);
2055 if (r)
2056 goto out_unbind;
2057
2058 return 0;
2059
2060 out_unbind:
2061 if (mem->flags & KVM_MEM_GUEST_MEMFD)
2062 kvm_gmem_unbind(new);
2063 out:
2064 kfree(new);
2065 return r;
2066 }
2067
kvm_set_internal_memslot(struct kvm * kvm,const struct kvm_userspace_memory_region2 * mem)2068 int kvm_set_internal_memslot(struct kvm *kvm,
2069 const struct kvm_userspace_memory_region2 *mem)
2070 {
2071 if (WARN_ON_ONCE(mem->slot < KVM_USER_MEM_SLOTS))
2072 return -EINVAL;
2073
2074 if (WARN_ON_ONCE(mem->flags))
2075 return -EINVAL;
2076
2077 return kvm_set_memory_region(kvm, mem);
2078 }
2079 EXPORT_SYMBOL_GPL(kvm_set_internal_memslot);
2080
kvm_vm_ioctl_set_memory_region(struct kvm * kvm,struct kvm_userspace_memory_region2 * mem)2081 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
2082 struct kvm_userspace_memory_region2 *mem)
2083 {
2084 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
2085 return -EINVAL;
2086
2087 guard(mutex)(&kvm->slots_lock);
2088 return kvm_set_memory_region(kvm, mem);
2089 }
2090
2091 #ifndef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
2092 /**
2093 * kvm_get_dirty_log - get a snapshot of dirty pages
2094 * @kvm: pointer to kvm instance
2095 * @log: slot id and address to which we copy the log
2096 * @is_dirty: set to '1' if any dirty pages were found
2097 * @memslot: set to the associated memslot, always valid on success
2098 */
kvm_get_dirty_log(struct kvm * kvm,struct kvm_dirty_log * log,int * is_dirty,struct kvm_memory_slot ** memslot)2099 int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
2100 int *is_dirty, struct kvm_memory_slot **memslot)
2101 {
2102 struct kvm_memslots *slots;
2103 int i, as_id, id;
2104 unsigned long n;
2105 unsigned long any = 0;
2106
2107 /* Dirty ring tracking may be exclusive to dirty log tracking */
2108 if (!kvm_use_dirty_bitmap(kvm))
2109 return -ENXIO;
2110
2111 *memslot = NULL;
2112 *is_dirty = 0;
2113
2114 as_id = log->slot >> 16;
2115 id = (u16)log->slot;
2116 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
2117 return -EINVAL;
2118
2119 slots = __kvm_memslots(kvm, as_id);
2120 *memslot = id_to_memslot(slots, id);
2121 if (!(*memslot) || !(*memslot)->dirty_bitmap)
2122 return -ENOENT;
2123
2124 kvm_arch_sync_dirty_log(kvm, *memslot);
2125
2126 n = kvm_dirty_bitmap_bytes(*memslot);
2127
2128 for (i = 0; !any && i < n/sizeof(long); ++i)
2129 any = (*memslot)->dirty_bitmap[i];
2130
2131 if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n))
2132 return -EFAULT;
2133
2134 if (any)
2135 *is_dirty = 1;
2136 return 0;
2137 }
2138 EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
2139
2140 #else /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
2141 /**
2142 * kvm_get_dirty_log_protect - get a snapshot of dirty pages
2143 * and reenable dirty page tracking for the corresponding pages.
2144 * @kvm: pointer to kvm instance
2145 * @log: slot id and address to which we copy the log
2146 *
2147 * We need to keep it in mind that VCPU threads can write to the bitmap
2148 * concurrently. So, to avoid losing track of dirty pages we keep the
2149 * following order:
2150 *
2151 * 1. Take a snapshot of the bit and clear it if needed.
2152 * 2. Write protect the corresponding page.
2153 * 3. Copy the snapshot to the userspace.
2154 * 4. Upon return caller flushes TLB's if needed.
2155 *
2156 * Between 2 and 4, the guest may write to the page using the remaining TLB
2157 * entry. This is not a problem because the page is reported dirty using
2158 * the snapshot taken before and step 4 ensures that writes done after
2159 * exiting to userspace will be logged for the next call.
2160 *
2161 */
kvm_get_dirty_log_protect(struct kvm * kvm,struct kvm_dirty_log * log)2162 static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log)
2163 {
2164 struct kvm_memslots *slots;
2165 struct kvm_memory_slot *memslot;
2166 int i, as_id, id;
2167 unsigned long n;
2168 unsigned long *dirty_bitmap;
2169 unsigned long *dirty_bitmap_buffer;
2170 bool flush;
2171
2172 /* Dirty ring tracking may be exclusive to dirty log tracking */
2173 if (!kvm_use_dirty_bitmap(kvm))
2174 return -ENXIO;
2175
2176 as_id = log->slot >> 16;
2177 id = (u16)log->slot;
2178 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
2179 return -EINVAL;
2180
2181 slots = __kvm_memslots(kvm, as_id);
2182 memslot = id_to_memslot(slots, id);
2183 if (!memslot || !memslot->dirty_bitmap)
2184 return -ENOENT;
2185
2186 dirty_bitmap = memslot->dirty_bitmap;
2187
2188 kvm_arch_sync_dirty_log(kvm, memslot);
2189
2190 n = kvm_dirty_bitmap_bytes(memslot);
2191 flush = false;
2192 if (kvm->manual_dirty_log_protect) {
2193 /*
2194 * Unlike kvm_get_dirty_log, we always return false in *flush,
2195 * because no flush is needed until KVM_CLEAR_DIRTY_LOG. There
2196 * is some code duplication between this function and
2197 * kvm_get_dirty_log, but hopefully all architecture
2198 * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
2199 * can be eliminated.
2200 */
2201 dirty_bitmap_buffer = dirty_bitmap;
2202 } else {
2203 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
2204 memset(dirty_bitmap_buffer, 0, n);
2205
2206 KVM_MMU_LOCK(kvm);
2207 for (i = 0; i < n / sizeof(long); i++) {
2208 unsigned long mask;
2209 gfn_t offset;
2210
2211 if (!dirty_bitmap[i])
2212 continue;
2213
2214 flush = true;
2215 mask = xchg(&dirty_bitmap[i], 0);
2216 dirty_bitmap_buffer[i] = mask;
2217
2218 offset = i * BITS_PER_LONG;
2219 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
2220 offset, mask);
2221 }
2222 KVM_MMU_UNLOCK(kvm);
2223 }
2224
2225 if (flush)
2226 kvm_flush_remote_tlbs_memslot(kvm, memslot);
2227
2228 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
2229 return -EFAULT;
2230 return 0;
2231 }
2232
2233
2234 /**
2235 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
2236 * @kvm: kvm instance
2237 * @log: slot id and address to which we copy the log
2238 *
2239 * Steps 1-4 below provide general overview of dirty page logging. See
2240 * kvm_get_dirty_log_protect() function description for additional details.
2241 *
2242 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
2243 * always flush the TLB (step 4) even if previous step failed and the dirty
2244 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
2245 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
2246 * writes will be marked dirty for next log read.
2247 *
2248 * 1. Take a snapshot of the bit and clear it if needed.
2249 * 2. Write protect the corresponding page.
2250 * 3. Copy the snapshot to the userspace.
2251 * 4. Flush TLB's if needed.
2252 */
kvm_vm_ioctl_get_dirty_log(struct kvm * kvm,struct kvm_dirty_log * log)2253 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
2254 struct kvm_dirty_log *log)
2255 {
2256 int r;
2257
2258 mutex_lock(&kvm->slots_lock);
2259
2260 r = kvm_get_dirty_log_protect(kvm, log);
2261
2262 mutex_unlock(&kvm->slots_lock);
2263 return r;
2264 }
2265
2266 /**
2267 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
2268 * and reenable dirty page tracking for the corresponding pages.
2269 * @kvm: pointer to kvm instance
2270 * @log: slot id and address from which to fetch the bitmap of dirty pages
2271 */
kvm_clear_dirty_log_protect(struct kvm * kvm,struct kvm_clear_dirty_log * log)2272 static int kvm_clear_dirty_log_protect(struct kvm *kvm,
2273 struct kvm_clear_dirty_log *log)
2274 {
2275 struct kvm_memslots *slots;
2276 struct kvm_memory_slot *memslot;
2277 int as_id, id;
2278 gfn_t offset;
2279 unsigned long i, n;
2280 unsigned long *dirty_bitmap;
2281 unsigned long *dirty_bitmap_buffer;
2282 bool flush;
2283
2284 /* Dirty ring tracking may be exclusive to dirty log tracking */
2285 if (!kvm_use_dirty_bitmap(kvm))
2286 return -ENXIO;
2287
2288 as_id = log->slot >> 16;
2289 id = (u16)log->slot;
2290 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
2291 return -EINVAL;
2292
2293 if (log->first_page & 63)
2294 return -EINVAL;
2295
2296 slots = __kvm_memslots(kvm, as_id);
2297 memslot = id_to_memslot(slots, id);
2298 if (!memslot || !memslot->dirty_bitmap)
2299 return -ENOENT;
2300
2301 dirty_bitmap = memslot->dirty_bitmap;
2302
2303 n = ALIGN(log->num_pages, BITS_PER_LONG) / 8;
2304
2305 if (log->first_page > memslot->npages ||
2306 log->num_pages > memslot->npages - log->first_page ||
2307 (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))
2308 return -EINVAL;
2309
2310 kvm_arch_sync_dirty_log(kvm, memslot);
2311
2312 flush = false;
2313 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
2314 if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
2315 return -EFAULT;
2316
2317 KVM_MMU_LOCK(kvm);
2318 for (offset = log->first_page, i = offset / BITS_PER_LONG,
2319 n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
2320 i++, offset += BITS_PER_LONG) {
2321 unsigned long mask = *dirty_bitmap_buffer++;
2322 atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
2323 if (!mask)
2324 continue;
2325
2326 mask &= atomic_long_fetch_andnot(mask, p);
2327
2328 /*
2329 * mask contains the bits that really have been cleared. This
2330 * never includes any bits beyond the length of the memslot (if
2331 * the length is not aligned to 64 pages), therefore it is not
2332 * a problem if userspace sets them in log->dirty_bitmap.
2333 */
2334 if (mask) {
2335 flush = true;
2336 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
2337 offset, mask);
2338 }
2339 }
2340 KVM_MMU_UNLOCK(kvm);
2341
2342 if (flush)
2343 kvm_flush_remote_tlbs_memslot(kvm, memslot);
2344
2345 return 0;
2346 }
2347
kvm_vm_ioctl_clear_dirty_log(struct kvm * kvm,struct kvm_clear_dirty_log * log)2348 static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,
2349 struct kvm_clear_dirty_log *log)
2350 {
2351 int r;
2352
2353 mutex_lock(&kvm->slots_lock);
2354
2355 r = kvm_clear_dirty_log_protect(kvm, log);
2356
2357 mutex_unlock(&kvm->slots_lock);
2358 return r;
2359 }
2360 #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
2361
2362 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
kvm_supported_mem_attributes(struct kvm * kvm)2363 static u64 kvm_supported_mem_attributes(struct kvm *kvm)
2364 {
2365 if (!kvm || kvm_arch_has_private_mem(kvm))
2366 return KVM_MEMORY_ATTRIBUTE_PRIVATE;
2367
2368 return 0;
2369 }
2370
2371 /*
2372 * Returns true if _all_ gfns in the range [@start, @end) have attributes
2373 * such that the bits in @mask match @attrs.
2374 */
kvm_range_has_memory_attributes(struct kvm * kvm,gfn_t start,gfn_t end,unsigned long mask,unsigned long attrs)2375 bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
2376 unsigned long mask, unsigned long attrs)
2377 {
2378 XA_STATE(xas, &kvm->mem_attr_array, start);
2379 unsigned long index;
2380 void *entry;
2381
2382 mask &= kvm_supported_mem_attributes(kvm);
2383 if (attrs & ~mask)
2384 return false;
2385
2386 if (end == start + 1)
2387 return (kvm_get_memory_attributes(kvm, start) & mask) == attrs;
2388
2389 guard(rcu)();
2390 if (!attrs)
2391 return !xas_find(&xas, end - 1);
2392
2393 for (index = start; index < end; index++) {
2394 do {
2395 entry = xas_next(&xas);
2396 } while (xas_retry(&xas, entry));
2397
2398 if (xas.xa_index != index ||
2399 (xa_to_value(entry) & mask) != attrs)
2400 return false;
2401 }
2402
2403 return true;
2404 }
2405
kvm_handle_gfn_range(struct kvm * kvm,struct kvm_mmu_notifier_range * range)2406 static __always_inline void kvm_handle_gfn_range(struct kvm *kvm,
2407 struct kvm_mmu_notifier_range *range)
2408 {
2409 struct kvm_gfn_range gfn_range;
2410 struct kvm_memory_slot *slot;
2411 struct kvm_memslots *slots;
2412 struct kvm_memslot_iter iter;
2413 bool found_memslot = false;
2414 bool ret = false;
2415 int i;
2416
2417 gfn_range.arg = range->arg;
2418 gfn_range.may_block = range->may_block;
2419
2420 /*
2421 * If/when KVM supports more attributes beyond private .vs shared, this
2422 * _could_ set KVM_FILTER_{SHARED,PRIVATE} appropriately if the entire target
2423 * range already has the desired private vs. shared state (it's unclear
2424 * if that is a net win). For now, KVM reaches this point if and only
2425 * if the private flag is being toggled, i.e. all mappings are in play.
2426 */
2427
2428 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
2429 slots = __kvm_memslots(kvm, i);
2430
2431 kvm_for_each_memslot_in_gfn_range(&iter, slots, range->start, range->end) {
2432 slot = iter.slot;
2433 gfn_range.slot = slot;
2434
2435 gfn_range.start = max(range->start, slot->base_gfn);
2436 gfn_range.end = min(range->end, slot->base_gfn + slot->npages);
2437 if (gfn_range.start >= gfn_range.end)
2438 continue;
2439
2440 if (!found_memslot) {
2441 found_memslot = true;
2442 KVM_MMU_LOCK(kvm);
2443 if (!IS_KVM_NULL_FN(range->on_lock))
2444 range->on_lock(kvm);
2445 }
2446
2447 ret |= range->handler(kvm, &gfn_range);
2448 }
2449 }
2450
2451 if (range->flush_on_ret && ret)
2452 kvm_flush_remote_tlbs(kvm);
2453
2454 if (found_memslot)
2455 KVM_MMU_UNLOCK(kvm);
2456 }
2457
kvm_pre_set_memory_attributes(struct kvm * kvm,struct kvm_gfn_range * range)2458 static bool kvm_pre_set_memory_attributes(struct kvm *kvm,
2459 struct kvm_gfn_range *range)
2460 {
2461 /*
2462 * Unconditionally add the range to the invalidation set, regardless of
2463 * whether or not the arch callback actually needs to zap SPTEs. E.g.
2464 * if KVM supports RWX attributes in the future and the attributes are
2465 * going from R=>RW, zapping isn't strictly necessary. Unconditionally
2466 * adding the range allows KVM to require that MMU invalidations add at
2467 * least one range between begin() and end(), e.g. allows KVM to detect
2468 * bugs where the add() is missed. Relaxing the rule *might* be safe,
2469 * but it's not obvious that allowing new mappings while the attributes
2470 * are in flux is desirable or worth the complexity.
2471 */
2472 kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
2473
2474 return kvm_arch_pre_set_memory_attributes(kvm, range);
2475 }
2476
2477 /* Set @attributes for the gfn range [@start, @end). */
kvm_vm_set_mem_attributes(struct kvm * kvm,gfn_t start,gfn_t end,unsigned long attributes)2478 static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
2479 unsigned long attributes)
2480 {
2481 struct kvm_mmu_notifier_range pre_set_range = {
2482 .start = start,
2483 .end = end,
2484 .arg.attributes = attributes,
2485 .handler = kvm_pre_set_memory_attributes,
2486 .on_lock = kvm_mmu_invalidate_begin,
2487 .flush_on_ret = true,
2488 .may_block = true,
2489 };
2490 struct kvm_mmu_notifier_range post_set_range = {
2491 .start = start,
2492 .end = end,
2493 .arg.attributes = attributes,
2494 .handler = kvm_arch_post_set_memory_attributes,
2495 .on_lock = kvm_mmu_invalidate_end,
2496 .may_block = true,
2497 };
2498 unsigned long i;
2499 void *entry;
2500 int r = 0;
2501
2502 entry = attributes ? xa_mk_value(attributes) : NULL;
2503
2504 mutex_lock(&kvm->slots_lock);
2505
2506 /* Nothing to do if the entire range as the desired attributes. */
2507 if (kvm_range_has_memory_attributes(kvm, start, end, ~0, attributes))
2508 goto out_unlock;
2509
2510 /*
2511 * Reserve memory ahead of time to avoid having to deal with failures
2512 * partway through setting the new attributes.
2513 */
2514 for (i = start; i < end; i++) {
2515 r = xa_reserve(&kvm->mem_attr_array, i, GFP_KERNEL_ACCOUNT);
2516 if (r)
2517 goto out_unlock;
2518 }
2519
2520 kvm_handle_gfn_range(kvm, &pre_set_range);
2521
2522 for (i = start; i < end; i++) {
2523 r = xa_err(xa_store(&kvm->mem_attr_array, i, entry,
2524 GFP_KERNEL_ACCOUNT));
2525 KVM_BUG_ON(r, kvm);
2526 }
2527
2528 kvm_handle_gfn_range(kvm, &post_set_range);
2529
2530 out_unlock:
2531 mutex_unlock(&kvm->slots_lock);
2532
2533 return r;
2534 }
kvm_vm_ioctl_set_mem_attributes(struct kvm * kvm,struct kvm_memory_attributes * attrs)2535 static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm,
2536 struct kvm_memory_attributes *attrs)
2537 {
2538 gfn_t start, end;
2539
2540 /* flags is currently not used. */
2541 if (attrs->flags)
2542 return -EINVAL;
2543 if (attrs->attributes & ~kvm_supported_mem_attributes(kvm))
2544 return -EINVAL;
2545 if (attrs->size == 0 || attrs->address + attrs->size < attrs->address)
2546 return -EINVAL;
2547 if (!PAGE_ALIGNED(attrs->address) || !PAGE_ALIGNED(attrs->size))
2548 return -EINVAL;
2549
2550 start = attrs->address >> PAGE_SHIFT;
2551 end = (attrs->address + attrs->size) >> PAGE_SHIFT;
2552
2553 /*
2554 * xarray tracks data using "unsigned long", and as a result so does
2555 * KVM. For simplicity, supports generic attributes only on 64-bit
2556 * architectures.
2557 */
2558 BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long));
2559
2560 return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes);
2561 }
2562 #endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
2563
gfn_to_memslot(struct kvm * kvm,gfn_t gfn)2564 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
2565 {
2566 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
2567 }
2568 EXPORT_SYMBOL_GPL(gfn_to_memslot);
2569
kvm_vcpu_gfn_to_memslot(struct kvm_vcpu * vcpu,gfn_t gfn)2570 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
2571 {
2572 struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu);
2573 u64 gen = slots->generation;
2574 struct kvm_memory_slot *slot;
2575
2576 /*
2577 * This also protects against using a memslot from a different address space,
2578 * since different address spaces have different generation numbers.
2579 */
2580 if (unlikely(gen != vcpu->last_used_slot_gen)) {
2581 vcpu->last_used_slot = NULL;
2582 vcpu->last_used_slot_gen = gen;
2583 }
2584
2585 slot = try_get_memslot(vcpu->last_used_slot, gfn);
2586 if (slot)
2587 return slot;
2588
2589 /*
2590 * Fall back to searching all memslots. We purposely use
2591 * search_memslots() instead of __gfn_to_memslot() to avoid
2592 * thrashing the VM-wide last_used_slot in kvm_memslots.
2593 */
2594 slot = search_memslots(slots, gfn, false);
2595 if (slot) {
2596 vcpu->last_used_slot = slot;
2597 return slot;
2598 }
2599
2600 return NULL;
2601 }
2602
kvm_is_visible_gfn(struct kvm * kvm,gfn_t gfn)2603 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
2604 {
2605 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
2606
2607 return kvm_is_visible_memslot(memslot);
2608 }
2609 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
2610
kvm_vcpu_is_visible_gfn(struct kvm_vcpu * vcpu,gfn_t gfn)2611 bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
2612 {
2613 struct kvm_memory_slot *memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2614
2615 return kvm_is_visible_memslot(memslot);
2616 }
2617 EXPORT_SYMBOL_GPL(kvm_vcpu_is_visible_gfn);
2618
kvm_host_page_size(struct kvm_vcpu * vcpu,gfn_t gfn)2619 unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn)
2620 {
2621 struct vm_area_struct *vma;
2622 unsigned long addr, size;
2623
2624 size = PAGE_SIZE;
2625
2626 addr = kvm_vcpu_gfn_to_hva_prot(vcpu, gfn, NULL);
2627 if (kvm_is_error_hva(addr))
2628 return PAGE_SIZE;
2629
2630 mmap_read_lock(current->mm);
2631 vma = find_vma(current->mm, addr);
2632 if (!vma)
2633 goto out;
2634
2635 size = vma_kernel_pagesize(vma);
2636
2637 out:
2638 mmap_read_unlock(current->mm);
2639
2640 return size;
2641 }
2642
memslot_is_readonly(const struct kvm_memory_slot * slot)2643 static bool memslot_is_readonly(const struct kvm_memory_slot *slot)
2644 {
2645 return slot->flags & KVM_MEM_READONLY;
2646 }
2647
__gfn_to_hva_many(const struct kvm_memory_slot * slot,gfn_t gfn,gfn_t * nr_pages,bool write)2648 static unsigned long __gfn_to_hva_many(const struct kvm_memory_slot *slot, gfn_t gfn,
2649 gfn_t *nr_pages, bool write)
2650 {
2651 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
2652 return KVM_HVA_ERR_BAD;
2653
2654 if (memslot_is_readonly(slot) && write)
2655 return KVM_HVA_ERR_RO_BAD;
2656
2657 if (nr_pages)
2658 *nr_pages = slot->npages - (gfn - slot->base_gfn);
2659
2660 return __gfn_to_hva_memslot(slot, gfn);
2661 }
2662
gfn_to_hva_many(struct kvm_memory_slot * slot,gfn_t gfn,gfn_t * nr_pages)2663 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
2664 gfn_t *nr_pages)
2665 {
2666 return __gfn_to_hva_many(slot, gfn, nr_pages, true);
2667 }
2668
gfn_to_hva_memslot(struct kvm_memory_slot * slot,gfn_t gfn)2669 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
2670 gfn_t gfn)
2671 {
2672 return gfn_to_hva_many(slot, gfn, NULL);
2673 }
2674 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
2675
gfn_to_hva(struct kvm * kvm,gfn_t gfn)2676 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
2677 {
2678 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
2679 }
2680 EXPORT_SYMBOL_GPL(gfn_to_hva);
2681
kvm_vcpu_gfn_to_hva(struct kvm_vcpu * vcpu,gfn_t gfn)2682 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
2683 {
2684 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
2685 }
2686 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
2687
2688 /*
2689 * Return the hva of a @gfn and the R/W attribute if possible.
2690 *
2691 * @slot: the kvm_memory_slot which contains @gfn
2692 * @gfn: the gfn to be translated
2693 * @writable: used to return the read/write attribute of the @slot if the hva
2694 * is valid and @writable is not NULL
2695 */
gfn_to_hva_memslot_prot(struct kvm_memory_slot * slot,gfn_t gfn,bool * writable)2696 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
2697 gfn_t gfn, bool *writable)
2698 {
2699 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
2700
2701 if (!kvm_is_error_hva(hva) && writable)
2702 *writable = !memslot_is_readonly(slot);
2703
2704 return hva;
2705 }
2706
gfn_to_hva_prot(struct kvm * kvm,gfn_t gfn,bool * writable)2707 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
2708 {
2709 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
2710
2711 return gfn_to_hva_memslot_prot(slot, gfn, writable);
2712 }
2713
kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu * vcpu,gfn_t gfn,bool * writable)2714 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
2715 {
2716 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2717
2718 return gfn_to_hva_memslot_prot(slot, gfn, writable);
2719 }
2720
kvm_is_ad_tracked_page(struct page * page)2721 static bool kvm_is_ad_tracked_page(struct page *page)
2722 {
2723 /*
2724 * Per page-flags.h, pages tagged PG_reserved "should in general not be
2725 * touched (e.g. set dirty) except by its owner".
2726 */
2727 return !PageReserved(page);
2728 }
2729
kvm_set_page_dirty(struct page * page)2730 static void kvm_set_page_dirty(struct page *page)
2731 {
2732 if (kvm_is_ad_tracked_page(page))
2733 SetPageDirty(page);
2734 }
2735
kvm_set_page_accessed(struct page * page)2736 static void kvm_set_page_accessed(struct page *page)
2737 {
2738 if (kvm_is_ad_tracked_page(page))
2739 mark_page_accessed(page);
2740 }
2741
kvm_release_page_clean(struct page * page)2742 void kvm_release_page_clean(struct page *page)
2743 {
2744 if (!page)
2745 return;
2746
2747 kvm_set_page_accessed(page);
2748 put_page(page);
2749 }
2750 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
2751
kvm_release_page_dirty(struct page * page)2752 void kvm_release_page_dirty(struct page *page)
2753 {
2754 if (!page)
2755 return;
2756
2757 kvm_set_page_dirty(page);
2758 kvm_release_page_clean(page);
2759 }
2760 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
2761
kvm_resolve_pfn(struct kvm_follow_pfn * kfp,struct page * page,struct follow_pfnmap_args * map,bool writable)2762 static kvm_pfn_t kvm_resolve_pfn(struct kvm_follow_pfn *kfp, struct page *page,
2763 struct follow_pfnmap_args *map, bool writable)
2764 {
2765 kvm_pfn_t pfn;
2766
2767 WARN_ON_ONCE(!!page == !!map);
2768
2769 if (kfp->map_writable)
2770 *kfp->map_writable = writable;
2771
2772 if (map)
2773 pfn = map->pfn;
2774 else
2775 pfn = page_to_pfn(page);
2776
2777 *kfp->refcounted_page = page;
2778
2779 return pfn;
2780 }
2781
2782 /*
2783 * The fast path to get the writable pfn which will be stored in @pfn,
2784 * true indicates success, otherwise false is returned.
2785 */
hva_to_pfn_fast(struct kvm_follow_pfn * kfp,kvm_pfn_t * pfn)2786 static bool hva_to_pfn_fast(struct kvm_follow_pfn *kfp, kvm_pfn_t *pfn)
2787 {
2788 struct page *page;
2789 bool r;
2790
2791 /*
2792 * Try the fast-only path when the caller wants to pin/get the page for
2793 * writing. If the caller only wants to read the page, KVM must go
2794 * down the full, slow path in order to avoid racing an operation that
2795 * breaks Copy-on-Write (CoW), e.g. so that KVM doesn't end up pointing
2796 * at the old, read-only page while mm/ points at a new, writable page.
2797 */
2798 if (!((kfp->flags & FOLL_WRITE) || kfp->map_writable))
2799 return false;
2800
2801 if (kfp->pin)
2802 r = pin_user_pages_fast(kfp->hva, 1, FOLL_WRITE, &page) == 1;
2803 else
2804 r = get_user_page_fast_only(kfp->hva, FOLL_WRITE, &page);
2805
2806 if (r) {
2807 *pfn = kvm_resolve_pfn(kfp, page, NULL, true);
2808 return true;
2809 }
2810
2811 return false;
2812 }
2813
2814 /*
2815 * The slow path to get the pfn of the specified host virtual address,
2816 * 1 indicates success, -errno is returned if error is detected.
2817 */
hva_to_pfn_slow(struct kvm_follow_pfn * kfp,kvm_pfn_t * pfn)2818 static int hva_to_pfn_slow(struct kvm_follow_pfn *kfp, kvm_pfn_t *pfn)
2819 {
2820 /*
2821 * When a VCPU accesses a page that is not mapped into the secondary
2822 * MMU, we lookup the page using GUP to map it, so the guest VCPU can
2823 * make progress. We always want to honor NUMA hinting faults in that
2824 * case, because GUP usage corresponds to memory accesses from the VCPU.
2825 * Otherwise, we'd not trigger NUMA hinting faults once a page is
2826 * mapped into the secondary MMU and gets accessed by a VCPU.
2827 *
2828 * Note that get_user_page_fast_only() and FOLL_WRITE for now
2829 * implicitly honor NUMA hinting faults and don't need this flag.
2830 */
2831 unsigned int flags = FOLL_HWPOISON | FOLL_HONOR_NUMA_FAULT | kfp->flags;
2832 struct page *page, *wpage;
2833 int npages;
2834
2835 if (kfp->pin)
2836 npages = pin_user_pages_unlocked(kfp->hva, 1, &page, flags);
2837 else
2838 npages = get_user_pages_unlocked(kfp->hva, 1, &page, flags);
2839 if (npages != 1)
2840 return npages;
2841
2842 /*
2843 * Pinning is mutually exclusive with opportunistically mapping a read
2844 * fault as writable, as KVM should never pin pages when mapping memory
2845 * into the guest (pinning is only for direct accesses from KVM).
2846 */
2847 if (WARN_ON_ONCE(kfp->map_writable && kfp->pin))
2848 goto out;
2849
2850 /* map read fault as writable if possible */
2851 if (!(flags & FOLL_WRITE) && kfp->map_writable &&
2852 get_user_page_fast_only(kfp->hva, FOLL_WRITE, &wpage)) {
2853 put_page(page);
2854 page = wpage;
2855 flags |= FOLL_WRITE;
2856 }
2857
2858 out:
2859 *pfn = kvm_resolve_pfn(kfp, page, NULL, flags & FOLL_WRITE);
2860 return npages;
2861 }
2862
vma_is_valid(struct vm_area_struct * vma,bool write_fault)2863 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
2864 {
2865 if (unlikely(!(vma->vm_flags & VM_READ)))
2866 return false;
2867
2868 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
2869 return false;
2870
2871 return true;
2872 }
2873
hva_to_pfn_remapped(struct vm_area_struct * vma,struct kvm_follow_pfn * kfp,kvm_pfn_t * p_pfn)2874 static int hva_to_pfn_remapped(struct vm_area_struct *vma,
2875 struct kvm_follow_pfn *kfp, kvm_pfn_t *p_pfn)
2876 {
2877 struct follow_pfnmap_args args = { .vma = vma, .address = kfp->hva };
2878 bool write_fault = kfp->flags & FOLL_WRITE;
2879 int r;
2880
2881 /*
2882 * Remapped memory cannot be pinned in any meaningful sense. Bail if
2883 * the caller wants to pin the page, i.e. access the page outside of
2884 * MMU notifier protection, and unsafe umappings are disallowed.
2885 */
2886 if (kfp->pin && !allow_unsafe_mappings)
2887 return -EINVAL;
2888
2889 r = follow_pfnmap_start(&args);
2890 if (r) {
2891 /*
2892 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
2893 * not call the fault handler, so do it here.
2894 */
2895 bool unlocked = false;
2896 r = fixup_user_fault(current->mm, kfp->hva,
2897 (write_fault ? FAULT_FLAG_WRITE : 0),
2898 &unlocked);
2899 if (unlocked)
2900 return -EAGAIN;
2901 if (r)
2902 return r;
2903
2904 r = follow_pfnmap_start(&args);
2905 if (r)
2906 return r;
2907 }
2908
2909 if (write_fault && !args.writable) {
2910 *p_pfn = KVM_PFN_ERR_RO_FAULT;
2911 goto out;
2912 }
2913
2914 *p_pfn = kvm_resolve_pfn(kfp, NULL, &args, args.writable);
2915 out:
2916 follow_pfnmap_end(&args);
2917 return r;
2918 }
2919
hva_to_pfn(struct kvm_follow_pfn * kfp)2920 kvm_pfn_t hva_to_pfn(struct kvm_follow_pfn *kfp)
2921 {
2922 struct vm_area_struct *vma;
2923 kvm_pfn_t pfn;
2924 int npages, r;
2925
2926 might_sleep();
2927
2928 if (WARN_ON_ONCE(!kfp->refcounted_page))
2929 return KVM_PFN_ERR_FAULT;
2930
2931 if (hva_to_pfn_fast(kfp, &pfn))
2932 return pfn;
2933
2934 npages = hva_to_pfn_slow(kfp, &pfn);
2935 if (npages == 1)
2936 return pfn;
2937 if (npages == -EINTR || npages == -EAGAIN)
2938 return KVM_PFN_ERR_SIGPENDING;
2939 if (npages == -EHWPOISON)
2940 return KVM_PFN_ERR_HWPOISON;
2941
2942 mmap_read_lock(current->mm);
2943 retry:
2944 vma = vma_lookup(current->mm, kfp->hva);
2945
2946 if (vma == NULL)
2947 pfn = KVM_PFN_ERR_FAULT;
2948 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
2949 r = hva_to_pfn_remapped(vma, kfp, &pfn);
2950 if (r == -EAGAIN)
2951 goto retry;
2952 if (r < 0)
2953 pfn = KVM_PFN_ERR_FAULT;
2954 } else {
2955 if ((kfp->flags & FOLL_NOWAIT) &&
2956 vma_is_valid(vma, kfp->flags & FOLL_WRITE))
2957 pfn = KVM_PFN_ERR_NEEDS_IO;
2958 else
2959 pfn = KVM_PFN_ERR_FAULT;
2960 }
2961 mmap_read_unlock(current->mm);
2962 return pfn;
2963 }
2964
kvm_follow_pfn(struct kvm_follow_pfn * kfp)2965 static kvm_pfn_t kvm_follow_pfn(struct kvm_follow_pfn *kfp)
2966 {
2967 kfp->hva = __gfn_to_hva_many(kfp->slot, kfp->gfn, NULL,
2968 kfp->flags & FOLL_WRITE);
2969
2970 if (kfp->hva == KVM_HVA_ERR_RO_BAD)
2971 return KVM_PFN_ERR_RO_FAULT;
2972
2973 if (kvm_is_error_hva(kfp->hva))
2974 return KVM_PFN_NOSLOT;
2975
2976 if (memslot_is_readonly(kfp->slot) && kfp->map_writable) {
2977 *kfp->map_writable = false;
2978 kfp->map_writable = NULL;
2979 }
2980
2981 return hva_to_pfn(kfp);
2982 }
2983
__kvm_faultin_pfn(const struct kvm_memory_slot * slot,gfn_t gfn,unsigned int foll,bool * writable,struct page ** refcounted_page)2984 kvm_pfn_t __kvm_faultin_pfn(const struct kvm_memory_slot *slot, gfn_t gfn,
2985 unsigned int foll, bool *writable,
2986 struct page **refcounted_page)
2987 {
2988 struct kvm_follow_pfn kfp = {
2989 .slot = slot,
2990 .gfn = gfn,
2991 .flags = foll,
2992 .map_writable = writable,
2993 .refcounted_page = refcounted_page,
2994 };
2995
2996 if (WARN_ON_ONCE(!writable || !refcounted_page))
2997 return KVM_PFN_ERR_FAULT;
2998
2999 *writable = false;
3000 *refcounted_page = NULL;
3001
3002 return kvm_follow_pfn(&kfp);
3003 }
3004 EXPORT_SYMBOL_GPL(__kvm_faultin_pfn);
3005
kvm_prefetch_pages(struct kvm_memory_slot * slot,gfn_t gfn,struct page ** pages,int nr_pages)3006 int kvm_prefetch_pages(struct kvm_memory_slot *slot, gfn_t gfn,
3007 struct page **pages, int nr_pages)
3008 {
3009 unsigned long addr;
3010 gfn_t entry = 0;
3011
3012 addr = gfn_to_hva_many(slot, gfn, &entry);
3013 if (kvm_is_error_hva(addr))
3014 return -1;
3015
3016 if (entry < nr_pages)
3017 return 0;
3018
3019 return get_user_pages_fast_only(addr, nr_pages, FOLL_WRITE, pages);
3020 }
3021 EXPORT_SYMBOL_GPL(kvm_prefetch_pages);
3022
3023 /*
3024 * Don't use this API unless you are absolutely, positively certain that KVM
3025 * needs to get a struct page, e.g. to pin the page for firmware DMA.
3026 *
3027 * FIXME: Users of this API likely need to FOLL_PIN the page, not just elevate
3028 * its refcount.
3029 */
__gfn_to_page(struct kvm * kvm,gfn_t gfn,bool write)3030 struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn, bool write)
3031 {
3032 struct page *refcounted_page = NULL;
3033 struct kvm_follow_pfn kfp = {
3034 .slot = gfn_to_memslot(kvm, gfn),
3035 .gfn = gfn,
3036 .flags = write ? FOLL_WRITE : 0,
3037 .refcounted_page = &refcounted_page,
3038 };
3039
3040 (void)kvm_follow_pfn(&kfp);
3041 return refcounted_page;
3042 }
3043 EXPORT_SYMBOL_GPL(__gfn_to_page);
3044
__kvm_vcpu_map(struct kvm_vcpu * vcpu,gfn_t gfn,struct kvm_host_map * map,bool writable)3045 int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map,
3046 bool writable)
3047 {
3048 struct kvm_follow_pfn kfp = {
3049 .slot = gfn_to_memslot(vcpu->kvm, gfn),
3050 .gfn = gfn,
3051 .flags = writable ? FOLL_WRITE : 0,
3052 .refcounted_page = &map->pinned_page,
3053 .pin = true,
3054 };
3055
3056 map->pinned_page = NULL;
3057 map->page = NULL;
3058 map->hva = NULL;
3059 map->gfn = gfn;
3060 map->writable = writable;
3061
3062 map->pfn = kvm_follow_pfn(&kfp);
3063 if (is_error_noslot_pfn(map->pfn))
3064 return -EINVAL;
3065
3066 if (pfn_valid(map->pfn)) {
3067 map->page = pfn_to_page(map->pfn);
3068 map->hva = kmap(map->page);
3069 #ifdef CONFIG_HAS_IOMEM
3070 } else {
3071 map->hva = memremap(pfn_to_hpa(map->pfn), PAGE_SIZE, MEMREMAP_WB);
3072 #endif
3073 }
3074
3075 return map->hva ? 0 : -EFAULT;
3076 }
3077 EXPORT_SYMBOL_GPL(__kvm_vcpu_map);
3078
kvm_vcpu_unmap(struct kvm_vcpu * vcpu,struct kvm_host_map * map)3079 void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map)
3080 {
3081 if (!map->hva)
3082 return;
3083
3084 if (map->page)
3085 kunmap(map->page);
3086 #ifdef CONFIG_HAS_IOMEM
3087 else
3088 memunmap(map->hva);
3089 #endif
3090
3091 if (map->writable)
3092 kvm_vcpu_mark_page_dirty(vcpu, map->gfn);
3093
3094 if (map->pinned_page) {
3095 if (map->writable)
3096 kvm_set_page_dirty(map->pinned_page);
3097 kvm_set_page_accessed(map->pinned_page);
3098 unpin_user_page(map->pinned_page);
3099 }
3100
3101 map->hva = NULL;
3102 map->page = NULL;
3103 map->pinned_page = NULL;
3104 }
3105 EXPORT_SYMBOL_GPL(kvm_vcpu_unmap);
3106
next_segment(unsigned long len,int offset)3107 static int next_segment(unsigned long len, int offset)
3108 {
3109 if (len > PAGE_SIZE - offset)
3110 return PAGE_SIZE - offset;
3111 else
3112 return len;
3113 }
3114
3115 /* Copy @len bytes from guest memory at '(@gfn * PAGE_SIZE) + @offset' to @data */
__kvm_read_guest_page(struct kvm_memory_slot * slot,gfn_t gfn,void * data,int offset,int len)3116 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
3117 void *data, int offset, int len)
3118 {
3119 int r;
3120 unsigned long addr;
3121
3122 if (WARN_ON_ONCE(offset + len > PAGE_SIZE))
3123 return -EFAULT;
3124
3125 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
3126 if (kvm_is_error_hva(addr))
3127 return -EFAULT;
3128 r = __copy_from_user(data, (void __user *)addr + offset, len);
3129 if (r)
3130 return -EFAULT;
3131 return 0;
3132 }
3133
kvm_read_guest_page(struct kvm * kvm,gfn_t gfn,void * data,int offset,int len)3134 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
3135 int len)
3136 {
3137 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
3138
3139 return __kvm_read_guest_page(slot, gfn, data, offset, len);
3140 }
3141 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
3142
kvm_vcpu_read_guest_page(struct kvm_vcpu * vcpu,gfn_t gfn,void * data,int offset,int len)3143 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
3144 int offset, int len)
3145 {
3146 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3147
3148 return __kvm_read_guest_page(slot, gfn, data, offset, len);
3149 }
3150 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
3151
kvm_read_guest(struct kvm * kvm,gpa_t gpa,void * data,unsigned long len)3152 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
3153 {
3154 gfn_t gfn = gpa >> PAGE_SHIFT;
3155 int seg;
3156 int offset = offset_in_page(gpa);
3157 int ret;
3158
3159 while ((seg = next_segment(len, offset)) != 0) {
3160 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
3161 if (ret < 0)
3162 return ret;
3163 offset = 0;
3164 len -= seg;
3165 data += seg;
3166 ++gfn;
3167 }
3168 return 0;
3169 }
3170 EXPORT_SYMBOL_GPL(kvm_read_guest);
3171
kvm_vcpu_read_guest(struct kvm_vcpu * vcpu,gpa_t gpa,void * data,unsigned long len)3172 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
3173 {
3174 gfn_t gfn = gpa >> PAGE_SHIFT;
3175 int seg;
3176 int offset = offset_in_page(gpa);
3177 int ret;
3178
3179 while ((seg = next_segment(len, offset)) != 0) {
3180 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
3181 if (ret < 0)
3182 return ret;
3183 offset = 0;
3184 len -= seg;
3185 data += seg;
3186 ++gfn;
3187 }
3188 return 0;
3189 }
3190 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
3191
__kvm_read_guest_atomic(struct kvm_memory_slot * slot,gfn_t gfn,void * data,int offset,unsigned long len)3192 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
3193 void *data, int offset, unsigned long len)
3194 {
3195 int r;
3196 unsigned long addr;
3197
3198 if (WARN_ON_ONCE(offset + len > PAGE_SIZE))
3199 return -EFAULT;
3200
3201 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
3202 if (kvm_is_error_hva(addr))
3203 return -EFAULT;
3204 pagefault_disable();
3205 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
3206 pagefault_enable();
3207 if (r)
3208 return -EFAULT;
3209 return 0;
3210 }
3211
kvm_vcpu_read_guest_atomic(struct kvm_vcpu * vcpu,gpa_t gpa,void * data,unsigned long len)3212 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
3213 void *data, unsigned long len)
3214 {
3215 gfn_t gfn = gpa >> PAGE_SHIFT;
3216 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3217 int offset = offset_in_page(gpa);
3218
3219 return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
3220 }
3221 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
3222
3223 /* Copy @len bytes from @data into guest memory at '(@gfn * PAGE_SIZE) + @offset' */
__kvm_write_guest_page(struct kvm * kvm,struct kvm_memory_slot * memslot,gfn_t gfn,const void * data,int offset,int len)3224 static int __kvm_write_guest_page(struct kvm *kvm,
3225 struct kvm_memory_slot *memslot, gfn_t gfn,
3226 const void *data, int offset, int len)
3227 {
3228 int r;
3229 unsigned long addr;
3230
3231 if (WARN_ON_ONCE(offset + len > PAGE_SIZE))
3232 return -EFAULT;
3233
3234 addr = gfn_to_hva_memslot(memslot, gfn);
3235 if (kvm_is_error_hva(addr))
3236 return -EFAULT;
3237 r = __copy_to_user((void __user *)addr + offset, data, len);
3238 if (r)
3239 return -EFAULT;
3240 mark_page_dirty_in_slot(kvm, memslot, gfn);
3241 return 0;
3242 }
3243
kvm_write_guest_page(struct kvm * kvm,gfn_t gfn,const void * data,int offset,int len)3244 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
3245 const void *data, int offset, int len)
3246 {
3247 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
3248
3249 return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len);
3250 }
3251 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
3252
kvm_vcpu_write_guest_page(struct kvm_vcpu * vcpu,gfn_t gfn,const void * data,int offset,int len)3253 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
3254 const void *data, int offset, int len)
3255 {
3256 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3257
3258 return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len);
3259 }
3260 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
3261
kvm_write_guest(struct kvm * kvm,gpa_t gpa,const void * data,unsigned long len)3262 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
3263 unsigned long len)
3264 {
3265 gfn_t gfn = gpa >> PAGE_SHIFT;
3266 int seg;
3267 int offset = offset_in_page(gpa);
3268 int ret;
3269
3270 while ((seg = next_segment(len, offset)) != 0) {
3271 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
3272 if (ret < 0)
3273 return ret;
3274 offset = 0;
3275 len -= seg;
3276 data += seg;
3277 ++gfn;
3278 }
3279 return 0;
3280 }
3281 EXPORT_SYMBOL_GPL(kvm_write_guest);
3282
kvm_vcpu_write_guest(struct kvm_vcpu * vcpu,gpa_t gpa,const void * data,unsigned long len)3283 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
3284 unsigned long len)
3285 {
3286 gfn_t gfn = gpa >> PAGE_SHIFT;
3287 int seg;
3288 int offset = offset_in_page(gpa);
3289 int ret;
3290
3291 while ((seg = next_segment(len, offset)) != 0) {
3292 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
3293 if (ret < 0)
3294 return ret;
3295 offset = 0;
3296 len -= seg;
3297 data += seg;
3298 ++gfn;
3299 }
3300 return 0;
3301 }
3302 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
3303
__kvm_gfn_to_hva_cache_init(struct kvm_memslots * slots,struct gfn_to_hva_cache * ghc,gpa_t gpa,unsigned long len)3304 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
3305 struct gfn_to_hva_cache *ghc,
3306 gpa_t gpa, unsigned long len)
3307 {
3308 int offset = offset_in_page(gpa);
3309 gfn_t start_gfn = gpa >> PAGE_SHIFT;
3310 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
3311 gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
3312 gfn_t nr_pages_avail;
3313
3314 /* Update ghc->generation before performing any error checks. */
3315 ghc->generation = slots->generation;
3316
3317 if (start_gfn > end_gfn) {
3318 ghc->hva = KVM_HVA_ERR_BAD;
3319 return -EINVAL;
3320 }
3321
3322 /*
3323 * If the requested region crosses two memslots, we still
3324 * verify that the entire region is valid here.
3325 */
3326 for ( ; start_gfn <= end_gfn; start_gfn += nr_pages_avail) {
3327 ghc->memslot = __gfn_to_memslot(slots, start_gfn);
3328 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
3329 &nr_pages_avail);
3330 if (kvm_is_error_hva(ghc->hva))
3331 return -EFAULT;
3332 }
3333
3334 /* Use the slow path for cross page reads and writes. */
3335 if (nr_pages_needed == 1)
3336 ghc->hva += offset;
3337 else
3338 ghc->memslot = NULL;
3339
3340 ghc->gpa = gpa;
3341 ghc->len = len;
3342 return 0;
3343 }
3344
kvm_gfn_to_hva_cache_init(struct kvm * kvm,struct gfn_to_hva_cache * ghc,gpa_t gpa,unsigned long len)3345 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3346 gpa_t gpa, unsigned long len)
3347 {
3348 struct kvm_memslots *slots = kvm_memslots(kvm);
3349 return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
3350 }
3351 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
3352
kvm_write_guest_offset_cached(struct kvm * kvm,struct gfn_to_hva_cache * ghc,void * data,unsigned int offset,unsigned long len)3353 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3354 void *data, unsigned int offset,
3355 unsigned long len)
3356 {
3357 struct kvm_memslots *slots = kvm_memslots(kvm);
3358 int r;
3359 gpa_t gpa = ghc->gpa + offset;
3360
3361 if (WARN_ON_ONCE(len + offset > ghc->len))
3362 return -EINVAL;
3363
3364 if (slots->generation != ghc->generation) {
3365 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
3366 return -EFAULT;
3367 }
3368
3369 if (kvm_is_error_hva(ghc->hva))
3370 return -EFAULT;
3371
3372 if (unlikely(!ghc->memslot))
3373 return kvm_write_guest(kvm, gpa, data, len);
3374
3375 r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
3376 if (r)
3377 return -EFAULT;
3378 mark_page_dirty_in_slot(kvm, ghc->memslot, gpa >> PAGE_SHIFT);
3379
3380 return 0;
3381 }
3382 EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
3383
kvm_write_guest_cached(struct kvm * kvm,struct gfn_to_hva_cache * ghc,void * data,unsigned long len)3384 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3385 void *data, unsigned long len)
3386 {
3387 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
3388 }
3389 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
3390
kvm_read_guest_offset_cached(struct kvm * kvm,struct gfn_to_hva_cache * ghc,void * data,unsigned int offset,unsigned long len)3391 int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3392 void *data, unsigned int offset,
3393 unsigned long len)
3394 {
3395 struct kvm_memslots *slots = kvm_memslots(kvm);
3396 int r;
3397 gpa_t gpa = ghc->gpa + offset;
3398
3399 if (WARN_ON_ONCE(len + offset > ghc->len))
3400 return -EINVAL;
3401
3402 if (slots->generation != ghc->generation) {
3403 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
3404 return -EFAULT;
3405 }
3406
3407 if (kvm_is_error_hva(ghc->hva))
3408 return -EFAULT;
3409
3410 if (unlikely(!ghc->memslot))
3411 return kvm_read_guest(kvm, gpa, data, len);
3412
3413 r = __copy_from_user(data, (void __user *)ghc->hva + offset, len);
3414 if (r)
3415 return -EFAULT;
3416
3417 return 0;
3418 }
3419 EXPORT_SYMBOL_GPL(kvm_read_guest_offset_cached);
3420
kvm_read_guest_cached(struct kvm * kvm,struct gfn_to_hva_cache * ghc,void * data,unsigned long len)3421 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3422 void *data, unsigned long len)
3423 {
3424 return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len);
3425 }
3426 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
3427
kvm_clear_guest(struct kvm * kvm,gpa_t gpa,unsigned long len)3428 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
3429 {
3430 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
3431 gfn_t gfn = gpa >> PAGE_SHIFT;
3432 int seg;
3433 int offset = offset_in_page(gpa);
3434 int ret;
3435
3436 while ((seg = next_segment(len, offset)) != 0) {
3437 ret = kvm_write_guest_page(kvm, gfn, zero_page, offset, seg);
3438 if (ret < 0)
3439 return ret;
3440 offset = 0;
3441 len -= seg;
3442 ++gfn;
3443 }
3444 return 0;
3445 }
3446 EXPORT_SYMBOL_GPL(kvm_clear_guest);
3447
mark_page_dirty_in_slot(struct kvm * kvm,const struct kvm_memory_slot * memslot,gfn_t gfn)3448 void mark_page_dirty_in_slot(struct kvm *kvm,
3449 const struct kvm_memory_slot *memslot,
3450 gfn_t gfn)
3451 {
3452 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
3453
3454 #ifdef CONFIG_HAVE_KVM_DIRTY_RING
3455 if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm))
3456 return;
3457
3458 WARN_ON_ONCE(!vcpu && !kvm_arch_allow_write_without_running_vcpu(kvm));
3459 #endif
3460
3461 if (memslot && kvm_slot_dirty_track_enabled(memslot)) {
3462 unsigned long rel_gfn = gfn - memslot->base_gfn;
3463 u32 slot = (memslot->as_id << 16) | memslot->id;
3464
3465 if (kvm->dirty_ring_size && vcpu)
3466 kvm_dirty_ring_push(vcpu, slot, rel_gfn);
3467 else if (memslot->dirty_bitmap)
3468 set_bit_le(rel_gfn, memslot->dirty_bitmap);
3469 }
3470 }
3471 EXPORT_SYMBOL_GPL(mark_page_dirty_in_slot);
3472
mark_page_dirty(struct kvm * kvm,gfn_t gfn)3473 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
3474 {
3475 struct kvm_memory_slot *memslot;
3476
3477 memslot = gfn_to_memslot(kvm, gfn);
3478 mark_page_dirty_in_slot(kvm, memslot, gfn);
3479 }
3480 EXPORT_SYMBOL_GPL(mark_page_dirty);
3481
kvm_vcpu_mark_page_dirty(struct kvm_vcpu * vcpu,gfn_t gfn)3482 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
3483 {
3484 struct kvm_memory_slot *memslot;
3485
3486 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3487 mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn);
3488 }
3489 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
3490
kvm_sigset_activate(struct kvm_vcpu * vcpu)3491 void kvm_sigset_activate(struct kvm_vcpu *vcpu)
3492 {
3493 if (!vcpu->sigset_active)
3494 return;
3495
3496 /*
3497 * This does a lockless modification of ->real_blocked, which is fine
3498 * because, only current can change ->real_blocked and all readers of
3499 * ->real_blocked don't care as long ->real_blocked is always a subset
3500 * of ->blocked.
3501 */
3502 sigprocmask(SIG_SETMASK, &vcpu->sigset, ¤t->real_blocked);
3503 }
3504
kvm_sigset_deactivate(struct kvm_vcpu * vcpu)3505 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
3506 {
3507 if (!vcpu->sigset_active)
3508 return;
3509
3510 sigprocmask(SIG_SETMASK, ¤t->real_blocked, NULL);
3511 sigemptyset(¤t->real_blocked);
3512 }
3513
grow_halt_poll_ns(struct kvm_vcpu * vcpu)3514 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
3515 {
3516 unsigned int old, val, grow, grow_start;
3517
3518 old = val = vcpu->halt_poll_ns;
3519 grow_start = READ_ONCE(halt_poll_ns_grow_start);
3520 grow = READ_ONCE(halt_poll_ns_grow);
3521 if (!grow)
3522 goto out;
3523
3524 val *= grow;
3525 if (val < grow_start)
3526 val = grow_start;
3527
3528 vcpu->halt_poll_ns = val;
3529 out:
3530 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
3531 }
3532
shrink_halt_poll_ns(struct kvm_vcpu * vcpu)3533 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
3534 {
3535 unsigned int old, val, shrink, grow_start;
3536
3537 old = val = vcpu->halt_poll_ns;
3538 shrink = READ_ONCE(halt_poll_ns_shrink);
3539 grow_start = READ_ONCE(halt_poll_ns_grow_start);
3540 if (shrink == 0)
3541 val = 0;
3542 else
3543 val /= shrink;
3544
3545 if (val < grow_start)
3546 val = 0;
3547
3548 vcpu->halt_poll_ns = val;
3549 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
3550 }
3551
kvm_vcpu_check_block(struct kvm_vcpu * vcpu)3552 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
3553 {
3554 int ret = -EINTR;
3555 int idx = srcu_read_lock(&vcpu->kvm->srcu);
3556
3557 if (kvm_arch_vcpu_runnable(vcpu))
3558 goto out;
3559 if (kvm_cpu_has_pending_timer(vcpu))
3560 goto out;
3561 if (signal_pending(current))
3562 goto out;
3563 if (kvm_check_request(KVM_REQ_UNBLOCK, vcpu))
3564 goto out;
3565
3566 ret = 0;
3567 out:
3568 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3569 return ret;
3570 }
3571
3572 /*
3573 * Block the vCPU until the vCPU is runnable, an event arrives, or a signal is
3574 * pending. This is mostly used when halting a vCPU, but may also be used
3575 * directly for other vCPU non-runnable states, e.g. x86's Wait-For-SIPI.
3576 */
kvm_vcpu_block(struct kvm_vcpu * vcpu)3577 bool kvm_vcpu_block(struct kvm_vcpu *vcpu)
3578 {
3579 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
3580 bool waited = false;
3581
3582 vcpu->stat.generic.blocking = 1;
3583
3584 preempt_disable();
3585 kvm_arch_vcpu_blocking(vcpu);
3586 prepare_to_rcuwait(wait);
3587 preempt_enable();
3588
3589 for (;;) {
3590 set_current_state(TASK_INTERRUPTIBLE);
3591
3592 if (kvm_vcpu_check_block(vcpu) < 0)
3593 break;
3594
3595 waited = true;
3596 schedule();
3597 }
3598
3599 preempt_disable();
3600 finish_rcuwait(wait);
3601 kvm_arch_vcpu_unblocking(vcpu);
3602 preempt_enable();
3603
3604 vcpu->stat.generic.blocking = 0;
3605
3606 return waited;
3607 }
3608
update_halt_poll_stats(struct kvm_vcpu * vcpu,ktime_t start,ktime_t end,bool success)3609 static inline void update_halt_poll_stats(struct kvm_vcpu *vcpu, ktime_t start,
3610 ktime_t end, bool success)
3611 {
3612 struct kvm_vcpu_stat_generic *stats = &vcpu->stat.generic;
3613 u64 poll_ns = ktime_to_ns(ktime_sub(end, start));
3614
3615 ++vcpu->stat.generic.halt_attempted_poll;
3616
3617 if (success) {
3618 ++vcpu->stat.generic.halt_successful_poll;
3619
3620 if (!vcpu_valid_wakeup(vcpu))
3621 ++vcpu->stat.generic.halt_poll_invalid;
3622
3623 stats->halt_poll_success_ns += poll_ns;
3624 KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_success_hist, poll_ns);
3625 } else {
3626 stats->halt_poll_fail_ns += poll_ns;
3627 KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_fail_hist, poll_ns);
3628 }
3629 }
3630
kvm_vcpu_max_halt_poll_ns(struct kvm_vcpu * vcpu)3631 static unsigned int kvm_vcpu_max_halt_poll_ns(struct kvm_vcpu *vcpu)
3632 {
3633 struct kvm *kvm = vcpu->kvm;
3634
3635 if (kvm->override_halt_poll_ns) {
3636 /*
3637 * Ensure kvm->max_halt_poll_ns is not read before
3638 * kvm->override_halt_poll_ns.
3639 *
3640 * Pairs with the smp_wmb() when enabling KVM_CAP_HALT_POLL.
3641 */
3642 smp_rmb();
3643 return READ_ONCE(kvm->max_halt_poll_ns);
3644 }
3645
3646 return READ_ONCE(halt_poll_ns);
3647 }
3648
3649 /*
3650 * Emulate a vCPU halt condition, e.g. HLT on x86, WFI on arm, etc... If halt
3651 * polling is enabled, busy wait for a short time before blocking to avoid the
3652 * expensive block+unblock sequence if a wake event arrives soon after the vCPU
3653 * is halted.
3654 */
kvm_vcpu_halt(struct kvm_vcpu * vcpu)3655 void kvm_vcpu_halt(struct kvm_vcpu *vcpu)
3656 {
3657 unsigned int max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu);
3658 bool halt_poll_allowed = !kvm_arch_no_poll(vcpu);
3659 ktime_t start, cur, poll_end;
3660 bool waited = false;
3661 bool do_halt_poll;
3662 u64 halt_ns;
3663
3664 if (vcpu->halt_poll_ns > max_halt_poll_ns)
3665 vcpu->halt_poll_ns = max_halt_poll_ns;
3666
3667 do_halt_poll = halt_poll_allowed && vcpu->halt_poll_ns;
3668
3669 start = cur = poll_end = ktime_get();
3670 if (do_halt_poll) {
3671 ktime_t stop = ktime_add_ns(start, vcpu->halt_poll_ns);
3672
3673 do {
3674 if (kvm_vcpu_check_block(vcpu) < 0)
3675 goto out;
3676 cpu_relax();
3677 poll_end = cur = ktime_get();
3678 } while (kvm_vcpu_can_poll(cur, stop));
3679 }
3680
3681 waited = kvm_vcpu_block(vcpu);
3682
3683 cur = ktime_get();
3684 if (waited) {
3685 vcpu->stat.generic.halt_wait_ns +=
3686 ktime_to_ns(cur) - ktime_to_ns(poll_end);
3687 KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.generic.halt_wait_hist,
3688 ktime_to_ns(cur) - ktime_to_ns(poll_end));
3689 }
3690 out:
3691 /* The total time the vCPU was "halted", including polling time. */
3692 halt_ns = ktime_to_ns(cur) - ktime_to_ns(start);
3693
3694 /*
3695 * Note, halt-polling is considered successful so long as the vCPU was
3696 * never actually scheduled out, i.e. even if the wake event arrived
3697 * after of the halt-polling loop itself, but before the full wait.
3698 */
3699 if (do_halt_poll)
3700 update_halt_poll_stats(vcpu, start, poll_end, !waited);
3701
3702 if (halt_poll_allowed) {
3703 /* Recompute the max halt poll time in case it changed. */
3704 max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu);
3705
3706 if (!vcpu_valid_wakeup(vcpu)) {
3707 shrink_halt_poll_ns(vcpu);
3708 } else if (max_halt_poll_ns) {
3709 if (halt_ns <= vcpu->halt_poll_ns)
3710 ;
3711 /* we had a long block, shrink polling */
3712 else if (vcpu->halt_poll_ns &&
3713 halt_ns > max_halt_poll_ns)
3714 shrink_halt_poll_ns(vcpu);
3715 /* we had a short halt and our poll time is too small */
3716 else if (vcpu->halt_poll_ns < max_halt_poll_ns &&
3717 halt_ns < max_halt_poll_ns)
3718 grow_halt_poll_ns(vcpu);
3719 } else {
3720 vcpu->halt_poll_ns = 0;
3721 }
3722 }
3723
3724 trace_kvm_vcpu_wakeup(halt_ns, waited, vcpu_valid_wakeup(vcpu));
3725 }
3726 EXPORT_SYMBOL_GPL(kvm_vcpu_halt);
3727
kvm_vcpu_wake_up(struct kvm_vcpu * vcpu)3728 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
3729 {
3730 if (__kvm_vcpu_wake_up(vcpu)) {
3731 WRITE_ONCE(vcpu->ready, true);
3732 ++vcpu->stat.generic.halt_wakeup;
3733 return true;
3734 }
3735
3736 return false;
3737 }
3738 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
3739
3740 #ifndef CONFIG_S390
3741 /*
3742 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
3743 */
kvm_vcpu_kick(struct kvm_vcpu * vcpu)3744 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
3745 {
3746 int me, cpu;
3747
3748 if (kvm_vcpu_wake_up(vcpu))
3749 return;
3750
3751 me = get_cpu();
3752 /*
3753 * The only state change done outside the vcpu mutex is IN_GUEST_MODE
3754 * to EXITING_GUEST_MODE. Therefore the moderately expensive "should
3755 * kick" check does not need atomic operations if kvm_vcpu_kick is used
3756 * within the vCPU thread itself.
3757 */
3758 if (vcpu == __this_cpu_read(kvm_running_vcpu)) {
3759 if (vcpu->mode == IN_GUEST_MODE)
3760 WRITE_ONCE(vcpu->mode, EXITING_GUEST_MODE);
3761 goto out;
3762 }
3763
3764 /*
3765 * Note, the vCPU could get migrated to a different pCPU at any point
3766 * after kvm_arch_vcpu_should_kick(), which could result in sending an
3767 * IPI to the previous pCPU. But, that's ok because the purpose of the
3768 * IPI is to force the vCPU to leave IN_GUEST_MODE, and migrating the
3769 * vCPU also requires it to leave IN_GUEST_MODE.
3770 */
3771 if (kvm_arch_vcpu_should_kick(vcpu)) {
3772 cpu = READ_ONCE(vcpu->cpu);
3773 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
3774 smp_send_reschedule(cpu);
3775 }
3776 out:
3777 put_cpu();
3778 }
3779 EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
3780 #endif /* !CONFIG_S390 */
3781
kvm_vcpu_yield_to(struct kvm_vcpu * target)3782 int kvm_vcpu_yield_to(struct kvm_vcpu *target)
3783 {
3784 struct task_struct *task = NULL;
3785 int ret;
3786
3787 if (!read_trylock(&target->pid_lock))
3788 return 0;
3789
3790 if (target->pid)
3791 task = get_pid_task(target->pid, PIDTYPE_PID);
3792
3793 read_unlock(&target->pid_lock);
3794
3795 if (!task)
3796 return 0;
3797 ret = yield_to(task, 1);
3798 put_task_struct(task);
3799
3800 return ret;
3801 }
3802 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
3803
3804 /*
3805 * Helper that checks whether a VCPU is eligible for directed yield.
3806 * Most eligible candidate to yield is decided by following heuristics:
3807 *
3808 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
3809 * (preempted lock holder), indicated by @in_spin_loop.
3810 * Set at the beginning and cleared at the end of interception/PLE handler.
3811 *
3812 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
3813 * chance last time (mostly it has become eligible now since we have probably
3814 * yielded to lockholder in last iteration. This is done by toggling
3815 * @dy_eligible each time a VCPU checked for eligibility.)
3816 *
3817 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
3818 * to preempted lock-holder could result in wrong VCPU selection and CPU
3819 * burning. Giving priority for a potential lock-holder increases lock
3820 * progress.
3821 *
3822 * Since algorithm is based on heuristics, accessing another VCPU data without
3823 * locking does not harm. It may result in trying to yield to same VCPU, fail
3824 * and continue with next VCPU and so on.
3825 */
kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu * vcpu)3826 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
3827 {
3828 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
3829 bool eligible;
3830
3831 eligible = !vcpu->spin_loop.in_spin_loop ||
3832 vcpu->spin_loop.dy_eligible;
3833
3834 if (vcpu->spin_loop.in_spin_loop)
3835 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
3836
3837 return eligible;
3838 #else
3839 return true;
3840 #endif
3841 }
3842
3843 /*
3844 * Unlike kvm_arch_vcpu_runnable, this function is called outside
3845 * a vcpu_load/vcpu_put pair. However, for most architectures
3846 * kvm_arch_vcpu_runnable does not require vcpu_load.
3847 */
kvm_arch_dy_runnable(struct kvm_vcpu * vcpu)3848 bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
3849 {
3850 return kvm_arch_vcpu_runnable(vcpu);
3851 }
3852
vcpu_dy_runnable(struct kvm_vcpu * vcpu)3853 static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu)
3854 {
3855 if (kvm_arch_dy_runnable(vcpu))
3856 return true;
3857
3858 #ifdef CONFIG_KVM_ASYNC_PF
3859 if (!list_empty_careful(&vcpu->async_pf.done))
3860 return true;
3861 #endif
3862
3863 return false;
3864 }
3865
3866 /*
3867 * By default, simply query the target vCPU's current mode when checking if a
3868 * vCPU was preempted in kernel mode. All architectures except x86 (or more
3869 * specifical, except VMX) allow querying whether or not a vCPU is in kernel
3870 * mode even if the vCPU is NOT loaded, i.e. using kvm_arch_vcpu_in_kernel()
3871 * directly for cross-vCPU checks is functionally correct and accurate.
3872 */
kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu * vcpu)3873 bool __weak kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu)
3874 {
3875 return kvm_arch_vcpu_in_kernel(vcpu);
3876 }
3877
kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu * vcpu)3878 bool __weak kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)
3879 {
3880 return false;
3881 }
3882
kvm_vcpu_on_spin(struct kvm_vcpu * me,bool yield_to_kernel_mode)3883 void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
3884 {
3885 int nr_vcpus, start, i, idx, yielded;
3886 struct kvm *kvm = me->kvm;
3887 struct kvm_vcpu *vcpu;
3888 int try = 3;
3889
3890 nr_vcpus = atomic_read(&kvm->online_vcpus);
3891 if (nr_vcpus < 2)
3892 return;
3893
3894 /* Pairs with the smp_wmb() in kvm_vm_ioctl_create_vcpu(). */
3895 smp_rmb();
3896
3897 kvm_vcpu_set_in_spin_loop(me, true);
3898
3899 /*
3900 * The current vCPU ("me") is spinning in kernel mode, i.e. is likely
3901 * waiting for a resource to become available. Attempt to yield to a
3902 * vCPU that is runnable, but not currently running, e.g. because the
3903 * vCPU was preempted by a higher priority task. With luck, the vCPU
3904 * that was preempted is holding a lock or some other resource that the
3905 * current vCPU is waiting to acquire, and yielding to the other vCPU
3906 * will allow it to make forward progress and release the lock (or kick
3907 * the spinning vCPU, etc).
3908 *
3909 * Since KVM has no insight into what exactly the guest is doing,
3910 * approximate a round-robin selection by iterating over all vCPUs,
3911 * starting at the last boosted vCPU. I.e. if N=kvm->last_boosted_vcpu,
3912 * iterate over vCPU[N+1]..vCPU[N-1], wrapping as needed.
3913 *
3914 * Note, this is inherently racy, e.g. if multiple vCPUs are spinning,
3915 * they may all try to yield to the same vCPU(s). But as above, this
3916 * is all best effort due to KVM's lack of visibility into the guest.
3917 */
3918 start = READ_ONCE(kvm->last_boosted_vcpu) + 1;
3919 for (i = 0; i < nr_vcpus; i++) {
3920 idx = (start + i) % nr_vcpus;
3921 if (idx == me->vcpu_idx)
3922 continue;
3923
3924 vcpu = xa_load(&kvm->vcpu_array, idx);
3925 if (!READ_ONCE(vcpu->ready))
3926 continue;
3927 if (kvm_vcpu_is_blocking(vcpu) && !vcpu_dy_runnable(vcpu))
3928 continue;
3929
3930 /*
3931 * Treat the target vCPU as being in-kernel if it has a pending
3932 * interrupt, as the vCPU trying to yield may be spinning
3933 * waiting on IPI delivery, i.e. the target vCPU is in-kernel
3934 * for the purposes of directed yield.
3935 */
3936 if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
3937 !kvm_arch_dy_has_pending_interrupt(vcpu) &&
3938 !kvm_arch_vcpu_preempted_in_kernel(vcpu))
3939 continue;
3940
3941 if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
3942 continue;
3943
3944 yielded = kvm_vcpu_yield_to(vcpu);
3945 if (yielded > 0) {
3946 WRITE_ONCE(kvm->last_boosted_vcpu, i);
3947 break;
3948 } else if (yielded < 0 && !--try) {
3949 break;
3950 }
3951 }
3952 kvm_vcpu_set_in_spin_loop(me, false);
3953
3954 /* Ensure vcpu is not eligible during next spinloop */
3955 kvm_vcpu_set_dy_eligible(me, false);
3956 }
3957 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
3958
kvm_page_in_dirty_ring(struct kvm * kvm,unsigned long pgoff)3959 static bool kvm_page_in_dirty_ring(struct kvm *kvm, unsigned long pgoff)
3960 {
3961 #ifdef CONFIG_HAVE_KVM_DIRTY_RING
3962 return (pgoff >= KVM_DIRTY_LOG_PAGE_OFFSET) &&
3963 (pgoff < KVM_DIRTY_LOG_PAGE_OFFSET +
3964 kvm->dirty_ring_size / PAGE_SIZE);
3965 #else
3966 return false;
3967 #endif
3968 }
3969
kvm_vcpu_fault(struct vm_fault * vmf)3970 static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
3971 {
3972 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
3973 struct page *page;
3974
3975 if (vmf->pgoff == 0)
3976 page = virt_to_page(vcpu->run);
3977 #ifdef CONFIG_X86
3978 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
3979 page = virt_to_page(vcpu->arch.pio_data);
3980 #endif
3981 #ifdef CONFIG_KVM_MMIO
3982 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
3983 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
3984 #endif
3985 else if (kvm_page_in_dirty_ring(vcpu->kvm, vmf->pgoff))
3986 page = kvm_dirty_ring_get_page(
3987 &vcpu->dirty_ring,
3988 vmf->pgoff - KVM_DIRTY_LOG_PAGE_OFFSET);
3989 else
3990 return kvm_arch_vcpu_fault(vcpu, vmf);
3991 get_page(page);
3992 vmf->page = page;
3993 return 0;
3994 }
3995
3996 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
3997 .fault = kvm_vcpu_fault,
3998 };
3999
kvm_vcpu_mmap(struct file * file,struct vm_area_struct * vma)4000 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
4001 {
4002 struct kvm_vcpu *vcpu = file->private_data;
4003 unsigned long pages = vma_pages(vma);
4004
4005 if ((kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff) ||
4006 kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff + pages - 1)) &&
4007 ((vma->vm_flags & VM_EXEC) || !(vma->vm_flags & VM_SHARED)))
4008 return -EINVAL;
4009
4010 vma->vm_ops = &kvm_vcpu_vm_ops;
4011 return 0;
4012 }
4013
kvm_vcpu_release(struct inode * inode,struct file * filp)4014 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
4015 {
4016 struct kvm_vcpu *vcpu = filp->private_data;
4017
4018 kvm_put_kvm(vcpu->kvm);
4019 return 0;
4020 }
4021
4022 static struct file_operations kvm_vcpu_fops = {
4023 .release = kvm_vcpu_release,
4024 .unlocked_ioctl = kvm_vcpu_ioctl,
4025 .mmap = kvm_vcpu_mmap,
4026 .llseek = noop_llseek,
4027 KVM_COMPAT(kvm_vcpu_compat_ioctl),
4028 };
4029
4030 /*
4031 * Allocates an inode for the vcpu.
4032 */
create_vcpu_fd(struct kvm_vcpu * vcpu)4033 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
4034 {
4035 char name[8 + 1 + ITOA_MAX_LEN + 1];
4036
4037 snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
4038 return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
4039 }
4040
4041 #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS
vcpu_get_pid(void * data,u64 * val)4042 static int vcpu_get_pid(void *data, u64 *val)
4043 {
4044 struct kvm_vcpu *vcpu = data;
4045
4046 read_lock(&vcpu->pid_lock);
4047 *val = pid_nr(vcpu->pid);
4048 read_unlock(&vcpu->pid_lock);
4049 return 0;
4050 }
4051
4052 DEFINE_SIMPLE_ATTRIBUTE(vcpu_get_pid_fops, vcpu_get_pid, NULL, "%llu\n");
4053
kvm_create_vcpu_debugfs(struct kvm_vcpu * vcpu)4054 static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
4055 {
4056 struct dentry *debugfs_dentry;
4057 char dir_name[ITOA_MAX_LEN * 2];
4058
4059 if (!debugfs_initialized())
4060 return;
4061
4062 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
4063 debugfs_dentry = debugfs_create_dir(dir_name,
4064 vcpu->kvm->debugfs_dentry);
4065 debugfs_create_file("pid", 0444, debugfs_dentry, vcpu,
4066 &vcpu_get_pid_fops);
4067
4068 kvm_arch_create_vcpu_debugfs(vcpu, debugfs_dentry);
4069 }
4070 #endif
4071
4072 /*
4073 * Creates some virtual cpus. Good luck creating more than one.
4074 */
kvm_vm_ioctl_create_vcpu(struct kvm * kvm,unsigned long id)4075 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id)
4076 {
4077 int r;
4078 struct kvm_vcpu *vcpu;
4079 struct page *page;
4080
4081 /*
4082 * KVM tracks vCPU IDs as 'int', be kind to userspace and reject
4083 * too-large values instead of silently truncating.
4084 *
4085 * Ensure KVM_MAX_VCPU_IDS isn't pushed above INT_MAX without first
4086 * changing the storage type (at the very least, IDs should be tracked
4087 * as unsigned ints).
4088 */
4089 BUILD_BUG_ON(KVM_MAX_VCPU_IDS > INT_MAX);
4090 if (id >= KVM_MAX_VCPU_IDS)
4091 return -EINVAL;
4092
4093 mutex_lock(&kvm->lock);
4094 if (kvm->created_vcpus >= kvm->max_vcpus) {
4095 mutex_unlock(&kvm->lock);
4096 return -EINVAL;
4097 }
4098
4099 r = kvm_arch_vcpu_precreate(kvm, id);
4100 if (r) {
4101 mutex_unlock(&kvm->lock);
4102 return r;
4103 }
4104
4105 kvm->created_vcpus++;
4106 mutex_unlock(&kvm->lock);
4107
4108 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
4109 if (!vcpu) {
4110 r = -ENOMEM;
4111 goto vcpu_decrement;
4112 }
4113
4114 BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE);
4115 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
4116 if (!page) {
4117 r = -ENOMEM;
4118 goto vcpu_free;
4119 }
4120 vcpu->run = page_address(page);
4121
4122 kvm_vcpu_init(vcpu, kvm, id);
4123
4124 r = kvm_arch_vcpu_create(vcpu);
4125 if (r)
4126 goto vcpu_free_run_page;
4127
4128 if (kvm->dirty_ring_size) {
4129 r = kvm_dirty_ring_alloc(&vcpu->dirty_ring,
4130 id, kvm->dirty_ring_size);
4131 if (r)
4132 goto arch_vcpu_destroy;
4133 }
4134
4135 mutex_lock(&kvm->lock);
4136
4137 if (kvm_get_vcpu_by_id(kvm, id)) {
4138 r = -EEXIST;
4139 goto unlock_vcpu_destroy;
4140 }
4141
4142 vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
4143 r = xa_insert(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT);
4144 WARN_ON_ONCE(r == -EBUSY);
4145 if (r)
4146 goto unlock_vcpu_destroy;
4147
4148 /*
4149 * Now it's all set up, let userspace reach it. Grab the vCPU's mutex
4150 * so that userspace can't invoke vCPU ioctl()s until the vCPU is fully
4151 * visible (per online_vcpus), e.g. so that KVM doesn't get tricked
4152 * into a NULL-pointer dereference because KVM thinks the _current_
4153 * vCPU doesn't exist. As a bonus, taking vcpu->mutex ensures lockdep
4154 * knows it's taken *inside* kvm->lock.
4155 */
4156 mutex_lock(&vcpu->mutex);
4157 kvm_get_kvm(kvm);
4158 r = create_vcpu_fd(vcpu);
4159 if (r < 0)
4160 goto kvm_put_xa_erase;
4161
4162 /*
4163 * Pairs with smp_rmb() in kvm_get_vcpu. Store the vcpu
4164 * pointer before kvm->online_vcpu's incremented value.
4165 */
4166 smp_wmb();
4167 atomic_inc(&kvm->online_vcpus);
4168 mutex_unlock(&vcpu->mutex);
4169
4170 mutex_unlock(&kvm->lock);
4171 kvm_arch_vcpu_postcreate(vcpu);
4172 kvm_create_vcpu_debugfs(vcpu);
4173 return r;
4174
4175 kvm_put_xa_erase:
4176 mutex_unlock(&vcpu->mutex);
4177 kvm_put_kvm_no_destroy(kvm);
4178 xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx);
4179 unlock_vcpu_destroy:
4180 mutex_unlock(&kvm->lock);
4181 kvm_dirty_ring_free(&vcpu->dirty_ring);
4182 arch_vcpu_destroy:
4183 kvm_arch_vcpu_destroy(vcpu);
4184 vcpu_free_run_page:
4185 free_page((unsigned long)vcpu->run);
4186 vcpu_free:
4187 kmem_cache_free(kvm_vcpu_cache, vcpu);
4188 vcpu_decrement:
4189 mutex_lock(&kvm->lock);
4190 kvm->created_vcpus--;
4191 mutex_unlock(&kvm->lock);
4192 return r;
4193 }
4194
kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu * vcpu,sigset_t * sigset)4195 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
4196 {
4197 if (sigset) {
4198 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
4199 vcpu->sigset_active = 1;
4200 vcpu->sigset = *sigset;
4201 } else
4202 vcpu->sigset_active = 0;
4203 return 0;
4204 }
4205
kvm_vcpu_stats_read(struct file * file,char __user * user_buffer,size_t size,loff_t * offset)4206 static ssize_t kvm_vcpu_stats_read(struct file *file, char __user *user_buffer,
4207 size_t size, loff_t *offset)
4208 {
4209 struct kvm_vcpu *vcpu = file->private_data;
4210
4211 return kvm_stats_read(vcpu->stats_id, &kvm_vcpu_stats_header,
4212 &kvm_vcpu_stats_desc[0], &vcpu->stat,
4213 sizeof(vcpu->stat), user_buffer, size, offset);
4214 }
4215
kvm_vcpu_stats_release(struct inode * inode,struct file * file)4216 static int kvm_vcpu_stats_release(struct inode *inode, struct file *file)
4217 {
4218 struct kvm_vcpu *vcpu = file->private_data;
4219
4220 kvm_put_kvm(vcpu->kvm);
4221 return 0;
4222 }
4223
4224 static const struct file_operations kvm_vcpu_stats_fops = {
4225 .owner = THIS_MODULE,
4226 .read = kvm_vcpu_stats_read,
4227 .release = kvm_vcpu_stats_release,
4228 .llseek = noop_llseek,
4229 };
4230
kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu * vcpu)4231 static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu)
4232 {
4233 int fd;
4234 struct file *file;
4235 char name[15 + ITOA_MAX_LEN + 1];
4236
4237 snprintf(name, sizeof(name), "kvm-vcpu-stats:%d", vcpu->vcpu_id);
4238
4239 fd = get_unused_fd_flags(O_CLOEXEC);
4240 if (fd < 0)
4241 return fd;
4242
4243 file = anon_inode_getfile_fmode(name, &kvm_vcpu_stats_fops, vcpu,
4244 O_RDONLY, FMODE_PREAD);
4245 if (IS_ERR(file)) {
4246 put_unused_fd(fd);
4247 return PTR_ERR(file);
4248 }
4249
4250 kvm_get_kvm(vcpu->kvm);
4251 fd_install(fd, file);
4252
4253 return fd;
4254 }
4255
4256 #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY
kvm_vcpu_pre_fault_memory(struct kvm_vcpu * vcpu,struct kvm_pre_fault_memory * range)4257 static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
4258 struct kvm_pre_fault_memory *range)
4259 {
4260 int idx;
4261 long r;
4262 u64 full_size;
4263
4264 if (range->flags)
4265 return -EINVAL;
4266
4267 if (!PAGE_ALIGNED(range->gpa) ||
4268 !PAGE_ALIGNED(range->size) ||
4269 range->gpa + range->size <= range->gpa)
4270 return -EINVAL;
4271
4272 vcpu_load(vcpu);
4273 idx = srcu_read_lock(&vcpu->kvm->srcu);
4274
4275 full_size = range->size;
4276 do {
4277 if (signal_pending(current)) {
4278 r = -EINTR;
4279 break;
4280 }
4281
4282 r = kvm_arch_vcpu_pre_fault_memory(vcpu, range);
4283 if (WARN_ON_ONCE(r == 0 || r == -EIO))
4284 break;
4285
4286 if (r < 0)
4287 break;
4288
4289 range->size -= r;
4290 range->gpa += r;
4291 cond_resched();
4292 } while (range->size);
4293
4294 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4295 vcpu_put(vcpu);
4296
4297 /* Return success if at least one page was mapped successfully. */
4298 return full_size == range->size ? r : 0;
4299 }
4300 #endif
4301
kvm_wait_for_vcpu_online(struct kvm_vcpu * vcpu)4302 static int kvm_wait_for_vcpu_online(struct kvm_vcpu *vcpu)
4303 {
4304 struct kvm *kvm = vcpu->kvm;
4305
4306 /*
4307 * In practice, this happy path will always be taken, as a well-behaved
4308 * VMM will never invoke a vCPU ioctl() before KVM_CREATE_VCPU returns.
4309 */
4310 if (likely(vcpu->vcpu_idx < atomic_read(&kvm->online_vcpus)))
4311 return 0;
4312
4313 /*
4314 * Acquire and release the vCPU's mutex to wait for vCPU creation to
4315 * complete (kvm_vm_ioctl_create_vcpu() holds the mutex until the vCPU
4316 * is fully online).
4317 */
4318 if (mutex_lock_killable(&vcpu->mutex))
4319 return -EINTR;
4320
4321 mutex_unlock(&vcpu->mutex);
4322
4323 if (WARN_ON_ONCE(!kvm_get_vcpu(kvm, vcpu->vcpu_idx)))
4324 return -EIO;
4325
4326 return 0;
4327 }
4328
kvm_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)4329 static long kvm_vcpu_ioctl(struct file *filp,
4330 unsigned int ioctl, unsigned long arg)
4331 {
4332 struct kvm_vcpu *vcpu = filp->private_data;
4333 void __user *argp = (void __user *)arg;
4334 int r;
4335 struct kvm_fpu *fpu = NULL;
4336 struct kvm_sregs *kvm_sregs = NULL;
4337
4338 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)
4339 return -EIO;
4340
4341 if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
4342 return -EINVAL;
4343
4344 /*
4345 * Wait for the vCPU to be online before handling the ioctl(), as KVM
4346 * assumes the vCPU is reachable via vcpu_array, i.e. may dereference
4347 * a NULL pointer if userspace invokes an ioctl() before KVM is ready.
4348 */
4349 r = kvm_wait_for_vcpu_online(vcpu);
4350 if (r)
4351 return r;
4352
4353 /*
4354 * Some architectures have vcpu ioctls that are asynchronous to vcpu
4355 * execution; mutex_lock() would break them.
4356 */
4357 r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
4358 if (r != -ENOIOCTLCMD)
4359 return r;
4360
4361 if (mutex_lock_killable(&vcpu->mutex))
4362 return -EINTR;
4363 switch (ioctl) {
4364 case KVM_RUN: {
4365 struct pid *oldpid;
4366 r = -EINVAL;
4367 if (arg)
4368 goto out;
4369
4370 /*
4371 * Note, vcpu->pid is primarily protected by vcpu->mutex. The
4372 * dedicated r/w lock allows other tasks, e.g. other vCPUs, to
4373 * read vcpu->pid while this vCPU is in KVM_RUN, e.g. to yield
4374 * directly to this vCPU
4375 */
4376 oldpid = vcpu->pid;
4377 if (unlikely(oldpid != task_pid(current))) {
4378 /* The thread running this VCPU changed. */
4379 struct pid *newpid;
4380
4381 r = kvm_arch_vcpu_run_pid_change(vcpu);
4382 if (r)
4383 break;
4384
4385 newpid = get_task_pid(current, PIDTYPE_PID);
4386 write_lock(&vcpu->pid_lock);
4387 vcpu->pid = newpid;
4388 write_unlock(&vcpu->pid_lock);
4389
4390 put_pid(oldpid);
4391 }
4392 vcpu->wants_to_run = !READ_ONCE(vcpu->run->immediate_exit__unsafe);
4393 r = kvm_arch_vcpu_ioctl_run(vcpu);
4394 vcpu->wants_to_run = false;
4395
4396 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
4397 break;
4398 }
4399 case KVM_GET_REGS: {
4400 struct kvm_regs *kvm_regs;
4401
4402 r = -ENOMEM;
4403 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
4404 if (!kvm_regs)
4405 goto out;
4406 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
4407 if (r)
4408 goto out_free1;
4409 r = -EFAULT;
4410 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
4411 goto out_free1;
4412 r = 0;
4413 out_free1:
4414 kfree(kvm_regs);
4415 break;
4416 }
4417 case KVM_SET_REGS: {
4418 struct kvm_regs *kvm_regs;
4419
4420 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
4421 if (IS_ERR(kvm_regs)) {
4422 r = PTR_ERR(kvm_regs);
4423 goto out;
4424 }
4425 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
4426 kfree(kvm_regs);
4427 break;
4428 }
4429 case KVM_GET_SREGS: {
4430 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
4431 r = -ENOMEM;
4432 if (!kvm_sregs)
4433 goto out;
4434 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
4435 if (r)
4436 goto out;
4437 r = -EFAULT;
4438 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
4439 goto out;
4440 r = 0;
4441 break;
4442 }
4443 case KVM_SET_SREGS: {
4444 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
4445 if (IS_ERR(kvm_sregs)) {
4446 r = PTR_ERR(kvm_sregs);
4447 kvm_sregs = NULL;
4448 goto out;
4449 }
4450 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
4451 break;
4452 }
4453 case KVM_GET_MP_STATE: {
4454 struct kvm_mp_state mp_state;
4455
4456 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
4457 if (r)
4458 goto out;
4459 r = -EFAULT;
4460 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
4461 goto out;
4462 r = 0;
4463 break;
4464 }
4465 case KVM_SET_MP_STATE: {
4466 struct kvm_mp_state mp_state;
4467
4468 r = -EFAULT;
4469 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
4470 goto out;
4471 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
4472 break;
4473 }
4474 case KVM_TRANSLATE: {
4475 struct kvm_translation tr;
4476
4477 r = -EFAULT;
4478 if (copy_from_user(&tr, argp, sizeof(tr)))
4479 goto out;
4480 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
4481 if (r)
4482 goto out;
4483 r = -EFAULT;
4484 if (copy_to_user(argp, &tr, sizeof(tr)))
4485 goto out;
4486 r = 0;
4487 break;
4488 }
4489 case KVM_SET_GUEST_DEBUG: {
4490 struct kvm_guest_debug dbg;
4491
4492 r = -EFAULT;
4493 if (copy_from_user(&dbg, argp, sizeof(dbg)))
4494 goto out;
4495 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
4496 break;
4497 }
4498 case KVM_SET_SIGNAL_MASK: {
4499 struct kvm_signal_mask __user *sigmask_arg = argp;
4500 struct kvm_signal_mask kvm_sigmask;
4501 sigset_t sigset, *p;
4502
4503 p = NULL;
4504 if (argp) {
4505 r = -EFAULT;
4506 if (copy_from_user(&kvm_sigmask, argp,
4507 sizeof(kvm_sigmask)))
4508 goto out;
4509 r = -EINVAL;
4510 if (kvm_sigmask.len != sizeof(sigset))
4511 goto out;
4512 r = -EFAULT;
4513 if (copy_from_user(&sigset, sigmask_arg->sigset,
4514 sizeof(sigset)))
4515 goto out;
4516 p = &sigset;
4517 }
4518 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
4519 break;
4520 }
4521 case KVM_GET_FPU: {
4522 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
4523 r = -ENOMEM;
4524 if (!fpu)
4525 goto out;
4526 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
4527 if (r)
4528 goto out;
4529 r = -EFAULT;
4530 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
4531 goto out;
4532 r = 0;
4533 break;
4534 }
4535 case KVM_SET_FPU: {
4536 fpu = memdup_user(argp, sizeof(*fpu));
4537 if (IS_ERR(fpu)) {
4538 r = PTR_ERR(fpu);
4539 fpu = NULL;
4540 goto out;
4541 }
4542 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
4543 break;
4544 }
4545 case KVM_GET_STATS_FD: {
4546 r = kvm_vcpu_ioctl_get_stats_fd(vcpu);
4547 break;
4548 }
4549 #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY
4550 case KVM_PRE_FAULT_MEMORY: {
4551 struct kvm_pre_fault_memory range;
4552
4553 r = -EFAULT;
4554 if (copy_from_user(&range, argp, sizeof(range)))
4555 break;
4556 r = kvm_vcpu_pre_fault_memory(vcpu, &range);
4557 /* Pass back leftover range. */
4558 if (copy_to_user(argp, &range, sizeof(range)))
4559 r = -EFAULT;
4560 break;
4561 }
4562 #endif
4563 default:
4564 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
4565 }
4566 out:
4567 mutex_unlock(&vcpu->mutex);
4568 kfree(fpu);
4569 kfree(kvm_sregs);
4570 return r;
4571 }
4572
4573 #ifdef CONFIG_KVM_COMPAT
kvm_vcpu_compat_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)4574 static long kvm_vcpu_compat_ioctl(struct file *filp,
4575 unsigned int ioctl, unsigned long arg)
4576 {
4577 struct kvm_vcpu *vcpu = filp->private_data;
4578 void __user *argp = compat_ptr(arg);
4579 int r;
4580
4581 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)
4582 return -EIO;
4583
4584 switch (ioctl) {
4585 case KVM_SET_SIGNAL_MASK: {
4586 struct kvm_signal_mask __user *sigmask_arg = argp;
4587 struct kvm_signal_mask kvm_sigmask;
4588 sigset_t sigset;
4589
4590 if (argp) {
4591 r = -EFAULT;
4592 if (copy_from_user(&kvm_sigmask, argp,
4593 sizeof(kvm_sigmask)))
4594 goto out;
4595 r = -EINVAL;
4596 if (kvm_sigmask.len != sizeof(compat_sigset_t))
4597 goto out;
4598 r = -EFAULT;
4599 if (get_compat_sigset(&sigset,
4600 (compat_sigset_t __user *)sigmask_arg->sigset))
4601 goto out;
4602 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
4603 } else
4604 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
4605 break;
4606 }
4607 default:
4608 r = kvm_vcpu_ioctl(filp, ioctl, arg);
4609 }
4610
4611 out:
4612 return r;
4613 }
4614 #endif
4615
kvm_device_mmap(struct file * filp,struct vm_area_struct * vma)4616 static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma)
4617 {
4618 struct kvm_device *dev = filp->private_data;
4619
4620 if (dev->ops->mmap)
4621 return dev->ops->mmap(dev, vma);
4622
4623 return -ENODEV;
4624 }
4625
kvm_device_ioctl_attr(struct kvm_device * dev,int (* accessor)(struct kvm_device * dev,struct kvm_device_attr * attr),unsigned long arg)4626 static int kvm_device_ioctl_attr(struct kvm_device *dev,
4627 int (*accessor)(struct kvm_device *dev,
4628 struct kvm_device_attr *attr),
4629 unsigned long arg)
4630 {
4631 struct kvm_device_attr attr;
4632
4633 if (!accessor)
4634 return -EPERM;
4635
4636 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4637 return -EFAULT;
4638
4639 return accessor(dev, &attr);
4640 }
4641
kvm_device_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)4642 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
4643 unsigned long arg)
4644 {
4645 struct kvm_device *dev = filp->private_data;
4646
4647 if (dev->kvm->mm != current->mm || dev->kvm->vm_dead)
4648 return -EIO;
4649
4650 switch (ioctl) {
4651 case KVM_SET_DEVICE_ATTR:
4652 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
4653 case KVM_GET_DEVICE_ATTR:
4654 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
4655 case KVM_HAS_DEVICE_ATTR:
4656 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
4657 default:
4658 if (dev->ops->ioctl)
4659 return dev->ops->ioctl(dev, ioctl, arg);
4660
4661 return -ENOTTY;
4662 }
4663 }
4664
kvm_device_release(struct inode * inode,struct file * filp)4665 static int kvm_device_release(struct inode *inode, struct file *filp)
4666 {
4667 struct kvm_device *dev = filp->private_data;
4668 struct kvm *kvm = dev->kvm;
4669
4670 if (dev->ops->release) {
4671 mutex_lock(&kvm->lock);
4672 list_del_rcu(&dev->vm_node);
4673 synchronize_rcu();
4674 dev->ops->release(dev);
4675 mutex_unlock(&kvm->lock);
4676 }
4677
4678 kvm_put_kvm(kvm);
4679 return 0;
4680 }
4681
4682 static struct file_operations kvm_device_fops = {
4683 .unlocked_ioctl = kvm_device_ioctl,
4684 .release = kvm_device_release,
4685 KVM_COMPAT(kvm_device_ioctl),
4686 .mmap = kvm_device_mmap,
4687 };
4688
kvm_device_from_filp(struct file * filp)4689 struct kvm_device *kvm_device_from_filp(struct file *filp)
4690 {
4691 if (filp->f_op != &kvm_device_fops)
4692 return NULL;
4693
4694 return filp->private_data;
4695 }
4696
4697 static const struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
4698 #ifdef CONFIG_KVM_MPIC
4699 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops,
4700 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops,
4701 #endif
4702 };
4703
kvm_register_device_ops(const struct kvm_device_ops * ops,u32 type)4704 int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type)
4705 {
4706 if (type >= ARRAY_SIZE(kvm_device_ops_table))
4707 return -ENOSPC;
4708
4709 if (kvm_device_ops_table[type] != NULL)
4710 return -EEXIST;
4711
4712 kvm_device_ops_table[type] = ops;
4713 return 0;
4714 }
4715
kvm_unregister_device_ops(u32 type)4716 void kvm_unregister_device_ops(u32 type)
4717 {
4718 if (kvm_device_ops_table[type] != NULL)
4719 kvm_device_ops_table[type] = NULL;
4720 }
4721
kvm_ioctl_create_device(struct kvm * kvm,struct kvm_create_device * cd)4722 static int kvm_ioctl_create_device(struct kvm *kvm,
4723 struct kvm_create_device *cd)
4724 {
4725 const struct kvm_device_ops *ops;
4726 struct kvm_device *dev;
4727 bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
4728 int type;
4729 int ret;
4730
4731 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
4732 return -ENODEV;
4733
4734 type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table));
4735 ops = kvm_device_ops_table[type];
4736 if (ops == NULL)
4737 return -ENODEV;
4738
4739 if (test)
4740 return 0;
4741
4742 dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
4743 if (!dev)
4744 return -ENOMEM;
4745
4746 dev->ops = ops;
4747 dev->kvm = kvm;
4748
4749 mutex_lock(&kvm->lock);
4750 ret = ops->create(dev, type);
4751 if (ret < 0) {
4752 mutex_unlock(&kvm->lock);
4753 kfree(dev);
4754 return ret;
4755 }
4756 list_add_rcu(&dev->vm_node, &kvm->devices);
4757 mutex_unlock(&kvm->lock);
4758
4759 if (ops->init)
4760 ops->init(dev);
4761
4762 kvm_get_kvm(kvm);
4763 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
4764 if (ret < 0) {
4765 kvm_put_kvm_no_destroy(kvm);
4766 mutex_lock(&kvm->lock);
4767 list_del_rcu(&dev->vm_node);
4768 synchronize_rcu();
4769 if (ops->release)
4770 ops->release(dev);
4771 mutex_unlock(&kvm->lock);
4772 if (ops->destroy)
4773 ops->destroy(dev);
4774 return ret;
4775 }
4776
4777 cd->fd = ret;
4778 return 0;
4779 }
4780
kvm_vm_ioctl_check_extension_generic(struct kvm * kvm,long arg)4781 static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
4782 {
4783 switch (arg) {
4784 case KVM_CAP_USER_MEMORY:
4785 case KVM_CAP_USER_MEMORY2:
4786 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
4787 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
4788 case KVM_CAP_INTERNAL_ERROR_DATA:
4789 #ifdef CONFIG_HAVE_KVM_MSI
4790 case KVM_CAP_SIGNAL_MSI:
4791 #endif
4792 #ifdef CONFIG_HAVE_KVM_IRQCHIP
4793 case KVM_CAP_IRQFD:
4794 #endif
4795 case KVM_CAP_IOEVENTFD_ANY_LENGTH:
4796 case KVM_CAP_CHECK_EXTENSION_VM:
4797 case KVM_CAP_ENABLE_CAP_VM:
4798 case KVM_CAP_HALT_POLL:
4799 return 1;
4800 #ifdef CONFIG_KVM_MMIO
4801 case KVM_CAP_COALESCED_MMIO:
4802 return KVM_COALESCED_MMIO_PAGE_OFFSET;
4803 case KVM_CAP_COALESCED_PIO:
4804 return 1;
4805 #endif
4806 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
4807 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
4808 return KVM_DIRTY_LOG_MANUAL_CAPS;
4809 #endif
4810 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
4811 case KVM_CAP_IRQ_ROUTING:
4812 return KVM_MAX_IRQ_ROUTES;
4813 #endif
4814 #if KVM_MAX_NR_ADDRESS_SPACES > 1
4815 case KVM_CAP_MULTI_ADDRESS_SPACE:
4816 if (kvm)
4817 return kvm_arch_nr_memslot_as_ids(kvm);
4818 return KVM_MAX_NR_ADDRESS_SPACES;
4819 #endif
4820 case KVM_CAP_NR_MEMSLOTS:
4821 return KVM_USER_MEM_SLOTS;
4822 case KVM_CAP_DIRTY_LOG_RING:
4823 #ifdef CONFIG_HAVE_KVM_DIRTY_RING_TSO
4824 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
4825 #else
4826 return 0;
4827 #endif
4828 case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
4829 #ifdef CONFIG_HAVE_KVM_DIRTY_RING_ACQ_REL
4830 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
4831 #else
4832 return 0;
4833 #endif
4834 #ifdef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP
4835 case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP:
4836 #endif
4837 case KVM_CAP_BINARY_STATS_FD:
4838 case KVM_CAP_SYSTEM_EVENT_DATA:
4839 case KVM_CAP_DEVICE_CTRL:
4840 return 1;
4841 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
4842 case KVM_CAP_MEMORY_ATTRIBUTES:
4843 return kvm_supported_mem_attributes(kvm);
4844 #endif
4845 #ifdef CONFIG_KVM_PRIVATE_MEM
4846 case KVM_CAP_GUEST_MEMFD:
4847 return !kvm || kvm_arch_has_private_mem(kvm);
4848 #endif
4849 default:
4850 break;
4851 }
4852 return kvm_vm_ioctl_check_extension(kvm, arg);
4853 }
4854
kvm_vm_ioctl_enable_dirty_log_ring(struct kvm * kvm,u32 size)4855 static int kvm_vm_ioctl_enable_dirty_log_ring(struct kvm *kvm, u32 size)
4856 {
4857 int r;
4858
4859 if (!KVM_DIRTY_LOG_PAGE_OFFSET)
4860 return -EINVAL;
4861
4862 /* the size should be power of 2 */
4863 if (!size || (size & (size - 1)))
4864 return -EINVAL;
4865
4866 /* Should be bigger to keep the reserved entries, or a page */
4867 if (size < kvm_dirty_ring_get_rsvd_entries() *
4868 sizeof(struct kvm_dirty_gfn) || size < PAGE_SIZE)
4869 return -EINVAL;
4870
4871 if (size > KVM_DIRTY_RING_MAX_ENTRIES *
4872 sizeof(struct kvm_dirty_gfn))
4873 return -E2BIG;
4874
4875 /* We only allow it to set once */
4876 if (kvm->dirty_ring_size)
4877 return -EINVAL;
4878
4879 mutex_lock(&kvm->lock);
4880
4881 if (kvm->created_vcpus) {
4882 /* We don't allow to change this value after vcpu created */
4883 r = -EINVAL;
4884 } else {
4885 kvm->dirty_ring_size = size;
4886 r = 0;
4887 }
4888
4889 mutex_unlock(&kvm->lock);
4890 return r;
4891 }
4892
kvm_vm_ioctl_reset_dirty_pages(struct kvm * kvm)4893 static int kvm_vm_ioctl_reset_dirty_pages(struct kvm *kvm)
4894 {
4895 unsigned long i;
4896 struct kvm_vcpu *vcpu;
4897 int cleared = 0;
4898
4899 if (!kvm->dirty_ring_size)
4900 return -EINVAL;
4901
4902 mutex_lock(&kvm->slots_lock);
4903
4904 kvm_for_each_vcpu(i, vcpu, kvm)
4905 cleared += kvm_dirty_ring_reset(vcpu->kvm, &vcpu->dirty_ring);
4906
4907 mutex_unlock(&kvm->slots_lock);
4908
4909 if (cleared)
4910 kvm_flush_remote_tlbs(kvm);
4911
4912 return cleared;
4913 }
4914
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)4915 int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
4916 struct kvm_enable_cap *cap)
4917 {
4918 return -EINVAL;
4919 }
4920
kvm_are_all_memslots_empty(struct kvm * kvm)4921 bool kvm_are_all_memslots_empty(struct kvm *kvm)
4922 {
4923 int i;
4924
4925 lockdep_assert_held(&kvm->slots_lock);
4926
4927 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
4928 if (!kvm_memslots_empty(__kvm_memslots(kvm, i)))
4929 return false;
4930 }
4931
4932 return true;
4933 }
4934 EXPORT_SYMBOL_GPL(kvm_are_all_memslots_empty);
4935
kvm_vm_ioctl_enable_cap_generic(struct kvm * kvm,struct kvm_enable_cap * cap)4936 static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
4937 struct kvm_enable_cap *cap)
4938 {
4939 switch (cap->cap) {
4940 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
4941 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: {
4942 u64 allowed_options = KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE;
4943
4944 if (cap->args[0] & KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE)
4945 allowed_options = KVM_DIRTY_LOG_MANUAL_CAPS;
4946
4947 if (cap->flags || (cap->args[0] & ~allowed_options))
4948 return -EINVAL;
4949 kvm->manual_dirty_log_protect = cap->args[0];
4950 return 0;
4951 }
4952 #endif
4953 case KVM_CAP_HALT_POLL: {
4954 if (cap->flags || cap->args[0] != (unsigned int)cap->args[0])
4955 return -EINVAL;
4956
4957 kvm->max_halt_poll_ns = cap->args[0];
4958
4959 /*
4960 * Ensure kvm->override_halt_poll_ns does not become visible
4961 * before kvm->max_halt_poll_ns.
4962 *
4963 * Pairs with the smp_rmb() in kvm_vcpu_max_halt_poll_ns().
4964 */
4965 smp_wmb();
4966 kvm->override_halt_poll_ns = true;
4967
4968 return 0;
4969 }
4970 case KVM_CAP_DIRTY_LOG_RING:
4971 case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
4972 if (!kvm_vm_ioctl_check_extension_generic(kvm, cap->cap))
4973 return -EINVAL;
4974
4975 return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);
4976 case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP: {
4977 int r = -EINVAL;
4978
4979 if (!IS_ENABLED(CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP) ||
4980 !kvm->dirty_ring_size || cap->flags)
4981 return r;
4982
4983 mutex_lock(&kvm->slots_lock);
4984
4985 /*
4986 * For simplicity, allow enabling ring+bitmap if and only if
4987 * there are no memslots, e.g. to ensure all memslots allocate
4988 * a bitmap after the capability is enabled.
4989 */
4990 if (kvm_are_all_memslots_empty(kvm)) {
4991 kvm->dirty_ring_with_bitmap = true;
4992 r = 0;
4993 }
4994
4995 mutex_unlock(&kvm->slots_lock);
4996
4997 return r;
4998 }
4999 default:
5000 return kvm_vm_ioctl_enable_cap(kvm, cap);
5001 }
5002 }
5003
kvm_vm_stats_read(struct file * file,char __user * user_buffer,size_t size,loff_t * offset)5004 static ssize_t kvm_vm_stats_read(struct file *file, char __user *user_buffer,
5005 size_t size, loff_t *offset)
5006 {
5007 struct kvm *kvm = file->private_data;
5008
5009 return kvm_stats_read(kvm->stats_id, &kvm_vm_stats_header,
5010 &kvm_vm_stats_desc[0], &kvm->stat,
5011 sizeof(kvm->stat), user_buffer, size, offset);
5012 }
5013
kvm_vm_stats_release(struct inode * inode,struct file * file)5014 static int kvm_vm_stats_release(struct inode *inode, struct file *file)
5015 {
5016 struct kvm *kvm = file->private_data;
5017
5018 kvm_put_kvm(kvm);
5019 return 0;
5020 }
5021
5022 static const struct file_operations kvm_vm_stats_fops = {
5023 .owner = THIS_MODULE,
5024 .read = kvm_vm_stats_read,
5025 .release = kvm_vm_stats_release,
5026 .llseek = noop_llseek,
5027 };
5028
kvm_vm_ioctl_get_stats_fd(struct kvm * kvm)5029 static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm)
5030 {
5031 int fd;
5032 struct file *file;
5033
5034 fd = get_unused_fd_flags(O_CLOEXEC);
5035 if (fd < 0)
5036 return fd;
5037
5038 file = anon_inode_getfile_fmode("kvm-vm-stats",
5039 &kvm_vm_stats_fops, kvm, O_RDONLY, FMODE_PREAD);
5040 if (IS_ERR(file)) {
5041 put_unused_fd(fd);
5042 return PTR_ERR(file);
5043 }
5044
5045 kvm_get_kvm(kvm);
5046 fd_install(fd, file);
5047
5048 return fd;
5049 }
5050
5051 #define SANITY_CHECK_MEM_REGION_FIELD(field) \
5052 do { \
5053 BUILD_BUG_ON(offsetof(struct kvm_userspace_memory_region, field) != \
5054 offsetof(struct kvm_userspace_memory_region2, field)); \
5055 BUILD_BUG_ON(sizeof_field(struct kvm_userspace_memory_region, field) != \
5056 sizeof_field(struct kvm_userspace_memory_region2, field)); \
5057 } while (0)
5058
kvm_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5059 static long kvm_vm_ioctl(struct file *filp,
5060 unsigned int ioctl, unsigned long arg)
5061 {
5062 struct kvm *kvm = filp->private_data;
5063 void __user *argp = (void __user *)arg;
5064 int r;
5065
5066 if (kvm->mm != current->mm || kvm->vm_dead)
5067 return -EIO;
5068 switch (ioctl) {
5069 case KVM_CREATE_VCPU:
5070 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
5071 break;
5072 case KVM_ENABLE_CAP: {
5073 struct kvm_enable_cap cap;
5074
5075 r = -EFAULT;
5076 if (copy_from_user(&cap, argp, sizeof(cap)))
5077 goto out;
5078 r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
5079 break;
5080 }
5081 case KVM_SET_USER_MEMORY_REGION2:
5082 case KVM_SET_USER_MEMORY_REGION: {
5083 struct kvm_userspace_memory_region2 mem;
5084 unsigned long size;
5085
5086 if (ioctl == KVM_SET_USER_MEMORY_REGION) {
5087 /*
5088 * Fields beyond struct kvm_userspace_memory_region shouldn't be
5089 * accessed, but avoid leaking kernel memory in case of a bug.
5090 */
5091 memset(&mem, 0, sizeof(mem));
5092 size = sizeof(struct kvm_userspace_memory_region);
5093 } else {
5094 size = sizeof(struct kvm_userspace_memory_region2);
5095 }
5096
5097 /* Ensure the common parts of the two structs are identical. */
5098 SANITY_CHECK_MEM_REGION_FIELD(slot);
5099 SANITY_CHECK_MEM_REGION_FIELD(flags);
5100 SANITY_CHECK_MEM_REGION_FIELD(guest_phys_addr);
5101 SANITY_CHECK_MEM_REGION_FIELD(memory_size);
5102 SANITY_CHECK_MEM_REGION_FIELD(userspace_addr);
5103
5104 r = -EFAULT;
5105 if (copy_from_user(&mem, argp, size))
5106 goto out;
5107
5108 r = -EINVAL;
5109 if (ioctl == KVM_SET_USER_MEMORY_REGION &&
5110 (mem.flags & ~KVM_SET_USER_MEMORY_REGION_V1_FLAGS))
5111 goto out;
5112
5113 r = kvm_vm_ioctl_set_memory_region(kvm, &mem);
5114 break;
5115 }
5116 case KVM_GET_DIRTY_LOG: {
5117 struct kvm_dirty_log log;
5118
5119 r = -EFAULT;
5120 if (copy_from_user(&log, argp, sizeof(log)))
5121 goto out;
5122 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
5123 break;
5124 }
5125 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
5126 case KVM_CLEAR_DIRTY_LOG: {
5127 struct kvm_clear_dirty_log log;
5128
5129 r = -EFAULT;
5130 if (copy_from_user(&log, argp, sizeof(log)))
5131 goto out;
5132 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
5133 break;
5134 }
5135 #endif
5136 #ifdef CONFIG_KVM_MMIO
5137 case KVM_REGISTER_COALESCED_MMIO: {
5138 struct kvm_coalesced_mmio_zone zone;
5139
5140 r = -EFAULT;
5141 if (copy_from_user(&zone, argp, sizeof(zone)))
5142 goto out;
5143 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
5144 break;
5145 }
5146 case KVM_UNREGISTER_COALESCED_MMIO: {
5147 struct kvm_coalesced_mmio_zone zone;
5148
5149 r = -EFAULT;
5150 if (copy_from_user(&zone, argp, sizeof(zone)))
5151 goto out;
5152 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
5153 break;
5154 }
5155 #endif
5156 case KVM_IRQFD: {
5157 struct kvm_irqfd data;
5158
5159 r = -EFAULT;
5160 if (copy_from_user(&data, argp, sizeof(data)))
5161 goto out;
5162 r = kvm_irqfd(kvm, &data);
5163 break;
5164 }
5165 case KVM_IOEVENTFD: {
5166 struct kvm_ioeventfd data;
5167
5168 r = -EFAULT;
5169 if (copy_from_user(&data, argp, sizeof(data)))
5170 goto out;
5171 r = kvm_ioeventfd(kvm, &data);
5172 break;
5173 }
5174 #ifdef CONFIG_HAVE_KVM_MSI
5175 case KVM_SIGNAL_MSI: {
5176 struct kvm_msi msi;
5177
5178 r = -EFAULT;
5179 if (copy_from_user(&msi, argp, sizeof(msi)))
5180 goto out;
5181 r = kvm_send_userspace_msi(kvm, &msi);
5182 break;
5183 }
5184 #endif
5185 #ifdef __KVM_HAVE_IRQ_LINE
5186 case KVM_IRQ_LINE_STATUS:
5187 case KVM_IRQ_LINE: {
5188 struct kvm_irq_level irq_event;
5189
5190 r = -EFAULT;
5191 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
5192 goto out;
5193
5194 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
5195 ioctl == KVM_IRQ_LINE_STATUS);
5196 if (r)
5197 goto out;
5198
5199 r = -EFAULT;
5200 if (ioctl == KVM_IRQ_LINE_STATUS) {
5201 if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
5202 goto out;
5203 }
5204
5205 r = 0;
5206 break;
5207 }
5208 #endif
5209 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
5210 case KVM_SET_GSI_ROUTING: {
5211 struct kvm_irq_routing routing;
5212 struct kvm_irq_routing __user *urouting;
5213 struct kvm_irq_routing_entry *entries = NULL;
5214
5215 r = -EFAULT;
5216 if (copy_from_user(&routing, argp, sizeof(routing)))
5217 goto out;
5218 r = -EINVAL;
5219 if (!kvm_arch_can_set_irq_routing(kvm))
5220 goto out;
5221 if (routing.nr > KVM_MAX_IRQ_ROUTES)
5222 goto out;
5223 if (routing.flags)
5224 goto out;
5225 if (routing.nr) {
5226 urouting = argp;
5227 entries = vmemdup_array_user(urouting->entries,
5228 routing.nr, sizeof(*entries));
5229 if (IS_ERR(entries)) {
5230 r = PTR_ERR(entries);
5231 goto out;
5232 }
5233 }
5234 r = kvm_set_irq_routing(kvm, entries, routing.nr,
5235 routing.flags);
5236 kvfree(entries);
5237 break;
5238 }
5239 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
5240 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
5241 case KVM_SET_MEMORY_ATTRIBUTES: {
5242 struct kvm_memory_attributes attrs;
5243
5244 r = -EFAULT;
5245 if (copy_from_user(&attrs, argp, sizeof(attrs)))
5246 goto out;
5247
5248 r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs);
5249 break;
5250 }
5251 #endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
5252 case KVM_CREATE_DEVICE: {
5253 struct kvm_create_device cd;
5254
5255 r = -EFAULT;
5256 if (copy_from_user(&cd, argp, sizeof(cd)))
5257 goto out;
5258
5259 r = kvm_ioctl_create_device(kvm, &cd);
5260 if (r)
5261 goto out;
5262
5263 r = -EFAULT;
5264 if (copy_to_user(argp, &cd, sizeof(cd)))
5265 goto out;
5266
5267 r = 0;
5268 break;
5269 }
5270 case KVM_CHECK_EXTENSION:
5271 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
5272 break;
5273 case KVM_RESET_DIRTY_RINGS:
5274 r = kvm_vm_ioctl_reset_dirty_pages(kvm);
5275 break;
5276 case KVM_GET_STATS_FD:
5277 r = kvm_vm_ioctl_get_stats_fd(kvm);
5278 break;
5279 #ifdef CONFIG_KVM_PRIVATE_MEM
5280 case KVM_CREATE_GUEST_MEMFD: {
5281 struct kvm_create_guest_memfd guest_memfd;
5282
5283 r = -EFAULT;
5284 if (copy_from_user(&guest_memfd, argp, sizeof(guest_memfd)))
5285 goto out;
5286
5287 r = kvm_gmem_create(kvm, &guest_memfd);
5288 break;
5289 }
5290 #endif
5291 default:
5292 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
5293 }
5294 out:
5295 return r;
5296 }
5297
5298 #ifdef CONFIG_KVM_COMPAT
5299 struct compat_kvm_dirty_log {
5300 __u32 slot;
5301 __u32 padding1;
5302 union {
5303 compat_uptr_t dirty_bitmap; /* one bit per page */
5304 __u64 padding2;
5305 };
5306 };
5307
5308 struct compat_kvm_clear_dirty_log {
5309 __u32 slot;
5310 __u32 num_pages;
5311 __u64 first_page;
5312 union {
5313 compat_uptr_t dirty_bitmap; /* one bit per page */
5314 __u64 padding2;
5315 };
5316 };
5317
kvm_arch_vm_compat_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5318 long __weak kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl,
5319 unsigned long arg)
5320 {
5321 return -ENOTTY;
5322 }
5323
kvm_vm_compat_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5324 static long kvm_vm_compat_ioctl(struct file *filp,
5325 unsigned int ioctl, unsigned long arg)
5326 {
5327 struct kvm *kvm = filp->private_data;
5328 int r;
5329
5330 if (kvm->mm != current->mm || kvm->vm_dead)
5331 return -EIO;
5332
5333 r = kvm_arch_vm_compat_ioctl(filp, ioctl, arg);
5334 if (r != -ENOTTY)
5335 return r;
5336
5337 switch (ioctl) {
5338 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
5339 case KVM_CLEAR_DIRTY_LOG: {
5340 struct compat_kvm_clear_dirty_log compat_log;
5341 struct kvm_clear_dirty_log log;
5342
5343 if (copy_from_user(&compat_log, (void __user *)arg,
5344 sizeof(compat_log)))
5345 return -EFAULT;
5346 log.slot = compat_log.slot;
5347 log.num_pages = compat_log.num_pages;
5348 log.first_page = compat_log.first_page;
5349 log.padding2 = compat_log.padding2;
5350 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
5351
5352 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
5353 break;
5354 }
5355 #endif
5356 case KVM_GET_DIRTY_LOG: {
5357 struct compat_kvm_dirty_log compat_log;
5358 struct kvm_dirty_log log;
5359
5360 if (copy_from_user(&compat_log, (void __user *)arg,
5361 sizeof(compat_log)))
5362 return -EFAULT;
5363 log.slot = compat_log.slot;
5364 log.padding1 = compat_log.padding1;
5365 log.padding2 = compat_log.padding2;
5366 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
5367
5368 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
5369 break;
5370 }
5371 default:
5372 r = kvm_vm_ioctl(filp, ioctl, arg);
5373 }
5374 return r;
5375 }
5376 #endif
5377
5378 static struct file_operations kvm_vm_fops = {
5379 .release = kvm_vm_release,
5380 .unlocked_ioctl = kvm_vm_ioctl,
5381 .llseek = noop_llseek,
5382 KVM_COMPAT(kvm_vm_compat_ioctl),
5383 };
5384
file_is_kvm(struct file * file)5385 bool file_is_kvm(struct file *file)
5386 {
5387 return file && file->f_op == &kvm_vm_fops;
5388 }
5389 EXPORT_SYMBOL_GPL(file_is_kvm);
5390
kvm_dev_ioctl_create_vm(unsigned long type)5391 static int kvm_dev_ioctl_create_vm(unsigned long type)
5392 {
5393 char fdname[ITOA_MAX_LEN + 1];
5394 int r, fd;
5395 struct kvm *kvm;
5396 struct file *file;
5397
5398 fd = get_unused_fd_flags(O_CLOEXEC);
5399 if (fd < 0)
5400 return fd;
5401
5402 snprintf(fdname, sizeof(fdname), "%d", fd);
5403
5404 kvm = kvm_create_vm(type, fdname);
5405 if (IS_ERR(kvm)) {
5406 r = PTR_ERR(kvm);
5407 goto put_fd;
5408 }
5409
5410 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
5411 if (IS_ERR(file)) {
5412 r = PTR_ERR(file);
5413 goto put_kvm;
5414 }
5415
5416 /*
5417 * Don't call kvm_put_kvm anymore at this point; file->f_op is
5418 * already set, with ->release() being kvm_vm_release(). In error
5419 * cases it will be called by the final fput(file) and will take
5420 * care of doing kvm_put_kvm(kvm).
5421 */
5422 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
5423
5424 fd_install(fd, file);
5425 return fd;
5426
5427 put_kvm:
5428 kvm_put_kvm(kvm);
5429 put_fd:
5430 put_unused_fd(fd);
5431 return r;
5432 }
5433
kvm_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5434 static long kvm_dev_ioctl(struct file *filp,
5435 unsigned int ioctl, unsigned long arg)
5436 {
5437 int r = -EINVAL;
5438
5439 switch (ioctl) {
5440 case KVM_GET_API_VERSION:
5441 if (arg)
5442 goto out;
5443 r = KVM_API_VERSION;
5444 break;
5445 case KVM_CREATE_VM:
5446 r = kvm_dev_ioctl_create_vm(arg);
5447 break;
5448 case KVM_CHECK_EXTENSION:
5449 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
5450 break;
5451 case KVM_GET_VCPU_MMAP_SIZE:
5452 if (arg)
5453 goto out;
5454 r = PAGE_SIZE; /* struct kvm_run */
5455 #ifdef CONFIG_X86
5456 r += PAGE_SIZE; /* pio data page */
5457 #endif
5458 #ifdef CONFIG_KVM_MMIO
5459 r += PAGE_SIZE; /* coalesced mmio ring page */
5460 #endif
5461 break;
5462 default:
5463 return kvm_arch_dev_ioctl(filp, ioctl, arg);
5464 }
5465 out:
5466 return r;
5467 }
5468
5469 static struct file_operations kvm_chardev_ops = {
5470 .unlocked_ioctl = kvm_dev_ioctl,
5471 .llseek = noop_llseek,
5472 KVM_COMPAT(kvm_dev_ioctl),
5473 };
5474
5475 static struct miscdevice kvm_dev = {
5476 KVM_MINOR,
5477 "kvm",
5478 &kvm_chardev_ops,
5479 };
5480
5481 #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
5482 static bool enable_virt_at_load = true;
5483 module_param(enable_virt_at_load, bool, 0444);
5484
5485 __visible bool kvm_rebooting;
5486 EXPORT_SYMBOL_GPL(kvm_rebooting);
5487
5488 static DEFINE_PER_CPU(bool, virtualization_enabled);
5489 static DEFINE_MUTEX(kvm_usage_lock);
5490 static int kvm_usage_count;
5491
kvm_arch_enable_virtualization(void)5492 __weak void kvm_arch_enable_virtualization(void)
5493 {
5494
5495 }
5496
kvm_arch_disable_virtualization(void)5497 __weak void kvm_arch_disable_virtualization(void)
5498 {
5499
5500 }
5501
kvm_enable_virtualization_cpu(void)5502 static int kvm_enable_virtualization_cpu(void)
5503 {
5504 if (__this_cpu_read(virtualization_enabled))
5505 return 0;
5506
5507 if (kvm_arch_enable_virtualization_cpu()) {
5508 pr_info("kvm: enabling virtualization on CPU%d failed\n",
5509 raw_smp_processor_id());
5510 return -EIO;
5511 }
5512
5513 __this_cpu_write(virtualization_enabled, true);
5514 return 0;
5515 }
5516
kvm_online_cpu(unsigned int cpu)5517 static int kvm_online_cpu(unsigned int cpu)
5518 {
5519 /*
5520 * Abort the CPU online process if hardware virtualization cannot
5521 * be enabled. Otherwise running VMs would encounter unrecoverable
5522 * errors when scheduled to this CPU.
5523 */
5524 return kvm_enable_virtualization_cpu();
5525 }
5526
kvm_disable_virtualization_cpu(void * ign)5527 static void kvm_disable_virtualization_cpu(void *ign)
5528 {
5529 if (!__this_cpu_read(virtualization_enabled))
5530 return;
5531
5532 kvm_arch_disable_virtualization_cpu();
5533
5534 __this_cpu_write(virtualization_enabled, false);
5535 }
5536
kvm_offline_cpu(unsigned int cpu)5537 static int kvm_offline_cpu(unsigned int cpu)
5538 {
5539 kvm_disable_virtualization_cpu(NULL);
5540 return 0;
5541 }
5542
kvm_shutdown(void)5543 static void kvm_shutdown(void)
5544 {
5545 /*
5546 * Disable hardware virtualization and set kvm_rebooting to indicate
5547 * that KVM has asynchronously disabled hardware virtualization, i.e.
5548 * that relevant errors and exceptions aren't entirely unexpected.
5549 * Some flavors of hardware virtualization need to be disabled before
5550 * transferring control to firmware (to perform shutdown/reboot), e.g.
5551 * on x86, virtualization can block INIT interrupts, which are used by
5552 * firmware to pull APs back under firmware control. Note, this path
5553 * is used for both shutdown and reboot scenarios, i.e. neither name is
5554 * 100% comprehensive.
5555 */
5556 pr_info("kvm: exiting hardware virtualization\n");
5557 kvm_rebooting = true;
5558 on_each_cpu(kvm_disable_virtualization_cpu, NULL, 1);
5559 }
5560
kvm_suspend(void)5561 static int kvm_suspend(void)
5562 {
5563 /*
5564 * Secondary CPUs and CPU hotplug are disabled across the suspend/resume
5565 * callbacks, i.e. no need to acquire kvm_usage_lock to ensure the usage
5566 * count is stable. Assert that kvm_usage_lock is not held to ensure
5567 * the system isn't suspended while KVM is enabling hardware. Hardware
5568 * enabling can be preempted, but the task cannot be frozen until it has
5569 * dropped all locks (userspace tasks are frozen via a fake signal).
5570 */
5571 lockdep_assert_not_held(&kvm_usage_lock);
5572 lockdep_assert_irqs_disabled();
5573
5574 kvm_disable_virtualization_cpu(NULL);
5575 return 0;
5576 }
5577
kvm_resume(void)5578 static void kvm_resume(void)
5579 {
5580 lockdep_assert_not_held(&kvm_usage_lock);
5581 lockdep_assert_irqs_disabled();
5582
5583 WARN_ON_ONCE(kvm_enable_virtualization_cpu());
5584 }
5585
5586 static struct syscore_ops kvm_syscore_ops = {
5587 .suspend = kvm_suspend,
5588 .resume = kvm_resume,
5589 .shutdown = kvm_shutdown,
5590 };
5591
kvm_enable_virtualization(void)5592 static int kvm_enable_virtualization(void)
5593 {
5594 int r;
5595
5596 guard(mutex)(&kvm_usage_lock);
5597
5598 if (kvm_usage_count++)
5599 return 0;
5600
5601 kvm_arch_enable_virtualization();
5602
5603 r = cpuhp_setup_state(CPUHP_AP_KVM_ONLINE, "kvm/cpu:online",
5604 kvm_online_cpu, kvm_offline_cpu);
5605 if (r)
5606 goto err_cpuhp;
5607
5608 register_syscore_ops(&kvm_syscore_ops);
5609
5610 /*
5611 * Undo virtualization enabling and bail if the system is going down.
5612 * If userspace initiated a forced reboot, e.g. reboot -f, then it's
5613 * possible for an in-flight operation to enable virtualization after
5614 * syscore_shutdown() is called, i.e. without kvm_shutdown() being
5615 * invoked. Note, this relies on system_state being set _before_
5616 * kvm_shutdown(), e.g. to ensure either kvm_shutdown() is invoked
5617 * or this CPU observes the impending shutdown. Which is why KVM uses
5618 * a syscore ops hook instead of registering a dedicated reboot
5619 * notifier (the latter runs before system_state is updated).
5620 */
5621 if (system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF ||
5622 system_state == SYSTEM_RESTART) {
5623 r = -EBUSY;
5624 goto err_rebooting;
5625 }
5626
5627 return 0;
5628
5629 err_rebooting:
5630 unregister_syscore_ops(&kvm_syscore_ops);
5631 cpuhp_remove_state(CPUHP_AP_KVM_ONLINE);
5632 err_cpuhp:
5633 kvm_arch_disable_virtualization();
5634 --kvm_usage_count;
5635 return r;
5636 }
5637
kvm_disable_virtualization(void)5638 static void kvm_disable_virtualization(void)
5639 {
5640 guard(mutex)(&kvm_usage_lock);
5641
5642 if (--kvm_usage_count)
5643 return;
5644
5645 unregister_syscore_ops(&kvm_syscore_ops);
5646 cpuhp_remove_state(CPUHP_AP_KVM_ONLINE);
5647 kvm_arch_disable_virtualization();
5648 }
5649
kvm_init_virtualization(void)5650 static int kvm_init_virtualization(void)
5651 {
5652 if (enable_virt_at_load)
5653 return kvm_enable_virtualization();
5654
5655 return 0;
5656 }
5657
kvm_uninit_virtualization(void)5658 static void kvm_uninit_virtualization(void)
5659 {
5660 if (enable_virt_at_load)
5661 kvm_disable_virtualization();
5662 }
5663 #else /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */
kvm_enable_virtualization(void)5664 static int kvm_enable_virtualization(void)
5665 {
5666 return 0;
5667 }
5668
kvm_init_virtualization(void)5669 static int kvm_init_virtualization(void)
5670 {
5671 return 0;
5672 }
5673
kvm_disable_virtualization(void)5674 static void kvm_disable_virtualization(void)
5675 {
5676
5677 }
5678
kvm_uninit_virtualization(void)5679 static void kvm_uninit_virtualization(void)
5680 {
5681
5682 }
5683 #endif /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */
5684
kvm_iodevice_destructor(struct kvm_io_device * dev)5685 static void kvm_iodevice_destructor(struct kvm_io_device *dev)
5686 {
5687 if (dev->ops->destructor)
5688 dev->ops->destructor(dev);
5689 }
5690
kvm_io_bus_destroy(struct kvm_io_bus * bus)5691 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
5692 {
5693 int i;
5694
5695 for (i = 0; i < bus->dev_count; i++) {
5696 struct kvm_io_device *pos = bus->range[i].dev;
5697
5698 kvm_iodevice_destructor(pos);
5699 }
5700 kfree(bus);
5701 }
5702
kvm_io_bus_cmp(const struct kvm_io_range * r1,const struct kvm_io_range * r2)5703 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
5704 const struct kvm_io_range *r2)
5705 {
5706 gpa_t addr1 = r1->addr;
5707 gpa_t addr2 = r2->addr;
5708
5709 if (addr1 < addr2)
5710 return -1;
5711
5712 /* If r2->len == 0, match the exact address. If r2->len != 0,
5713 * accept any overlapping write. Any order is acceptable for
5714 * overlapping ranges, because kvm_io_bus_get_first_dev ensures
5715 * we process all of them.
5716 */
5717 if (r2->len) {
5718 addr1 += r1->len;
5719 addr2 += r2->len;
5720 }
5721
5722 if (addr1 > addr2)
5723 return 1;
5724
5725 return 0;
5726 }
5727
kvm_io_bus_sort_cmp(const void * p1,const void * p2)5728 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
5729 {
5730 return kvm_io_bus_cmp(p1, p2);
5731 }
5732
kvm_io_bus_get_first_dev(struct kvm_io_bus * bus,gpa_t addr,int len)5733 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
5734 gpa_t addr, int len)
5735 {
5736 struct kvm_io_range *range, key;
5737 int off;
5738
5739 key = (struct kvm_io_range) {
5740 .addr = addr,
5741 .len = len,
5742 };
5743
5744 range = bsearch(&key, bus->range, bus->dev_count,
5745 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
5746 if (range == NULL)
5747 return -ENOENT;
5748
5749 off = range - bus->range;
5750
5751 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
5752 off--;
5753
5754 return off;
5755 }
5756
__kvm_io_bus_write(struct kvm_vcpu * vcpu,struct kvm_io_bus * bus,struct kvm_io_range * range,const void * val)5757 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
5758 struct kvm_io_range *range, const void *val)
5759 {
5760 int idx;
5761
5762 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
5763 if (idx < 0)
5764 return -EOPNOTSUPP;
5765
5766 while (idx < bus->dev_count &&
5767 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
5768 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
5769 range->len, val))
5770 return idx;
5771 idx++;
5772 }
5773
5774 return -EOPNOTSUPP;
5775 }
5776
5777 /* kvm_io_bus_write - called under kvm->slots_lock */
kvm_io_bus_write(struct kvm_vcpu * vcpu,enum kvm_bus bus_idx,gpa_t addr,int len,const void * val)5778 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
5779 int len, const void *val)
5780 {
5781 struct kvm_io_bus *bus;
5782 struct kvm_io_range range;
5783 int r;
5784
5785 range = (struct kvm_io_range) {
5786 .addr = addr,
5787 .len = len,
5788 };
5789
5790 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
5791 if (!bus)
5792 return -ENOMEM;
5793 r = __kvm_io_bus_write(vcpu, bus, &range, val);
5794 return r < 0 ? r : 0;
5795 }
5796 EXPORT_SYMBOL_GPL(kvm_io_bus_write);
5797
5798 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
kvm_io_bus_write_cookie(struct kvm_vcpu * vcpu,enum kvm_bus bus_idx,gpa_t addr,int len,const void * val,long cookie)5799 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
5800 gpa_t addr, int len, const void *val, long cookie)
5801 {
5802 struct kvm_io_bus *bus;
5803 struct kvm_io_range range;
5804
5805 range = (struct kvm_io_range) {
5806 .addr = addr,
5807 .len = len,
5808 };
5809
5810 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
5811 if (!bus)
5812 return -ENOMEM;
5813
5814 /* First try the device referenced by cookie. */
5815 if ((cookie >= 0) && (cookie < bus->dev_count) &&
5816 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
5817 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
5818 val))
5819 return cookie;
5820
5821 /*
5822 * cookie contained garbage; fall back to search and return the
5823 * correct cookie value.
5824 */
5825 return __kvm_io_bus_write(vcpu, bus, &range, val);
5826 }
5827
__kvm_io_bus_read(struct kvm_vcpu * vcpu,struct kvm_io_bus * bus,struct kvm_io_range * range,void * val)5828 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
5829 struct kvm_io_range *range, void *val)
5830 {
5831 int idx;
5832
5833 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
5834 if (idx < 0)
5835 return -EOPNOTSUPP;
5836
5837 while (idx < bus->dev_count &&
5838 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
5839 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
5840 range->len, val))
5841 return idx;
5842 idx++;
5843 }
5844
5845 return -EOPNOTSUPP;
5846 }
5847
5848 /* kvm_io_bus_read - called under kvm->slots_lock */
kvm_io_bus_read(struct kvm_vcpu * vcpu,enum kvm_bus bus_idx,gpa_t addr,int len,void * val)5849 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
5850 int len, void *val)
5851 {
5852 struct kvm_io_bus *bus;
5853 struct kvm_io_range range;
5854 int r;
5855
5856 range = (struct kvm_io_range) {
5857 .addr = addr,
5858 .len = len,
5859 };
5860
5861 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
5862 if (!bus)
5863 return -ENOMEM;
5864 r = __kvm_io_bus_read(vcpu, bus, &range, val);
5865 return r < 0 ? r : 0;
5866 }
5867
kvm_io_bus_register_dev(struct kvm * kvm,enum kvm_bus bus_idx,gpa_t addr,int len,struct kvm_io_device * dev)5868 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
5869 int len, struct kvm_io_device *dev)
5870 {
5871 int i;
5872 struct kvm_io_bus *new_bus, *bus;
5873 struct kvm_io_range range;
5874
5875 lockdep_assert_held(&kvm->slots_lock);
5876
5877 bus = kvm_get_bus(kvm, bus_idx);
5878 if (!bus)
5879 return -ENOMEM;
5880
5881 /* exclude ioeventfd which is limited by maximum fd */
5882 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
5883 return -ENOSPC;
5884
5885 new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
5886 GFP_KERNEL_ACCOUNT);
5887 if (!new_bus)
5888 return -ENOMEM;
5889
5890 range = (struct kvm_io_range) {
5891 .addr = addr,
5892 .len = len,
5893 .dev = dev,
5894 };
5895
5896 for (i = 0; i < bus->dev_count; i++)
5897 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
5898 break;
5899
5900 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
5901 new_bus->dev_count++;
5902 new_bus->range[i] = range;
5903 memcpy(new_bus->range + i + 1, bus->range + i,
5904 (bus->dev_count - i) * sizeof(struct kvm_io_range));
5905 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
5906 synchronize_srcu_expedited(&kvm->srcu);
5907 kfree(bus);
5908
5909 return 0;
5910 }
5911
kvm_io_bus_unregister_dev(struct kvm * kvm,enum kvm_bus bus_idx,struct kvm_io_device * dev)5912 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
5913 struct kvm_io_device *dev)
5914 {
5915 int i;
5916 struct kvm_io_bus *new_bus, *bus;
5917
5918 lockdep_assert_held(&kvm->slots_lock);
5919
5920 bus = kvm_get_bus(kvm, bus_idx);
5921 if (!bus)
5922 return 0;
5923
5924 for (i = 0; i < bus->dev_count; i++) {
5925 if (bus->range[i].dev == dev) {
5926 break;
5927 }
5928 }
5929
5930 if (i == bus->dev_count)
5931 return 0;
5932
5933 new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
5934 GFP_KERNEL_ACCOUNT);
5935 if (new_bus) {
5936 memcpy(new_bus, bus, struct_size(bus, range, i));
5937 new_bus->dev_count--;
5938 memcpy(new_bus->range + i, bus->range + i + 1,
5939 flex_array_size(new_bus, range, new_bus->dev_count - i));
5940 }
5941
5942 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
5943 synchronize_srcu_expedited(&kvm->srcu);
5944
5945 /*
5946 * If NULL bus is installed, destroy the old bus, including all the
5947 * attached devices. Otherwise, destroy the caller's device only.
5948 */
5949 if (!new_bus) {
5950 pr_err("kvm: failed to shrink bus, removing it completely\n");
5951 kvm_io_bus_destroy(bus);
5952 return -ENOMEM;
5953 }
5954
5955 kvm_iodevice_destructor(dev);
5956 kfree(bus);
5957 return 0;
5958 }
5959
kvm_io_bus_get_dev(struct kvm * kvm,enum kvm_bus bus_idx,gpa_t addr)5960 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
5961 gpa_t addr)
5962 {
5963 struct kvm_io_bus *bus;
5964 int dev_idx, srcu_idx;
5965 struct kvm_io_device *iodev = NULL;
5966
5967 srcu_idx = srcu_read_lock(&kvm->srcu);
5968
5969 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
5970 if (!bus)
5971 goto out_unlock;
5972
5973 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
5974 if (dev_idx < 0)
5975 goto out_unlock;
5976
5977 iodev = bus->range[dev_idx].dev;
5978
5979 out_unlock:
5980 srcu_read_unlock(&kvm->srcu, srcu_idx);
5981
5982 return iodev;
5983 }
5984 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
5985
kvm_debugfs_open(struct inode * inode,struct file * file,int (* get)(void *,u64 *),int (* set)(void *,u64),const char * fmt)5986 static int kvm_debugfs_open(struct inode *inode, struct file *file,
5987 int (*get)(void *, u64 *), int (*set)(void *, u64),
5988 const char *fmt)
5989 {
5990 int ret;
5991 struct kvm_stat_data *stat_data = inode->i_private;
5992
5993 /*
5994 * The debugfs files are a reference to the kvm struct which
5995 * is still valid when kvm_destroy_vm is called. kvm_get_kvm_safe
5996 * avoids the race between open and the removal of the debugfs directory.
5997 */
5998 if (!kvm_get_kvm_safe(stat_data->kvm))
5999 return -ENOENT;
6000
6001 ret = simple_attr_open(inode, file, get,
6002 kvm_stats_debugfs_mode(stat_data->desc) & 0222
6003 ? set : NULL, fmt);
6004 if (ret)
6005 kvm_put_kvm(stat_data->kvm);
6006
6007 return ret;
6008 }
6009
kvm_debugfs_release(struct inode * inode,struct file * file)6010 static int kvm_debugfs_release(struct inode *inode, struct file *file)
6011 {
6012 struct kvm_stat_data *stat_data = inode->i_private;
6013
6014 simple_attr_release(inode, file);
6015 kvm_put_kvm(stat_data->kvm);
6016
6017 return 0;
6018 }
6019
kvm_get_stat_per_vm(struct kvm * kvm,size_t offset,u64 * val)6020 static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val)
6021 {
6022 *val = *(u64 *)((void *)(&kvm->stat) + offset);
6023
6024 return 0;
6025 }
6026
kvm_clear_stat_per_vm(struct kvm * kvm,size_t offset)6027 static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset)
6028 {
6029 *(u64 *)((void *)(&kvm->stat) + offset) = 0;
6030
6031 return 0;
6032 }
6033
kvm_get_stat_per_vcpu(struct kvm * kvm,size_t offset,u64 * val)6034 static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val)
6035 {
6036 unsigned long i;
6037 struct kvm_vcpu *vcpu;
6038
6039 *val = 0;
6040
6041 kvm_for_each_vcpu(i, vcpu, kvm)
6042 *val += *(u64 *)((void *)(&vcpu->stat) + offset);
6043
6044 return 0;
6045 }
6046
kvm_clear_stat_per_vcpu(struct kvm * kvm,size_t offset)6047 static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset)
6048 {
6049 unsigned long i;
6050 struct kvm_vcpu *vcpu;
6051
6052 kvm_for_each_vcpu(i, vcpu, kvm)
6053 *(u64 *)((void *)(&vcpu->stat) + offset) = 0;
6054
6055 return 0;
6056 }
6057
kvm_stat_data_get(void * data,u64 * val)6058 static int kvm_stat_data_get(void *data, u64 *val)
6059 {
6060 int r = -EFAULT;
6061 struct kvm_stat_data *stat_data = data;
6062
6063 switch (stat_data->kind) {
6064 case KVM_STAT_VM:
6065 r = kvm_get_stat_per_vm(stat_data->kvm,
6066 stat_data->desc->desc.offset, val);
6067 break;
6068 case KVM_STAT_VCPU:
6069 r = kvm_get_stat_per_vcpu(stat_data->kvm,
6070 stat_data->desc->desc.offset, val);
6071 break;
6072 }
6073
6074 return r;
6075 }
6076
kvm_stat_data_clear(void * data,u64 val)6077 static int kvm_stat_data_clear(void *data, u64 val)
6078 {
6079 int r = -EFAULT;
6080 struct kvm_stat_data *stat_data = data;
6081
6082 if (val)
6083 return -EINVAL;
6084
6085 switch (stat_data->kind) {
6086 case KVM_STAT_VM:
6087 r = kvm_clear_stat_per_vm(stat_data->kvm,
6088 stat_data->desc->desc.offset);
6089 break;
6090 case KVM_STAT_VCPU:
6091 r = kvm_clear_stat_per_vcpu(stat_data->kvm,
6092 stat_data->desc->desc.offset);
6093 break;
6094 }
6095
6096 return r;
6097 }
6098
kvm_stat_data_open(struct inode * inode,struct file * file)6099 static int kvm_stat_data_open(struct inode *inode, struct file *file)
6100 {
6101 __simple_attr_check_format("%llu\n", 0ull);
6102 return kvm_debugfs_open(inode, file, kvm_stat_data_get,
6103 kvm_stat_data_clear, "%llu\n");
6104 }
6105
6106 static const struct file_operations stat_fops_per_vm = {
6107 .owner = THIS_MODULE,
6108 .open = kvm_stat_data_open,
6109 .release = kvm_debugfs_release,
6110 .read = simple_attr_read,
6111 .write = simple_attr_write,
6112 };
6113
vm_stat_get(void * _offset,u64 * val)6114 static int vm_stat_get(void *_offset, u64 *val)
6115 {
6116 unsigned offset = (long)_offset;
6117 struct kvm *kvm;
6118 u64 tmp_val;
6119
6120 *val = 0;
6121 mutex_lock(&kvm_lock);
6122 list_for_each_entry(kvm, &vm_list, vm_list) {
6123 kvm_get_stat_per_vm(kvm, offset, &tmp_val);
6124 *val += tmp_val;
6125 }
6126 mutex_unlock(&kvm_lock);
6127 return 0;
6128 }
6129
vm_stat_clear(void * _offset,u64 val)6130 static int vm_stat_clear(void *_offset, u64 val)
6131 {
6132 unsigned offset = (long)_offset;
6133 struct kvm *kvm;
6134
6135 if (val)
6136 return -EINVAL;
6137
6138 mutex_lock(&kvm_lock);
6139 list_for_each_entry(kvm, &vm_list, vm_list) {
6140 kvm_clear_stat_per_vm(kvm, offset);
6141 }
6142 mutex_unlock(&kvm_lock);
6143
6144 return 0;
6145 }
6146
6147 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
6148 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_readonly_fops, vm_stat_get, NULL, "%llu\n");
6149
vcpu_stat_get(void * _offset,u64 * val)6150 static int vcpu_stat_get(void *_offset, u64 *val)
6151 {
6152 unsigned offset = (long)_offset;
6153 struct kvm *kvm;
6154 u64 tmp_val;
6155
6156 *val = 0;
6157 mutex_lock(&kvm_lock);
6158 list_for_each_entry(kvm, &vm_list, vm_list) {
6159 kvm_get_stat_per_vcpu(kvm, offset, &tmp_val);
6160 *val += tmp_val;
6161 }
6162 mutex_unlock(&kvm_lock);
6163 return 0;
6164 }
6165
vcpu_stat_clear(void * _offset,u64 val)6166 static int vcpu_stat_clear(void *_offset, u64 val)
6167 {
6168 unsigned offset = (long)_offset;
6169 struct kvm *kvm;
6170
6171 if (val)
6172 return -EINVAL;
6173
6174 mutex_lock(&kvm_lock);
6175 list_for_each_entry(kvm, &vm_list, vm_list) {
6176 kvm_clear_stat_per_vcpu(kvm, offset);
6177 }
6178 mutex_unlock(&kvm_lock);
6179
6180 return 0;
6181 }
6182
6183 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
6184 "%llu\n");
6185 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_readonly_fops, vcpu_stat_get, NULL, "%llu\n");
6186
kvm_uevent_notify_change(unsigned int type,struct kvm * kvm)6187 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
6188 {
6189 struct kobj_uevent_env *env;
6190 unsigned long long created, active;
6191
6192 if (!kvm_dev.this_device || !kvm)
6193 return;
6194
6195 mutex_lock(&kvm_lock);
6196 if (type == KVM_EVENT_CREATE_VM) {
6197 kvm_createvm_count++;
6198 kvm_active_vms++;
6199 } else if (type == KVM_EVENT_DESTROY_VM) {
6200 kvm_active_vms--;
6201 }
6202 created = kvm_createvm_count;
6203 active = kvm_active_vms;
6204 mutex_unlock(&kvm_lock);
6205
6206 env = kzalloc(sizeof(*env), GFP_KERNEL);
6207 if (!env)
6208 return;
6209
6210 add_uevent_var(env, "CREATED=%llu", created);
6211 add_uevent_var(env, "COUNT=%llu", active);
6212
6213 if (type == KVM_EVENT_CREATE_VM) {
6214 add_uevent_var(env, "EVENT=create");
6215 kvm->userspace_pid = task_pid_nr(current);
6216 } else if (type == KVM_EVENT_DESTROY_VM) {
6217 add_uevent_var(env, "EVENT=destroy");
6218 }
6219 add_uevent_var(env, "PID=%d", kvm->userspace_pid);
6220
6221 if (!IS_ERR(kvm->debugfs_dentry)) {
6222 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL);
6223
6224 if (p) {
6225 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
6226 if (!IS_ERR(tmp))
6227 add_uevent_var(env, "STATS_PATH=%s", tmp);
6228 kfree(p);
6229 }
6230 }
6231 /* no need for checks, since we are adding at most only 5 keys */
6232 env->envp[env->envp_idx++] = NULL;
6233 kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
6234 kfree(env);
6235 }
6236
kvm_init_debug(void)6237 static void kvm_init_debug(void)
6238 {
6239 const struct file_operations *fops;
6240 const struct _kvm_stats_desc *pdesc;
6241 int i;
6242
6243 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
6244
6245 for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) {
6246 pdesc = &kvm_vm_stats_desc[i];
6247 if (kvm_stats_debugfs_mode(pdesc) & 0222)
6248 fops = &vm_stat_fops;
6249 else
6250 fops = &vm_stat_readonly_fops;
6251 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
6252 kvm_debugfs_dir,
6253 (void *)(long)pdesc->desc.offset, fops);
6254 }
6255
6256 for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) {
6257 pdesc = &kvm_vcpu_stats_desc[i];
6258 if (kvm_stats_debugfs_mode(pdesc) & 0222)
6259 fops = &vcpu_stat_fops;
6260 else
6261 fops = &vcpu_stat_readonly_fops;
6262 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
6263 kvm_debugfs_dir,
6264 (void *)(long)pdesc->desc.offset, fops);
6265 }
6266 }
6267
6268 static inline
preempt_notifier_to_vcpu(struct preempt_notifier * pn)6269 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
6270 {
6271 return container_of(pn, struct kvm_vcpu, preempt_notifier);
6272 }
6273
kvm_sched_in(struct preempt_notifier * pn,int cpu)6274 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
6275 {
6276 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
6277
6278 WRITE_ONCE(vcpu->preempted, false);
6279 WRITE_ONCE(vcpu->ready, false);
6280
6281 __this_cpu_write(kvm_running_vcpu, vcpu);
6282 kvm_arch_vcpu_load(vcpu, cpu);
6283
6284 WRITE_ONCE(vcpu->scheduled_out, false);
6285 }
6286
kvm_sched_out(struct preempt_notifier * pn,struct task_struct * next)6287 static void kvm_sched_out(struct preempt_notifier *pn,
6288 struct task_struct *next)
6289 {
6290 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
6291
6292 WRITE_ONCE(vcpu->scheduled_out, true);
6293
6294 if (task_is_runnable(current) && vcpu->wants_to_run) {
6295 WRITE_ONCE(vcpu->preempted, true);
6296 WRITE_ONCE(vcpu->ready, true);
6297 }
6298 kvm_arch_vcpu_put(vcpu);
6299 __this_cpu_write(kvm_running_vcpu, NULL);
6300 }
6301
6302 /**
6303 * kvm_get_running_vcpu - get the vcpu running on the current CPU.
6304 *
6305 * We can disable preemption locally around accessing the per-CPU variable,
6306 * and use the resolved vcpu pointer after enabling preemption again,
6307 * because even if the current thread is migrated to another CPU, reading
6308 * the per-CPU value later will give us the same value as we update the
6309 * per-CPU variable in the preempt notifier handlers.
6310 */
kvm_get_running_vcpu(void)6311 struct kvm_vcpu *kvm_get_running_vcpu(void)
6312 {
6313 struct kvm_vcpu *vcpu;
6314
6315 preempt_disable();
6316 vcpu = __this_cpu_read(kvm_running_vcpu);
6317 preempt_enable();
6318
6319 return vcpu;
6320 }
6321 EXPORT_SYMBOL_GPL(kvm_get_running_vcpu);
6322
6323 /**
6324 * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus.
6325 */
kvm_get_running_vcpus(void)6326 struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
6327 {
6328 return &kvm_running_vcpu;
6329 }
6330
6331 #ifdef CONFIG_GUEST_PERF_EVENTS
kvm_guest_state(void)6332 static unsigned int kvm_guest_state(void)
6333 {
6334 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
6335 unsigned int state;
6336
6337 if (!kvm_arch_pmi_in_guest(vcpu))
6338 return 0;
6339
6340 state = PERF_GUEST_ACTIVE;
6341 if (!kvm_arch_vcpu_in_kernel(vcpu))
6342 state |= PERF_GUEST_USER;
6343
6344 return state;
6345 }
6346
kvm_guest_get_ip(void)6347 static unsigned long kvm_guest_get_ip(void)
6348 {
6349 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
6350
6351 /* Retrieving the IP must be guarded by a call to kvm_guest_state(). */
6352 if (WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu)))
6353 return 0;
6354
6355 return kvm_arch_vcpu_get_ip(vcpu);
6356 }
6357
6358 static struct perf_guest_info_callbacks kvm_guest_cbs = {
6359 .state = kvm_guest_state,
6360 .get_ip = kvm_guest_get_ip,
6361 .handle_intel_pt_intr = NULL,
6362 };
6363
kvm_register_perf_callbacks(unsigned int (* pt_intr_handler)(void))6364 void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void))
6365 {
6366 kvm_guest_cbs.handle_intel_pt_intr = pt_intr_handler;
6367 perf_register_guest_info_callbacks(&kvm_guest_cbs);
6368 }
kvm_unregister_perf_callbacks(void)6369 void kvm_unregister_perf_callbacks(void)
6370 {
6371 perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
6372 }
6373 #endif
6374
kvm_init(unsigned vcpu_size,unsigned vcpu_align,struct module * module)6375 int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
6376 {
6377 int r;
6378 int cpu;
6379
6380 /* A kmem cache lets us meet the alignment requirements of fx_save. */
6381 if (!vcpu_align)
6382 vcpu_align = __alignof__(struct kvm_vcpu);
6383 kvm_vcpu_cache =
6384 kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
6385 SLAB_ACCOUNT,
6386 offsetof(struct kvm_vcpu, arch),
6387 offsetofend(struct kvm_vcpu, stats_id)
6388 - offsetof(struct kvm_vcpu, arch),
6389 NULL);
6390 if (!kvm_vcpu_cache)
6391 return -ENOMEM;
6392
6393 for_each_possible_cpu(cpu) {
6394 if (!alloc_cpumask_var_node(&per_cpu(cpu_kick_mask, cpu),
6395 GFP_KERNEL, cpu_to_node(cpu))) {
6396 r = -ENOMEM;
6397 goto err_cpu_kick_mask;
6398 }
6399 }
6400
6401 r = kvm_irqfd_init();
6402 if (r)
6403 goto err_irqfd;
6404
6405 r = kvm_async_pf_init();
6406 if (r)
6407 goto err_async_pf;
6408
6409 kvm_chardev_ops.owner = module;
6410 kvm_vm_fops.owner = module;
6411 kvm_vcpu_fops.owner = module;
6412 kvm_device_fops.owner = module;
6413
6414 kvm_preempt_ops.sched_in = kvm_sched_in;
6415 kvm_preempt_ops.sched_out = kvm_sched_out;
6416
6417 kvm_init_debug();
6418
6419 r = kvm_vfio_ops_init();
6420 if (WARN_ON_ONCE(r))
6421 goto err_vfio;
6422
6423 kvm_gmem_init(module);
6424
6425 r = kvm_init_virtualization();
6426 if (r)
6427 goto err_virt;
6428
6429 /*
6430 * Registration _must_ be the very last thing done, as this exposes
6431 * /dev/kvm to userspace, i.e. all infrastructure must be setup!
6432 */
6433 r = misc_register(&kvm_dev);
6434 if (r) {
6435 pr_err("kvm: misc device register failed\n");
6436 goto err_register;
6437 }
6438
6439 return 0;
6440
6441 err_register:
6442 kvm_uninit_virtualization();
6443 err_virt:
6444 kvm_vfio_ops_exit();
6445 err_vfio:
6446 kvm_async_pf_deinit();
6447 err_async_pf:
6448 kvm_irqfd_exit();
6449 err_irqfd:
6450 err_cpu_kick_mask:
6451 for_each_possible_cpu(cpu)
6452 free_cpumask_var(per_cpu(cpu_kick_mask, cpu));
6453 kmem_cache_destroy(kvm_vcpu_cache);
6454 return r;
6455 }
6456 EXPORT_SYMBOL_GPL(kvm_init);
6457
kvm_exit(void)6458 void kvm_exit(void)
6459 {
6460 int cpu;
6461
6462 /*
6463 * Note, unregistering /dev/kvm doesn't strictly need to come first,
6464 * fops_get(), a.k.a. try_module_get(), prevents acquiring references
6465 * to KVM while the module is being stopped.
6466 */
6467 misc_deregister(&kvm_dev);
6468
6469 kvm_uninit_virtualization();
6470
6471 debugfs_remove_recursive(kvm_debugfs_dir);
6472 for_each_possible_cpu(cpu)
6473 free_cpumask_var(per_cpu(cpu_kick_mask, cpu));
6474 kmem_cache_destroy(kvm_vcpu_cache);
6475 kvm_vfio_ops_exit();
6476 kvm_async_pf_deinit();
6477 kvm_irqfd_exit();
6478 }
6479 EXPORT_SYMBOL_GPL(kvm_exit);
6480