xref: /linux/arch/x86/kernel/kvm.c (revision 621cde16e49b3ecf7d59a8106a20aaebfb4a59a9)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * KVM paravirt_ops implementation
4  *
5  * Copyright (C) 2007, Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
6  * Copyright IBM Corporation, 2007
7  *   Authors: Anthony Liguori <aliguori@us.ibm.com>
8  */
9 
10 #define pr_fmt(fmt) "kvm-guest: " fmt
11 
12 #include <linux/context_tracking.h>
13 #include <linux/init.h>
14 #include <linux/irq.h>
15 #include <linux/kernel.h>
16 #include <linux/kvm_para.h>
17 #include <linux/cpu.h>
18 #include <linux/mm.h>
19 #include <linux/highmem.h>
20 #include <linux/hardirq.h>
21 #include <linux/notifier.h>
22 #include <linux/reboot.h>
23 #include <linux/hash.h>
24 #include <linux/sched.h>
25 #include <linux/slab.h>
26 #include <linux/kprobes.h>
27 #include <linux/nmi.h>
28 #include <linux/swait.h>
29 #include <linux/syscore_ops.h>
30 #include <linux/cc_platform.h>
31 #include <linux/efi.h>
32 #include <asm/timer.h>
33 #include <asm/cpu.h>
34 #include <asm/traps.h>
35 #include <asm/desc.h>
36 #include <asm/tlbflush.h>
37 #include <asm/apic.h>
38 #include <asm/apicdef.h>
39 #include <asm/hypervisor.h>
40 #include <asm/tlb.h>
41 #include <asm/cpuidle_haltpoll.h>
42 #include <asm/ptrace.h>
43 #include <asm/reboot.h>
44 #include <asm/svm.h>
45 #include <asm/e820/api.h>
46 
47 DEFINE_STATIC_KEY_FALSE_RO(kvm_async_pf_enabled);
48 
49 static int kvmapf = 1;
50 
parse_no_kvmapf(char * arg)51 static int __init parse_no_kvmapf(char *arg)
52 {
53         kvmapf = 0;
54         return 0;
55 }
56 
57 early_param("no-kvmapf", parse_no_kvmapf);
58 
59 static int steal_acc = 1;
parse_no_stealacc(char * arg)60 static int __init parse_no_stealacc(char *arg)
61 {
62         steal_acc = 0;
63         return 0;
64 }
65 
66 early_param("no-steal-acc", parse_no_stealacc);
67 
68 static DEFINE_PER_CPU_READ_MOSTLY(bool, async_pf_enabled);
69 static DEFINE_PER_CPU_DECRYPTED(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64);
70 DEFINE_PER_CPU_DECRYPTED(struct kvm_steal_time, steal_time) __aligned(64) __visible;
71 static int has_steal_clock = 0;
72 
73 static int has_guest_poll = 0;
74 /*
75  * No need for any "IO delay" on KVM
76  */
kvm_io_delay(void)77 static void kvm_io_delay(void)
78 {
79 }
80 
81 #define KVM_TASK_SLEEP_HASHBITS 8
82 #define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS)
83 
84 struct kvm_task_sleep_node {
85 	struct hlist_node link;
86 	struct swait_queue_head wq;
87 	u32 token;
88 	int cpu;
89 };
90 
91 static struct kvm_task_sleep_head {
92 	raw_spinlock_t lock;
93 	struct hlist_head list;
94 } async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE];
95 
_find_apf_task(struct kvm_task_sleep_head * b,u32 token)96 static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b,
97 						  u32 token)
98 {
99 	struct hlist_node *p;
100 
101 	hlist_for_each(p, &b->list) {
102 		struct kvm_task_sleep_node *n =
103 			hlist_entry(p, typeof(*n), link);
104 		if (n->token == token)
105 			return n;
106 	}
107 
108 	return NULL;
109 }
110 
kvm_async_pf_queue_task(u32 token,struct kvm_task_sleep_node * n)111 static bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n)
112 {
113 	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
114 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
115 	struct kvm_task_sleep_node *e;
116 
117 	raw_spin_lock(&b->lock);
118 	e = _find_apf_task(b, token);
119 	if (e) {
120 		/* dummy entry exist -> wake up was delivered ahead of PF */
121 		hlist_del(&e->link);
122 		raw_spin_unlock(&b->lock);
123 		kfree(e);
124 		return false;
125 	}
126 
127 	n->token = token;
128 	n->cpu = smp_processor_id();
129 	init_swait_queue_head(&n->wq);
130 	hlist_add_head(&n->link, &b->list);
131 	raw_spin_unlock(&b->lock);
132 	return true;
133 }
134 
135 /*
136  * kvm_async_pf_task_wait_schedule - Wait for pagefault to be handled
137  * @token:	Token to identify the sleep node entry
138  *
139  * Invoked from the async pagefault handling code or from the VM exit page
140  * fault handler. In both cases RCU is watching.
141  */
kvm_async_pf_task_wait_schedule(u32 token)142 void kvm_async_pf_task_wait_schedule(u32 token)
143 {
144 	struct kvm_task_sleep_node n;
145 	DECLARE_SWAITQUEUE(wait);
146 
147 	lockdep_assert_irqs_disabled();
148 
149 	if (!kvm_async_pf_queue_task(token, &n))
150 		return;
151 
152 	for (;;) {
153 		prepare_to_swait_exclusive(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
154 		if (hlist_unhashed(&n.link))
155 			break;
156 
157 		local_irq_enable();
158 		schedule();
159 		local_irq_disable();
160 	}
161 	finish_swait(&n.wq, &wait);
162 }
163 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait_schedule);
164 
apf_task_wake_one(struct kvm_task_sleep_node * n)165 static void apf_task_wake_one(struct kvm_task_sleep_node *n)
166 {
167 	hlist_del_init(&n->link);
168 	if (swq_has_sleeper(&n->wq))
169 		swake_up_one(&n->wq);
170 }
171 
apf_task_wake_all(void)172 static void apf_task_wake_all(void)
173 {
174 	int i;
175 
176 	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) {
177 		struct kvm_task_sleep_head *b = &async_pf_sleepers[i];
178 		struct kvm_task_sleep_node *n;
179 		struct hlist_node *p, *next;
180 
181 		raw_spin_lock(&b->lock);
182 		hlist_for_each_safe(p, next, &b->list) {
183 			n = hlist_entry(p, typeof(*n), link);
184 			if (n->cpu == smp_processor_id())
185 				apf_task_wake_one(n);
186 		}
187 		raw_spin_unlock(&b->lock);
188 	}
189 }
190 
kvm_async_pf_task_wake(u32 token)191 void kvm_async_pf_task_wake(u32 token)
192 {
193 	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
194 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
195 	struct kvm_task_sleep_node *n, *dummy = NULL;
196 
197 	if (token == ~0) {
198 		apf_task_wake_all();
199 		return;
200 	}
201 
202 again:
203 	raw_spin_lock(&b->lock);
204 	n = _find_apf_task(b, token);
205 	if (!n) {
206 		/*
207 		 * Async #PF not yet handled, add a dummy entry for the token.
208 		 * Allocating the token must be down outside of the raw lock
209 		 * as the allocator is preemptible on PREEMPT_RT kernels.
210 		 */
211 		if (!dummy) {
212 			raw_spin_unlock(&b->lock);
213 			dummy = kzalloc(sizeof(*dummy), GFP_ATOMIC);
214 
215 			/*
216 			 * Continue looping on allocation failure, eventually
217 			 * the async #PF will be handled and allocating a new
218 			 * node will be unnecessary.
219 			 */
220 			if (!dummy)
221 				cpu_relax();
222 
223 			/*
224 			 * Recheck for async #PF completion before enqueueing
225 			 * the dummy token to avoid duplicate list entries.
226 			 */
227 			goto again;
228 		}
229 		dummy->token = token;
230 		dummy->cpu = smp_processor_id();
231 		init_swait_queue_head(&dummy->wq);
232 		hlist_add_head(&dummy->link, &b->list);
233 		dummy = NULL;
234 	} else {
235 		apf_task_wake_one(n);
236 	}
237 	raw_spin_unlock(&b->lock);
238 
239 	/* A dummy token might be allocated and ultimately not used.  */
240 	kfree(dummy);
241 }
242 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake);
243 
kvm_read_and_reset_apf_flags(void)244 noinstr u32 kvm_read_and_reset_apf_flags(void)
245 {
246 	u32 flags = 0;
247 
248 	if (__this_cpu_read(async_pf_enabled)) {
249 		flags = __this_cpu_read(apf_reason.flags);
250 		__this_cpu_write(apf_reason.flags, 0);
251 	}
252 
253 	return flags;
254 }
255 EXPORT_SYMBOL_GPL(kvm_read_and_reset_apf_flags);
256 
__kvm_handle_async_pf(struct pt_regs * regs,u32 token)257 noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token)
258 {
259 	u32 flags = kvm_read_and_reset_apf_flags();
260 	irqentry_state_t state;
261 
262 	if (!flags)
263 		return false;
264 
265 	state = irqentry_enter(regs);
266 	instrumentation_begin();
267 
268 	/*
269 	 * If the host managed to inject an async #PF into an interrupt
270 	 * disabled region, then die hard as this is not going to end well
271 	 * and the host side is seriously broken.
272 	 */
273 	if (unlikely(!(regs->flags & X86_EFLAGS_IF)))
274 		panic("Host injected async #PF in interrupt disabled region\n");
275 
276 	if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
277 		if (unlikely(!(user_mode(regs))))
278 			panic("Host injected async #PF in kernel mode\n");
279 		/* Page is swapped out by the host. */
280 		kvm_async_pf_task_wait_schedule(token);
281 	} else {
282 		WARN_ONCE(1, "Unexpected async PF flags: %x\n", flags);
283 	}
284 
285 	instrumentation_end();
286 	irqentry_exit(regs, state);
287 	return true;
288 }
289 
DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt)290 DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt)
291 {
292 	struct pt_regs *old_regs = set_irq_regs(regs);
293 	u32 token;
294 
295 	apic_eoi();
296 
297 	inc_irq_stat(irq_hv_callback_count);
298 
299 	if (__this_cpu_read(async_pf_enabled)) {
300 		token = __this_cpu_read(apf_reason.token);
301 		kvm_async_pf_task_wake(token);
302 		__this_cpu_write(apf_reason.token, 0);
303 		wrmsrl(MSR_KVM_ASYNC_PF_ACK, 1);
304 	}
305 
306 	set_irq_regs(old_regs);
307 }
308 
paravirt_ops_setup(void)309 static void __init paravirt_ops_setup(void)
310 {
311 	pv_info.name = "KVM";
312 
313 	if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY))
314 		pv_ops.cpu.io_delay = kvm_io_delay;
315 
316 #ifdef CONFIG_X86_IO_APIC
317 	no_timer_check = 1;
318 #endif
319 }
320 
kvm_register_steal_time(void)321 static void kvm_register_steal_time(void)
322 {
323 	int cpu = smp_processor_id();
324 	struct kvm_steal_time *st = &per_cpu(steal_time, cpu);
325 
326 	if (!has_steal_clock)
327 		return;
328 
329 	wrmsrl(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED));
330 	pr_debug("stealtime: cpu %d, msr %llx\n", cpu,
331 		(unsigned long long) slow_virt_to_phys(st));
332 }
333 
334 static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED;
335 
kvm_guest_apic_eoi_write(void)336 static notrace __maybe_unused void kvm_guest_apic_eoi_write(void)
337 {
338 	/**
339 	 * This relies on __test_and_clear_bit to modify the memory
340 	 * in a way that is atomic with respect to the local CPU.
341 	 * The hypervisor only accesses this memory from the local CPU so
342 	 * there's no need for lock or memory barriers.
343 	 * An optimization barrier is implied in apic write.
344 	 */
345 	if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi)))
346 		return;
347 	apic_native_eoi();
348 }
349 
kvm_guest_cpu_init(void)350 static void kvm_guest_cpu_init(void)
351 {
352 	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) {
353 		u64 pa;
354 
355 		WARN_ON_ONCE(!static_branch_likely(&kvm_async_pf_enabled));
356 
357 		pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason));
358 		pa |= KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;
359 
360 		if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT))
361 			pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
362 
363 		wrmsrl(MSR_KVM_ASYNC_PF_INT, HYPERVISOR_CALLBACK_VECTOR);
364 
365 		wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
366 		__this_cpu_write(async_pf_enabled, true);
367 		pr_debug("setup async PF for cpu %d\n", smp_processor_id());
368 	}
369 
370 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) {
371 		unsigned long pa;
372 
373 		/* Size alignment is implied but just to make it explicit. */
374 		BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4);
375 		__this_cpu_write(kvm_apic_eoi, 0);
376 		pa = slow_virt_to_phys(this_cpu_ptr(&kvm_apic_eoi))
377 			| KVM_MSR_ENABLED;
378 		wrmsrl(MSR_KVM_PV_EOI_EN, pa);
379 	}
380 
381 	if (has_steal_clock)
382 		kvm_register_steal_time();
383 }
384 
kvm_pv_disable_apf(void)385 static void kvm_pv_disable_apf(void)
386 {
387 	if (!__this_cpu_read(async_pf_enabled))
388 		return;
389 
390 	wrmsrl(MSR_KVM_ASYNC_PF_EN, 0);
391 	__this_cpu_write(async_pf_enabled, false);
392 
393 	pr_debug("disable async PF for cpu %d\n", smp_processor_id());
394 }
395 
kvm_disable_steal_time(void)396 static void kvm_disable_steal_time(void)
397 {
398 	if (!has_steal_clock)
399 		return;
400 
401 	wrmsr(MSR_KVM_STEAL_TIME, 0, 0);
402 }
403 
kvm_steal_clock(int cpu)404 static u64 kvm_steal_clock(int cpu)
405 {
406 	u64 steal;
407 	struct kvm_steal_time *src;
408 	int version;
409 
410 	src = &per_cpu(steal_time, cpu);
411 	do {
412 		version = src->version;
413 		virt_rmb();
414 		steal = src->steal;
415 		virt_rmb();
416 	} while ((version & 1) || (version != src->version));
417 
418 	return steal;
419 }
420 
__set_percpu_decrypted(void * ptr,unsigned long size)421 static inline void __set_percpu_decrypted(void *ptr, unsigned long size)
422 {
423 	early_set_memory_decrypted((unsigned long) ptr, size);
424 }
425 
426 /*
427  * Iterate through all possible CPUs and map the memory region pointed
428  * by apf_reason, steal_time and kvm_apic_eoi as decrypted at once.
429  *
430  * Note: we iterate through all possible CPUs to ensure that CPUs
431  * hotplugged will have their per-cpu variable already mapped as
432  * decrypted.
433  */
sev_map_percpu_data(void)434 static void __init sev_map_percpu_data(void)
435 {
436 	int cpu;
437 
438 	if (cc_vendor != CC_VENDOR_AMD ||
439 	    !cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
440 		return;
441 
442 	for_each_possible_cpu(cpu) {
443 		__set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason));
444 		__set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time));
445 		__set_percpu_decrypted(&per_cpu(kvm_apic_eoi, cpu), sizeof(kvm_apic_eoi));
446 	}
447 }
448 
kvm_guest_cpu_offline(bool shutdown)449 static void kvm_guest_cpu_offline(bool shutdown)
450 {
451 	kvm_disable_steal_time();
452 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
453 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
454 	if (kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL))
455 		wrmsrl(MSR_KVM_MIGRATION_CONTROL, 0);
456 	kvm_pv_disable_apf();
457 	if (!shutdown)
458 		apf_task_wake_all();
459 	kvmclock_disable();
460 }
461 
kvm_cpu_online(unsigned int cpu)462 static int kvm_cpu_online(unsigned int cpu)
463 {
464 	unsigned long flags;
465 
466 	local_irq_save(flags);
467 	kvm_guest_cpu_init();
468 	local_irq_restore(flags);
469 	return 0;
470 }
471 
472 #ifdef CONFIG_SMP
473 
474 static DEFINE_PER_CPU(cpumask_var_t, __pv_cpu_mask);
475 
pv_tlb_flush_supported(void)476 static bool pv_tlb_flush_supported(void)
477 {
478 	return (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
479 		!kvm_para_has_hint(KVM_HINTS_REALTIME) &&
480 		kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) &&
481 		!boot_cpu_has(X86_FEATURE_MWAIT) &&
482 		(num_possible_cpus() != 1));
483 }
484 
pv_ipi_supported(void)485 static bool pv_ipi_supported(void)
486 {
487 	return (kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI) &&
488 	       (num_possible_cpus() != 1));
489 }
490 
pv_sched_yield_supported(void)491 static bool pv_sched_yield_supported(void)
492 {
493 	return (kvm_para_has_feature(KVM_FEATURE_PV_SCHED_YIELD) &&
494 		!kvm_para_has_hint(KVM_HINTS_REALTIME) &&
495 	    kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) &&
496 	    !boot_cpu_has(X86_FEATURE_MWAIT) &&
497 	    (num_possible_cpus() != 1));
498 }
499 
500 #define KVM_IPI_CLUSTER_SIZE	(2 * BITS_PER_LONG)
501 
__send_ipi_mask(const struct cpumask * mask,int vector)502 static void __send_ipi_mask(const struct cpumask *mask, int vector)
503 {
504 	unsigned long flags;
505 	int cpu, min = 0, max = 0;
506 #ifdef CONFIG_X86_64
507 	__uint128_t ipi_bitmap = 0;
508 #else
509 	u64 ipi_bitmap = 0;
510 #endif
511 	u32 apic_id, icr;
512 	long ret;
513 
514 	if (cpumask_empty(mask))
515 		return;
516 
517 	local_irq_save(flags);
518 
519 	switch (vector) {
520 	default:
521 		icr = APIC_DM_FIXED | vector;
522 		break;
523 	case NMI_VECTOR:
524 		icr = APIC_DM_NMI;
525 		break;
526 	}
527 
528 	for_each_cpu(cpu, mask) {
529 		apic_id = per_cpu(x86_cpu_to_apicid, cpu);
530 		if (!ipi_bitmap) {
531 			min = max = apic_id;
532 		} else if (apic_id < min && max - apic_id < KVM_IPI_CLUSTER_SIZE) {
533 			ipi_bitmap <<= min - apic_id;
534 			min = apic_id;
535 		} else if (apic_id > min && apic_id < min + KVM_IPI_CLUSTER_SIZE) {
536 			max = apic_id < max ? max : apic_id;
537 		} else {
538 			ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap,
539 				(unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr);
540 			WARN_ONCE(ret < 0, "kvm-guest: failed to send PV IPI: %ld",
541 				  ret);
542 			min = max = apic_id;
543 			ipi_bitmap = 0;
544 		}
545 		__set_bit(apic_id - min, (unsigned long *)&ipi_bitmap);
546 	}
547 
548 	if (ipi_bitmap) {
549 		ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap,
550 			(unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr);
551 		WARN_ONCE(ret < 0, "kvm-guest: failed to send PV IPI: %ld",
552 			  ret);
553 	}
554 
555 	local_irq_restore(flags);
556 }
557 
kvm_send_ipi_mask(const struct cpumask * mask,int vector)558 static void kvm_send_ipi_mask(const struct cpumask *mask, int vector)
559 {
560 	__send_ipi_mask(mask, vector);
561 }
562 
kvm_send_ipi_mask_allbutself(const struct cpumask * mask,int vector)563 static void kvm_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
564 {
565 	unsigned int this_cpu = smp_processor_id();
566 	struct cpumask *new_mask = this_cpu_cpumask_var_ptr(__pv_cpu_mask);
567 	const struct cpumask *local_mask;
568 
569 	cpumask_copy(new_mask, mask);
570 	cpumask_clear_cpu(this_cpu, new_mask);
571 	local_mask = new_mask;
572 	__send_ipi_mask(local_mask, vector);
573 }
574 
setup_efi_kvm_sev_migration(void)575 static int __init setup_efi_kvm_sev_migration(void)
576 {
577 	efi_char16_t efi_sev_live_migration_enabled[] = L"SevLiveMigrationEnabled";
578 	efi_guid_t efi_variable_guid = AMD_SEV_MEM_ENCRYPT_GUID;
579 	efi_status_t status;
580 	unsigned long size;
581 	bool enabled;
582 
583 	if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) ||
584 	    !kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL))
585 		return 0;
586 
587 	if (!efi_enabled(EFI_BOOT))
588 		return 0;
589 
590 	if (!efi_enabled(EFI_RUNTIME_SERVICES)) {
591 		pr_info("%s : EFI runtime services are not enabled\n", __func__);
592 		return 0;
593 	}
594 
595 	size = sizeof(enabled);
596 
597 	/* Get variable contents into buffer */
598 	status = efi.get_variable(efi_sev_live_migration_enabled,
599 				  &efi_variable_guid, NULL, &size, &enabled);
600 
601 	if (status == EFI_NOT_FOUND) {
602 		pr_info("%s : EFI live migration variable not found\n", __func__);
603 		return 0;
604 	}
605 
606 	if (status != EFI_SUCCESS) {
607 		pr_info("%s : EFI variable retrieval failed\n", __func__);
608 		return 0;
609 	}
610 
611 	if (enabled == 0) {
612 		pr_info("%s: live migration disabled in EFI\n", __func__);
613 		return 0;
614 	}
615 
616 	pr_info("%s : live migration enabled in EFI\n", __func__);
617 	wrmsrl(MSR_KVM_MIGRATION_CONTROL, KVM_MIGRATION_READY);
618 
619 	return 1;
620 }
621 
622 late_initcall(setup_efi_kvm_sev_migration);
623 
624 /*
625  * Set the IPI entry points
626  */
kvm_setup_pv_ipi(void)627 static __init void kvm_setup_pv_ipi(void)
628 {
629 	apic_update_callback(send_IPI_mask, kvm_send_ipi_mask);
630 	apic_update_callback(send_IPI_mask_allbutself, kvm_send_ipi_mask_allbutself);
631 	pr_info("setup PV IPIs\n");
632 }
633 
kvm_smp_send_call_func_ipi(const struct cpumask * mask)634 static void kvm_smp_send_call_func_ipi(const struct cpumask *mask)
635 {
636 	int cpu;
637 
638 	native_send_call_func_ipi(mask);
639 
640 	/* Make sure other vCPUs get a chance to run if they need to. */
641 	for_each_cpu(cpu, mask) {
642 		if (!idle_cpu(cpu) && vcpu_is_preempted(cpu)) {
643 			kvm_hypercall1(KVM_HC_SCHED_YIELD, per_cpu(x86_cpu_to_apicid, cpu));
644 			break;
645 		}
646 	}
647 }
648 
kvm_flush_tlb_multi(const struct cpumask * cpumask,const struct flush_tlb_info * info)649 static void kvm_flush_tlb_multi(const struct cpumask *cpumask,
650 			const struct flush_tlb_info *info)
651 {
652 	u8 state;
653 	int cpu;
654 	struct kvm_steal_time *src;
655 	struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask);
656 
657 	cpumask_copy(flushmask, cpumask);
658 	/*
659 	 * We have to call flush only on online vCPUs. And
660 	 * queue flush_on_enter for pre-empted vCPUs
661 	 */
662 	for_each_cpu(cpu, flushmask) {
663 		/*
664 		 * The local vCPU is never preempted, so we do not explicitly
665 		 * skip check for local vCPU - it will never be cleared from
666 		 * flushmask.
667 		 */
668 		src = &per_cpu(steal_time, cpu);
669 		state = READ_ONCE(src->preempted);
670 		if ((state & KVM_VCPU_PREEMPTED)) {
671 			if (try_cmpxchg(&src->preempted, &state,
672 					state | KVM_VCPU_FLUSH_TLB))
673 				__cpumask_clear_cpu(cpu, flushmask);
674 		}
675 	}
676 
677 	native_flush_tlb_multi(flushmask, info);
678 }
679 
kvm_alloc_cpumask(void)680 static __init int kvm_alloc_cpumask(void)
681 {
682 	int cpu;
683 
684 	if (!kvm_para_available() || nopv)
685 		return 0;
686 
687 	if (pv_tlb_flush_supported() || pv_ipi_supported())
688 		for_each_possible_cpu(cpu) {
689 			zalloc_cpumask_var_node(per_cpu_ptr(&__pv_cpu_mask, cpu),
690 				GFP_KERNEL, cpu_to_node(cpu));
691 		}
692 
693 	return 0;
694 }
695 arch_initcall(kvm_alloc_cpumask);
696 
kvm_smp_prepare_boot_cpu(void)697 static void __init kvm_smp_prepare_boot_cpu(void)
698 {
699 	/*
700 	 * Map the per-cpu variables as decrypted before kvm_guest_cpu_init()
701 	 * shares the guest physical address with the hypervisor.
702 	 */
703 	sev_map_percpu_data();
704 
705 	kvm_guest_cpu_init();
706 	native_smp_prepare_boot_cpu();
707 	kvm_spinlock_init();
708 }
709 
kvm_cpu_down_prepare(unsigned int cpu)710 static int kvm_cpu_down_prepare(unsigned int cpu)
711 {
712 	unsigned long flags;
713 
714 	local_irq_save(flags);
715 	kvm_guest_cpu_offline(false);
716 	local_irq_restore(flags);
717 	return 0;
718 }
719 
720 #endif
721 
kvm_suspend(void)722 static int kvm_suspend(void)
723 {
724 	u64 val = 0;
725 
726 	kvm_guest_cpu_offline(false);
727 
728 #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
729 	if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL))
730 		rdmsrl(MSR_KVM_POLL_CONTROL, val);
731 	has_guest_poll = !(val & 1);
732 #endif
733 	return 0;
734 }
735 
kvm_resume(void)736 static void kvm_resume(void)
737 {
738 	kvm_cpu_online(raw_smp_processor_id());
739 
740 #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
741 	if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL) && has_guest_poll)
742 		wrmsrl(MSR_KVM_POLL_CONTROL, 0);
743 #endif
744 }
745 
746 static struct syscore_ops kvm_syscore_ops = {
747 	.suspend	= kvm_suspend,
748 	.resume		= kvm_resume,
749 };
750 
kvm_pv_guest_cpu_reboot(void * unused)751 static void kvm_pv_guest_cpu_reboot(void *unused)
752 {
753 	kvm_guest_cpu_offline(true);
754 }
755 
kvm_pv_reboot_notify(struct notifier_block * nb,unsigned long code,void * unused)756 static int kvm_pv_reboot_notify(struct notifier_block *nb,
757 				unsigned long code, void *unused)
758 {
759 	if (code == SYS_RESTART)
760 		on_each_cpu(kvm_pv_guest_cpu_reboot, NULL, 1);
761 	return NOTIFY_DONE;
762 }
763 
764 static struct notifier_block kvm_pv_reboot_nb = {
765 	.notifier_call = kvm_pv_reboot_notify,
766 };
767 
768 /*
769  * After a PV feature is registered, the host will keep writing to the
770  * registered memory location. If the guest happens to shutdown, this memory
771  * won't be valid. In cases like kexec, in which you install a new kernel, this
772  * means a random memory location will be kept being written.
773  */
774 #ifdef CONFIG_CRASH_DUMP
kvm_crash_shutdown(struct pt_regs * regs)775 static void kvm_crash_shutdown(struct pt_regs *regs)
776 {
777 	kvm_guest_cpu_offline(true);
778 	native_machine_crash_shutdown(regs);
779 }
780 #endif
781 
782 #if defined(CONFIG_X86_32) || !defined(CONFIG_SMP)
783 bool __kvm_vcpu_is_preempted(long cpu);
784 
__kvm_vcpu_is_preempted(long cpu)785 __visible bool __kvm_vcpu_is_preempted(long cpu)
786 {
787 	struct kvm_steal_time *src = &per_cpu(steal_time, cpu);
788 
789 	return !!(src->preempted & KVM_VCPU_PREEMPTED);
790 }
791 PV_CALLEE_SAVE_REGS_THUNK(__kvm_vcpu_is_preempted);
792 
793 #else
794 
795 #include <asm/asm-offsets.h>
796 
797 extern bool __raw_callee_save___kvm_vcpu_is_preempted(long);
798 
799 /*
800  * Hand-optimize version for x86-64 to avoid 8 64-bit register saving and
801  * restoring to/from the stack.
802  */
803 #define PV_VCPU_PREEMPTED_ASM						     \
804  "movq   __per_cpu_offset(,%rdi,8), %rax\n\t"				     \
805  "cmpb   $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax)\n\t" \
806  "setne  %al\n\t"
807 
808 DEFINE_ASM_FUNC(__raw_callee_save___kvm_vcpu_is_preempted,
809 		PV_VCPU_PREEMPTED_ASM, .text);
810 #endif
811 
kvm_guest_init(void)812 static void __init kvm_guest_init(void)
813 {
814 	int i;
815 
816 	paravirt_ops_setup();
817 	register_reboot_notifier(&kvm_pv_reboot_nb);
818 	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++)
819 		raw_spin_lock_init(&async_pf_sleepers[i].lock);
820 
821 	if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
822 		has_steal_clock = 1;
823 		static_call_update(pv_steal_clock, kvm_steal_clock);
824 
825 		pv_ops.lock.vcpu_is_preempted =
826 			PV_CALLEE_SAVE(__kvm_vcpu_is_preempted);
827 	}
828 
829 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
830 		apic_update_callback(eoi, kvm_guest_apic_eoi_write);
831 
832 	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) {
833 		static_branch_enable(&kvm_async_pf_enabled);
834 		sysvec_install(HYPERVISOR_CALLBACK_VECTOR, sysvec_kvm_asyncpf_interrupt);
835 	}
836 
837 #ifdef CONFIG_SMP
838 	if (pv_tlb_flush_supported()) {
839 		pv_ops.mmu.flush_tlb_multi = kvm_flush_tlb_multi;
840 		pv_ops.mmu.tlb_remove_table = tlb_remove_table;
841 		pr_info("KVM setup pv remote TLB flush\n");
842 	}
843 
844 	smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
845 	if (pv_sched_yield_supported()) {
846 		smp_ops.send_call_func_ipi = kvm_smp_send_call_func_ipi;
847 		pr_info("setup PV sched yield\n");
848 	}
849 	if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/kvm:online",
850 				      kvm_cpu_online, kvm_cpu_down_prepare) < 0)
851 		pr_err("failed to install cpu hotplug callbacks\n");
852 #else
853 	sev_map_percpu_data();
854 	kvm_guest_cpu_init();
855 #endif
856 
857 #ifdef CONFIG_CRASH_DUMP
858 	machine_ops.crash_shutdown = kvm_crash_shutdown;
859 #endif
860 
861 	register_syscore_ops(&kvm_syscore_ops);
862 
863 	/*
864 	 * Hard lockup detection is enabled by default. Disable it, as guests
865 	 * can get false positives too easily, for example if the host is
866 	 * overcommitted.
867 	 */
868 	hardlockup_detector_disable();
869 }
870 
__kvm_cpuid_base(void)871 static noinline uint32_t __kvm_cpuid_base(void)
872 {
873 	if (boot_cpu_data.cpuid_level < 0)
874 		return 0;	/* So we don't blow up on old processors */
875 
876 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
877 		return hypervisor_cpuid_base(KVM_SIGNATURE, 0);
878 
879 	return 0;
880 }
881 
kvm_cpuid_base(void)882 static inline uint32_t kvm_cpuid_base(void)
883 {
884 	static int kvm_cpuid_base = -1;
885 
886 	if (kvm_cpuid_base == -1)
887 		kvm_cpuid_base = __kvm_cpuid_base();
888 
889 	return kvm_cpuid_base;
890 }
891 
kvm_para_available(void)892 bool kvm_para_available(void)
893 {
894 	return kvm_cpuid_base() != 0;
895 }
896 EXPORT_SYMBOL_GPL(kvm_para_available);
897 
kvm_arch_para_features(void)898 unsigned int kvm_arch_para_features(void)
899 {
900 	return cpuid_eax(kvm_cpuid_base() | KVM_CPUID_FEATURES);
901 }
902 
kvm_arch_para_hints(void)903 unsigned int kvm_arch_para_hints(void)
904 {
905 	return cpuid_edx(kvm_cpuid_base() | KVM_CPUID_FEATURES);
906 }
907 EXPORT_SYMBOL_GPL(kvm_arch_para_hints);
908 
kvm_detect(void)909 static uint32_t __init kvm_detect(void)
910 {
911 	return kvm_cpuid_base();
912 }
913 
kvm_apic_init(void)914 static void __init kvm_apic_init(void)
915 {
916 #ifdef CONFIG_SMP
917 	if (pv_ipi_supported())
918 		kvm_setup_pv_ipi();
919 #endif
920 }
921 
kvm_msi_ext_dest_id(void)922 static bool __init kvm_msi_ext_dest_id(void)
923 {
924 	return kvm_para_has_feature(KVM_FEATURE_MSI_EXT_DEST_ID);
925 }
926 
kvm_sev_hc_page_enc_status(unsigned long pfn,int npages,bool enc)927 static void kvm_sev_hc_page_enc_status(unsigned long pfn, int npages, bool enc)
928 {
929 	kvm_sev_hypercall3(KVM_HC_MAP_GPA_RANGE, pfn << PAGE_SHIFT, npages,
930 			   KVM_MAP_GPA_RANGE_ENC_STAT(enc) | KVM_MAP_GPA_RANGE_PAGE_SZ_4K);
931 }
932 
kvm_init_platform(void)933 static void __init kvm_init_platform(void)
934 {
935 	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) &&
936 	    kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) {
937 		unsigned long nr_pages;
938 		int i;
939 
940 		pv_ops.mmu.notify_page_enc_status_changed =
941 			kvm_sev_hc_page_enc_status;
942 
943 		/*
944 		 * Reset the host's shared pages list related to kernel
945 		 * specific page encryption status settings before we load a
946 		 * new kernel by kexec. Reset the page encryption status
947 		 * during early boot instead of just before kexec to avoid SMP
948 		 * races during kvm_pv_guest_cpu_reboot().
949 		 * NOTE: We cannot reset the complete shared pages list
950 		 * here as we need to retain the UEFI/OVMF firmware
951 		 * specific settings.
952 		 */
953 
954 		for (i = 0; i < e820_table->nr_entries; i++) {
955 			struct e820_entry *entry = &e820_table->entries[i];
956 
957 			if (entry->type != E820_TYPE_RAM)
958 				continue;
959 
960 			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
961 
962 			kvm_sev_hypercall3(KVM_HC_MAP_GPA_RANGE, entry->addr,
963 				       nr_pages,
964 				       KVM_MAP_GPA_RANGE_ENCRYPTED | KVM_MAP_GPA_RANGE_PAGE_SZ_4K);
965 		}
966 
967 		/*
968 		 * Ensure that _bss_decrypted section is marked as decrypted in the
969 		 * shared pages list.
970 		 */
971 		early_set_mem_enc_dec_hypercall((unsigned long)__start_bss_decrypted,
972 						__end_bss_decrypted - __start_bss_decrypted, 0);
973 
974 		/*
975 		 * If not booted using EFI, enable Live migration support.
976 		 */
977 		if (!efi_enabled(EFI_BOOT))
978 			wrmsrl(MSR_KVM_MIGRATION_CONTROL,
979 			       KVM_MIGRATION_READY);
980 	}
981 	kvmclock_init();
982 	x86_platform.apic_post_init = kvm_apic_init;
983 }
984 
985 #if defined(CONFIG_AMD_MEM_ENCRYPT)
kvm_sev_es_hcall_prepare(struct ghcb * ghcb,struct pt_regs * regs)986 static void kvm_sev_es_hcall_prepare(struct ghcb *ghcb, struct pt_regs *regs)
987 {
988 	/* RAX and CPL are already in the GHCB */
989 	ghcb_set_rbx(ghcb, regs->bx);
990 	ghcb_set_rcx(ghcb, regs->cx);
991 	ghcb_set_rdx(ghcb, regs->dx);
992 	ghcb_set_rsi(ghcb, regs->si);
993 }
994 
kvm_sev_es_hcall_finish(struct ghcb * ghcb,struct pt_regs * regs)995 static bool kvm_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs)
996 {
997 	/* No checking of the return state needed */
998 	return true;
999 }
1000 #endif
1001 
1002 const __initconst struct hypervisor_x86 x86_hyper_kvm = {
1003 	.name				= "KVM",
1004 	.detect				= kvm_detect,
1005 	.type				= X86_HYPER_KVM,
1006 	.init.guest_late_init		= kvm_guest_init,
1007 	.init.x2apic_available		= kvm_para_available,
1008 	.init.msi_ext_dest_id		= kvm_msi_ext_dest_id,
1009 	.init.init_platform		= kvm_init_platform,
1010 #if defined(CONFIG_AMD_MEM_ENCRYPT)
1011 	.runtime.sev_es_hcall_prepare	= kvm_sev_es_hcall_prepare,
1012 	.runtime.sev_es_hcall_finish	= kvm_sev_es_hcall_finish,
1013 #endif
1014 };
1015 
activate_jump_labels(void)1016 static __init int activate_jump_labels(void)
1017 {
1018 	if (has_steal_clock) {
1019 		static_key_slow_inc(&paravirt_steal_enabled);
1020 		if (steal_acc)
1021 			static_key_slow_inc(&paravirt_steal_rq_enabled);
1022 	}
1023 
1024 	return 0;
1025 }
1026 arch_initcall(activate_jump_labels);
1027 
1028 #ifdef CONFIG_PARAVIRT_SPINLOCKS
1029 
1030 /* Kick a cpu by its apicid. Used to wake up a halted vcpu */
kvm_kick_cpu(int cpu)1031 static void kvm_kick_cpu(int cpu)
1032 {
1033 	unsigned long flags = 0;
1034 	u32 apicid;
1035 
1036 	apicid = per_cpu(x86_cpu_to_apicid, cpu);
1037 	kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid);
1038 }
1039 
1040 #include <asm/qspinlock.h>
1041 
kvm_wait(u8 * ptr,u8 val)1042 static void kvm_wait(u8 *ptr, u8 val)
1043 {
1044 	if (in_nmi())
1045 		return;
1046 
1047 	/*
1048 	 * halt until it's our turn and kicked. Note that we do safe halt
1049 	 * for irq enabled case to avoid hang when lock info is overwritten
1050 	 * in irq spinlock slowpath and no spurious interrupt occur to save us.
1051 	 */
1052 	if (irqs_disabled()) {
1053 		if (READ_ONCE(*ptr) == val)
1054 			halt();
1055 	} else {
1056 		local_irq_disable();
1057 
1058 		/* safe_halt() will enable IRQ */
1059 		if (READ_ONCE(*ptr) == val)
1060 			safe_halt();
1061 		else
1062 			local_irq_enable();
1063 	}
1064 }
1065 
1066 /*
1067  * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
1068  */
kvm_spinlock_init(void)1069 void __init kvm_spinlock_init(void)
1070 {
1071 	/*
1072 	 * In case host doesn't support KVM_FEATURE_PV_UNHALT there is still an
1073 	 * advantage of keeping virt_spin_lock_key enabled: virt_spin_lock() is
1074 	 * preferred over native qspinlock when vCPU is preempted.
1075 	 */
1076 	if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT)) {
1077 		pr_info("PV spinlocks disabled, no host support\n");
1078 		return;
1079 	}
1080 
1081 	/*
1082 	 * Disable PV spinlocks and use native qspinlock when dedicated pCPUs
1083 	 * are available.
1084 	 */
1085 	if (kvm_para_has_hint(KVM_HINTS_REALTIME)) {
1086 		pr_info("PV spinlocks disabled with KVM_HINTS_REALTIME hints\n");
1087 		goto out;
1088 	}
1089 
1090 	if (num_possible_cpus() == 1) {
1091 		pr_info("PV spinlocks disabled, single CPU\n");
1092 		goto out;
1093 	}
1094 
1095 	if (nopvspin) {
1096 		pr_info("PV spinlocks disabled, forced by \"nopvspin\" parameter\n");
1097 		goto out;
1098 	}
1099 
1100 	pr_info("PV spinlocks enabled\n");
1101 
1102 	__pv_init_lock_hash();
1103 	pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
1104 	pv_ops.lock.queued_spin_unlock =
1105 		PV_CALLEE_SAVE(__pv_queued_spin_unlock);
1106 	pv_ops.lock.wait = kvm_wait;
1107 	pv_ops.lock.kick = kvm_kick_cpu;
1108 
1109 	/*
1110 	 * When PV spinlock is enabled which is preferred over
1111 	 * virt_spin_lock(), virt_spin_lock_key's value is meaningless.
1112 	 * Just disable it anyway.
1113 	 */
1114 out:
1115 	static_branch_disable(&virt_spin_lock_key);
1116 }
1117 
1118 #endif	/* CONFIG_PARAVIRT_SPINLOCKS */
1119 
1120 #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
1121 
kvm_disable_host_haltpoll(void * i)1122 static void kvm_disable_host_haltpoll(void *i)
1123 {
1124 	wrmsrl(MSR_KVM_POLL_CONTROL, 0);
1125 }
1126 
kvm_enable_host_haltpoll(void * i)1127 static void kvm_enable_host_haltpoll(void *i)
1128 {
1129 	wrmsrl(MSR_KVM_POLL_CONTROL, 1);
1130 }
1131 
arch_haltpoll_enable(unsigned int cpu)1132 void arch_haltpoll_enable(unsigned int cpu)
1133 {
1134 	if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) {
1135 		pr_err_once("host does not support poll control\n");
1136 		pr_err_once("host upgrade recommended\n");
1137 		return;
1138 	}
1139 
1140 	/* Enable guest halt poll disables host halt poll */
1141 	smp_call_function_single(cpu, kvm_disable_host_haltpoll, NULL, 1);
1142 }
1143 EXPORT_SYMBOL_GPL(arch_haltpoll_enable);
1144 
arch_haltpoll_disable(unsigned int cpu)1145 void arch_haltpoll_disable(unsigned int cpu)
1146 {
1147 	if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL))
1148 		return;
1149 
1150 	/* Disable guest halt poll enables host halt poll */
1151 	smp_call_function_single(cpu, kvm_enable_host_haltpoll, NULL, 1);
1152 }
1153 EXPORT_SYMBOL_GPL(arch_haltpoll_disable);
1154 #endif
1155