xref: /linux/arch/x86/kvm/lapic.c (revision 8795a739e5c72abeec51caf36b6df2b37e5720c5)
1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 /*
4  * Local APIC virtualization
5  *
6  * Copyright (C) 2006 Qumranet, Inc.
7  * Copyright (C) 2007 Novell
8  * Copyright (C) 2007 Intel
9  * Copyright 2009 Red Hat, Inc. and/or its affiliates.
10  *
11  * Authors:
12  *   Dor Laor <dor.laor@qumranet.com>
13  *   Gregory Haskins <ghaskins@novell.com>
14  *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
15  *
16  * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
17  */
18 
19 #include <linux/kvm_host.h>
20 #include <linux/kvm.h>
21 #include <linux/mm.h>
22 #include <linux/highmem.h>
23 #include <linux/smp.h>
24 #include <linux/hrtimer.h>
25 #include <linux/io.h>
26 #include <linux/export.h>
27 #include <linux/math64.h>
28 #include <linux/slab.h>
29 #include <asm/processor.h>
30 #include <asm/msr.h>
31 #include <asm/page.h>
32 #include <asm/current.h>
33 #include <asm/apicdef.h>
34 #include <asm/delay.h>
35 #include <linux/atomic.h>
36 #include <linux/jump_label.h>
37 #include "kvm_cache_regs.h"
38 #include "irq.h"
39 #include "trace.h"
40 #include "x86.h"
41 #include "cpuid.h"
42 #include "hyperv.h"
43 
44 #ifndef CONFIG_X86_64
45 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
46 #else
47 #define mod_64(x, y) ((x) % (y))
48 #endif
49 
50 #define PRId64 "d"
51 #define PRIx64 "llx"
52 #define PRIu64 "u"
53 #define PRIo64 "o"
54 
55 /* 14 is the version for Xeon and Pentium 8.4.8*/
56 #define APIC_VERSION			(0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
57 #define LAPIC_MMIO_LENGTH		(1 << 12)
58 /* followed define is not in apicdef.h */
59 #define APIC_SHORT_MASK			0xc0000
60 #define APIC_DEST_NOSHORT		0x0
61 #define APIC_DEST_MASK			0x800
62 #define MAX_APIC_VECTOR			256
63 #define APIC_VECTORS_PER_REG		32
64 
65 #define APIC_BROADCAST			0xFF
66 #define X2APIC_BROADCAST		0xFFFFFFFFul
67 
68 static bool lapic_timer_advance_dynamic __read_mostly;
69 #define LAPIC_TIMER_ADVANCE_ADJUST_MIN	100	/* clock cycles */
70 #define LAPIC_TIMER_ADVANCE_ADJUST_MAX	10000	/* clock cycles */
71 #define LAPIC_TIMER_ADVANCE_NS_INIT	1000
72 #define LAPIC_TIMER_ADVANCE_NS_MAX     5000
73 /* step-by-step approximation to mitigate fluctuation */
74 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
75 
76 static inline int apic_test_vector(int vec, void *bitmap)
77 {
78 	return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
79 }
80 
81 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
82 {
83 	struct kvm_lapic *apic = vcpu->arch.apic;
84 
85 	return apic_test_vector(vector, apic->regs + APIC_ISR) ||
86 		apic_test_vector(vector, apic->regs + APIC_IRR);
87 }
88 
89 static inline int __apic_test_and_set_vector(int vec, void *bitmap)
90 {
91 	return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
92 }
93 
94 static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
95 {
96 	return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
97 }
98 
99 struct static_key_deferred apic_hw_disabled __read_mostly;
100 struct static_key_deferred apic_sw_disabled __read_mostly;
101 
102 static inline int apic_enabled(struct kvm_lapic *apic)
103 {
104 	return kvm_apic_sw_enabled(apic) &&	kvm_apic_hw_enabled(apic);
105 }
106 
107 #define LVT_MASK	\
108 	(APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
109 
110 #define LINT_MASK	\
111 	(LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
112 	 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
113 
114 static inline u8 kvm_xapic_id(struct kvm_lapic *apic)
115 {
116 	return kvm_lapic_get_reg(apic, APIC_ID) >> 24;
117 }
118 
119 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
120 {
121 	return apic->vcpu->vcpu_id;
122 }
123 
124 bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
125 {
126 	return pi_inject_timer && kvm_vcpu_apicv_active(vcpu);
127 }
128 EXPORT_SYMBOL_GPL(kvm_can_post_timer_interrupt);
129 
130 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu)
131 {
132 	return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE;
133 }
134 
135 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
136 		u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
137 	switch (map->mode) {
138 	case KVM_APIC_MODE_X2APIC: {
139 		u32 offset = (dest_id >> 16) * 16;
140 		u32 max_apic_id = map->max_apic_id;
141 
142 		if (offset <= max_apic_id) {
143 			u8 cluster_size = min(max_apic_id - offset + 1, 16U);
144 
145 			offset = array_index_nospec(offset, map->max_apic_id + 1);
146 			*cluster = &map->phys_map[offset];
147 			*mask = dest_id & (0xffff >> (16 - cluster_size));
148 		} else {
149 			*mask = 0;
150 		}
151 
152 		return true;
153 		}
154 	case KVM_APIC_MODE_XAPIC_FLAT:
155 		*cluster = map->xapic_flat_map;
156 		*mask = dest_id & 0xff;
157 		return true;
158 	case KVM_APIC_MODE_XAPIC_CLUSTER:
159 		*cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
160 		*mask = dest_id & 0xf;
161 		return true;
162 	default:
163 		/* Not optimized. */
164 		return false;
165 	}
166 }
167 
168 static void kvm_apic_map_free(struct rcu_head *rcu)
169 {
170 	struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
171 
172 	kvfree(map);
173 }
174 
175 static void recalculate_apic_map(struct kvm *kvm)
176 {
177 	struct kvm_apic_map *new, *old = NULL;
178 	struct kvm_vcpu *vcpu;
179 	int i;
180 	u32 max_id = 255; /* enough space for any xAPIC ID */
181 
182 	mutex_lock(&kvm->arch.apic_map_lock);
183 
184 	kvm_for_each_vcpu(i, vcpu, kvm)
185 		if (kvm_apic_present(vcpu))
186 			max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
187 
188 	new = kvzalloc(sizeof(struct kvm_apic_map) +
189 	                   sizeof(struct kvm_lapic *) * ((u64)max_id + 1),
190 			   GFP_KERNEL_ACCOUNT);
191 
192 	if (!new)
193 		goto out;
194 
195 	new->max_apic_id = max_id;
196 
197 	kvm_for_each_vcpu(i, vcpu, kvm) {
198 		struct kvm_lapic *apic = vcpu->arch.apic;
199 		struct kvm_lapic **cluster;
200 		u16 mask;
201 		u32 ldr;
202 		u8 xapic_id;
203 		u32 x2apic_id;
204 
205 		if (!kvm_apic_present(vcpu))
206 			continue;
207 
208 		xapic_id = kvm_xapic_id(apic);
209 		x2apic_id = kvm_x2apic_id(apic);
210 
211 		/* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
212 		if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
213 				x2apic_id <= new->max_apic_id)
214 			new->phys_map[x2apic_id] = apic;
215 		/*
216 		 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
217 		 * prevent them from masking VCPUs with APIC ID <= 0xff.
218 		 */
219 		if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
220 			new->phys_map[xapic_id] = apic;
221 
222 		if (!kvm_apic_sw_enabled(apic))
223 			continue;
224 
225 		ldr = kvm_lapic_get_reg(apic, APIC_LDR);
226 
227 		if (apic_x2apic_mode(apic)) {
228 			new->mode |= KVM_APIC_MODE_X2APIC;
229 		} else if (ldr) {
230 			ldr = GET_APIC_LOGICAL_ID(ldr);
231 			if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
232 				new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
233 			else
234 				new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
235 		}
236 
237 		if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
238 			continue;
239 
240 		if (mask)
241 			cluster[ffs(mask) - 1] = apic;
242 	}
243 out:
244 	old = rcu_dereference_protected(kvm->arch.apic_map,
245 			lockdep_is_held(&kvm->arch.apic_map_lock));
246 	rcu_assign_pointer(kvm->arch.apic_map, new);
247 	mutex_unlock(&kvm->arch.apic_map_lock);
248 
249 	if (old)
250 		call_rcu(&old->rcu, kvm_apic_map_free);
251 
252 	kvm_make_scan_ioapic_request(kvm);
253 }
254 
255 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
256 {
257 	bool enabled = val & APIC_SPIV_APIC_ENABLED;
258 
259 	kvm_lapic_set_reg(apic, APIC_SPIV, val);
260 
261 	if (enabled != apic->sw_enabled) {
262 		apic->sw_enabled = enabled;
263 		if (enabled)
264 			static_key_slow_dec_deferred(&apic_sw_disabled);
265 		else
266 			static_key_slow_inc(&apic_sw_disabled.key);
267 
268 		recalculate_apic_map(apic->vcpu->kvm);
269 	}
270 }
271 
272 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
273 {
274 	kvm_lapic_set_reg(apic, APIC_ID, id << 24);
275 	recalculate_apic_map(apic->vcpu->kvm);
276 }
277 
278 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
279 {
280 	kvm_lapic_set_reg(apic, APIC_LDR, id);
281 	recalculate_apic_map(apic->vcpu->kvm);
282 }
283 
284 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
285 {
286 	return ((id >> 4) << 16) | (1 << (id & 0xf));
287 }
288 
289 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
290 {
291 	u32 ldr = kvm_apic_calc_x2apic_ldr(id);
292 
293 	WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
294 
295 	kvm_lapic_set_reg(apic, APIC_ID, id);
296 	kvm_lapic_set_reg(apic, APIC_LDR, ldr);
297 	recalculate_apic_map(apic->vcpu->kvm);
298 }
299 
300 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
301 {
302 	return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
303 }
304 
305 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
306 {
307 	return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
308 }
309 
310 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
311 {
312 	return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
313 }
314 
315 static inline int apic_lvtt_period(struct kvm_lapic *apic)
316 {
317 	return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
318 }
319 
320 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
321 {
322 	return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
323 }
324 
325 static inline int apic_lvt_nmi_mode(u32 lvt_val)
326 {
327 	return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
328 }
329 
330 void kvm_apic_set_version(struct kvm_vcpu *vcpu)
331 {
332 	struct kvm_lapic *apic = vcpu->arch.apic;
333 	struct kvm_cpuid_entry2 *feat;
334 	u32 v = APIC_VERSION;
335 
336 	if (!lapic_in_kernel(vcpu))
337 		return;
338 
339 	/*
340 	 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
341 	 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
342 	 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
343 	 * version first and level-triggered interrupts never get EOIed in
344 	 * IOAPIC.
345 	 */
346 	feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
347 	if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) &&
348 	    !ioapic_in_kernel(vcpu->kvm))
349 		v |= APIC_LVR_DIRECTED_EOI;
350 	kvm_lapic_set_reg(apic, APIC_LVR, v);
351 }
352 
353 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
354 	LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
355 	LVT_MASK | APIC_MODE_MASK,	/* LVTTHMR */
356 	LVT_MASK | APIC_MODE_MASK,	/* LVTPC */
357 	LINT_MASK, LINT_MASK,	/* LVT0-1 */
358 	LVT_MASK		/* LVTERR */
359 };
360 
361 static int find_highest_vector(void *bitmap)
362 {
363 	int vec;
364 	u32 *reg;
365 
366 	for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
367 	     vec >= 0; vec -= APIC_VECTORS_PER_REG) {
368 		reg = bitmap + REG_POS(vec);
369 		if (*reg)
370 			return __fls(*reg) + vec;
371 	}
372 
373 	return -1;
374 }
375 
376 static u8 count_vectors(void *bitmap)
377 {
378 	int vec;
379 	u32 *reg;
380 	u8 count = 0;
381 
382 	for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
383 		reg = bitmap + REG_POS(vec);
384 		count += hweight32(*reg);
385 	}
386 
387 	return count;
388 }
389 
390 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
391 {
392 	u32 i, vec;
393 	u32 pir_val, irr_val, prev_irr_val;
394 	int max_updated_irr;
395 
396 	max_updated_irr = -1;
397 	*max_irr = -1;
398 
399 	for (i = vec = 0; i <= 7; i++, vec += 32) {
400 		pir_val = READ_ONCE(pir[i]);
401 		irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
402 		if (pir_val) {
403 			prev_irr_val = irr_val;
404 			irr_val |= xchg(&pir[i], 0);
405 			*((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
406 			if (prev_irr_val != irr_val) {
407 				max_updated_irr =
408 					__fls(irr_val ^ prev_irr_val) + vec;
409 			}
410 		}
411 		if (irr_val)
412 			*max_irr = __fls(irr_val) + vec;
413 	}
414 
415 	return ((max_updated_irr != -1) &&
416 		(max_updated_irr == *max_irr));
417 }
418 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
419 
420 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
421 {
422 	struct kvm_lapic *apic = vcpu->arch.apic;
423 
424 	return __kvm_apic_update_irr(pir, apic->regs, max_irr);
425 }
426 EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
427 
428 static inline int apic_search_irr(struct kvm_lapic *apic)
429 {
430 	return find_highest_vector(apic->regs + APIC_IRR);
431 }
432 
433 static inline int apic_find_highest_irr(struct kvm_lapic *apic)
434 {
435 	int result;
436 
437 	/*
438 	 * Note that irr_pending is just a hint. It will be always
439 	 * true with virtual interrupt delivery enabled.
440 	 */
441 	if (!apic->irr_pending)
442 		return -1;
443 
444 	result = apic_search_irr(apic);
445 	ASSERT(result == -1 || result >= 16);
446 
447 	return result;
448 }
449 
450 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
451 {
452 	struct kvm_vcpu *vcpu;
453 
454 	vcpu = apic->vcpu;
455 
456 	if (unlikely(vcpu->arch.apicv_active)) {
457 		/* need to update RVI */
458 		kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
459 		kvm_x86_ops->hwapic_irr_update(vcpu,
460 				apic_find_highest_irr(apic));
461 	} else {
462 		apic->irr_pending = false;
463 		kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
464 		if (apic_search_irr(apic) != -1)
465 			apic->irr_pending = true;
466 	}
467 }
468 
469 static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
470 {
471 	struct kvm_vcpu *vcpu;
472 
473 	if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
474 		return;
475 
476 	vcpu = apic->vcpu;
477 
478 	/*
479 	 * With APIC virtualization enabled, all caching is disabled
480 	 * because the processor can modify ISR under the hood.  Instead
481 	 * just set SVI.
482 	 */
483 	if (unlikely(vcpu->arch.apicv_active))
484 		kvm_x86_ops->hwapic_isr_update(vcpu, vec);
485 	else {
486 		++apic->isr_count;
487 		BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
488 		/*
489 		 * ISR (in service register) bit is set when injecting an interrupt.
490 		 * The highest vector is injected. Thus the latest bit set matches
491 		 * the highest bit in ISR.
492 		 */
493 		apic->highest_isr_cache = vec;
494 	}
495 }
496 
497 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
498 {
499 	int result;
500 
501 	/*
502 	 * Note that isr_count is always 1, and highest_isr_cache
503 	 * is always -1, with APIC virtualization enabled.
504 	 */
505 	if (!apic->isr_count)
506 		return -1;
507 	if (likely(apic->highest_isr_cache != -1))
508 		return apic->highest_isr_cache;
509 
510 	result = find_highest_vector(apic->regs + APIC_ISR);
511 	ASSERT(result == -1 || result >= 16);
512 
513 	return result;
514 }
515 
516 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
517 {
518 	struct kvm_vcpu *vcpu;
519 	if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
520 		return;
521 
522 	vcpu = apic->vcpu;
523 
524 	/*
525 	 * We do get here for APIC virtualization enabled if the guest
526 	 * uses the Hyper-V APIC enlightenment.  In this case we may need
527 	 * to trigger a new interrupt delivery by writing the SVI field;
528 	 * on the other hand isr_count and highest_isr_cache are unused
529 	 * and must be left alone.
530 	 */
531 	if (unlikely(vcpu->arch.apicv_active))
532 		kvm_x86_ops->hwapic_isr_update(vcpu,
533 					       apic_find_highest_isr(apic));
534 	else {
535 		--apic->isr_count;
536 		BUG_ON(apic->isr_count < 0);
537 		apic->highest_isr_cache = -1;
538 	}
539 }
540 
541 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
542 {
543 	/* This may race with setting of irr in __apic_accept_irq() and
544 	 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
545 	 * will cause vmexit immediately and the value will be recalculated
546 	 * on the next vmentry.
547 	 */
548 	return apic_find_highest_irr(vcpu->arch.apic);
549 }
550 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
551 
552 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
553 			     int vector, int level, int trig_mode,
554 			     struct dest_map *dest_map);
555 
556 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
557 		     struct dest_map *dest_map)
558 {
559 	struct kvm_lapic *apic = vcpu->arch.apic;
560 
561 	return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
562 			irq->level, irq->trig_mode, dest_map);
563 }
564 
565 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
566 		    unsigned long ipi_bitmap_high, u32 min,
567 		    unsigned long icr, int op_64_bit)
568 {
569 	int i;
570 	struct kvm_apic_map *map;
571 	struct kvm_vcpu *vcpu;
572 	struct kvm_lapic_irq irq = {0};
573 	int cluster_size = op_64_bit ? 64 : 32;
574 	int count = 0;
575 
576 	irq.vector = icr & APIC_VECTOR_MASK;
577 	irq.delivery_mode = icr & APIC_MODE_MASK;
578 	irq.level = (icr & APIC_INT_ASSERT) != 0;
579 	irq.trig_mode = icr & APIC_INT_LEVELTRIG;
580 
581 	if (icr & APIC_DEST_MASK)
582 		return -KVM_EINVAL;
583 	if (icr & APIC_SHORT_MASK)
584 		return -KVM_EINVAL;
585 
586 	rcu_read_lock();
587 	map = rcu_dereference(kvm->arch.apic_map);
588 
589 	if (unlikely(!map)) {
590 		count = -EOPNOTSUPP;
591 		goto out;
592 	}
593 
594 	if (min > map->max_apic_id)
595 		goto out;
596 	/* Bits above cluster_size are masked in the caller.  */
597 	for_each_set_bit(i, &ipi_bitmap_low,
598 		min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
599 		if (map->phys_map[min + i]) {
600 			vcpu = map->phys_map[min + i]->vcpu;
601 			count += kvm_apic_set_irq(vcpu, &irq, NULL);
602 		}
603 	}
604 
605 	min += cluster_size;
606 
607 	if (min > map->max_apic_id)
608 		goto out;
609 
610 	for_each_set_bit(i, &ipi_bitmap_high,
611 		min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
612 		if (map->phys_map[min + i]) {
613 			vcpu = map->phys_map[min + i]->vcpu;
614 			count += kvm_apic_set_irq(vcpu, &irq, NULL);
615 		}
616 	}
617 
618 out:
619 	rcu_read_unlock();
620 	return count;
621 }
622 
623 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
624 {
625 
626 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
627 				      sizeof(val));
628 }
629 
630 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
631 {
632 
633 	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
634 				      sizeof(*val));
635 }
636 
637 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
638 {
639 	return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
640 }
641 
642 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
643 {
644 	u8 val;
645 	if (pv_eoi_get_user(vcpu, &val) < 0)
646 		printk(KERN_WARNING "Can't read EOI MSR value: 0x%llx\n",
647 			   (unsigned long long)vcpu->arch.pv_eoi.msr_val);
648 	return val & 0x1;
649 }
650 
651 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
652 {
653 	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
654 		printk(KERN_WARNING "Can't set EOI MSR value: 0x%llx\n",
655 			   (unsigned long long)vcpu->arch.pv_eoi.msr_val);
656 		return;
657 	}
658 	__set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
659 }
660 
661 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
662 {
663 	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
664 		printk(KERN_WARNING "Can't clear EOI MSR value: 0x%llx\n",
665 			   (unsigned long long)vcpu->arch.pv_eoi.msr_val);
666 		return;
667 	}
668 	__clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
669 }
670 
671 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
672 {
673 	int highest_irr;
674 	if (apic->vcpu->arch.apicv_active)
675 		highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
676 	else
677 		highest_irr = apic_find_highest_irr(apic);
678 	if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
679 		return -1;
680 	return highest_irr;
681 }
682 
683 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
684 {
685 	u32 tpr, isrv, ppr, old_ppr;
686 	int isr;
687 
688 	old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
689 	tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
690 	isr = apic_find_highest_isr(apic);
691 	isrv = (isr != -1) ? isr : 0;
692 
693 	if ((tpr & 0xf0) >= (isrv & 0xf0))
694 		ppr = tpr & 0xff;
695 	else
696 		ppr = isrv & 0xf0;
697 
698 	*new_ppr = ppr;
699 	if (old_ppr != ppr)
700 		kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
701 
702 	return ppr < old_ppr;
703 }
704 
705 static void apic_update_ppr(struct kvm_lapic *apic)
706 {
707 	u32 ppr;
708 
709 	if (__apic_update_ppr(apic, &ppr) &&
710 	    apic_has_interrupt_for_ppr(apic, ppr) != -1)
711 		kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
712 }
713 
714 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
715 {
716 	apic_update_ppr(vcpu->arch.apic);
717 }
718 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
719 
720 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
721 {
722 	kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
723 	apic_update_ppr(apic);
724 }
725 
726 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
727 {
728 	return mda == (apic_x2apic_mode(apic) ?
729 			X2APIC_BROADCAST : APIC_BROADCAST);
730 }
731 
732 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
733 {
734 	if (kvm_apic_broadcast(apic, mda))
735 		return true;
736 
737 	if (apic_x2apic_mode(apic))
738 		return mda == kvm_x2apic_id(apic);
739 
740 	/*
741 	 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
742 	 * it were in x2APIC mode.  Hotplugged VCPUs start in xAPIC mode and
743 	 * this allows unique addressing of VCPUs with APIC ID over 0xff.
744 	 * The 0xff condition is needed because writeable xAPIC ID.
745 	 */
746 	if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
747 		return true;
748 
749 	return mda == kvm_xapic_id(apic);
750 }
751 
752 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
753 {
754 	u32 logical_id;
755 
756 	if (kvm_apic_broadcast(apic, mda))
757 		return true;
758 
759 	logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
760 
761 	if (apic_x2apic_mode(apic))
762 		return ((logical_id >> 16) == (mda >> 16))
763 		       && (logical_id & mda & 0xffff) != 0;
764 
765 	logical_id = GET_APIC_LOGICAL_ID(logical_id);
766 
767 	switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
768 	case APIC_DFR_FLAT:
769 		return (logical_id & mda) != 0;
770 	case APIC_DFR_CLUSTER:
771 		return ((logical_id >> 4) == (mda >> 4))
772 		       && (logical_id & mda & 0xf) != 0;
773 	default:
774 		return false;
775 	}
776 }
777 
778 /* The KVM local APIC implementation has two quirks:
779  *
780  *  - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
781  *    in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
782  *    KVM doesn't do that aliasing.
783  *
784  *  - in-kernel IOAPIC messages have to be delivered directly to
785  *    x2APIC, because the kernel does not support interrupt remapping.
786  *    In order to support broadcast without interrupt remapping, x2APIC
787  *    rewrites the destination of non-IPI messages from APIC_BROADCAST
788  *    to X2APIC_BROADCAST.
789  *
790  * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
791  * important when userspace wants to use x2APIC-format MSIs, because
792  * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
793  */
794 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
795 		struct kvm_lapic *source, struct kvm_lapic *target)
796 {
797 	bool ipi = source != NULL;
798 
799 	if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
800 	    !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
801 		return X2APIC_BROADCAST;
802 
803 	return dest_id;
804 }
805 
806 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
807 			   int short_hand, unsigned int dest, int dest_mode)
808 {
809 	struct kvm_lapic *target = vcpu->arch.apic;
810 	u32 mda = kvm_apic_mda(vcpu, dest, source, target);
811 
812 	ASSERT(target);
813 	switch (short_hand) {
814 	case APIC_DEST_NOSHORT:
815 		if (dest_mode == APIC_DEST_PHYSICAL)
816 			return kvm_apic_match_physical_addr(target, mda);
817 		else
818 			return kvm_apic_match_logical_addr(target, mda);
819 	case APIC_DEST_SELF:
820 		return target == source;
821 	case APIC_DEST_ALLINC:
822 		return true;
823 	case APIC_DEST_ALLBUT:
824 		return target != source;
825 	default:
826 		return false;
827 	}
828 }
829 EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
830 
831 int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
832 		       const unsigned long *bitmap, u32 bitmap_size)
833 {
834 	u32 mod;
835 	int i, idx = -1;
836 
837 	mod = vector % dest_vcpus;
838 
839 	for (i = 0; i <= mod; i++) {
840 		idx = find_next_bit(bitmap, bitmap_size, idx + 1);
841 		BUG_ON(idx == bitmap_size);
842 	}
843 
844 	return idx;
845 }
846 
847 static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
848 {
849 	if (!kvm->arch.disabled_lapic_found) {
850 		kvm->arch.disabled_lapic_found = true;
851 		printk(KERN_INFO
852 		       "Disabled LAPIC found during irq injection\n");
853 	}
854 }
855 
856 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
857 		struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
858 {
859 	if (kvm->arch.x2apic_broadcast_quirk_disabled) {
860 		if ((irq->dest_id == APIC_BROADCAST &&
861 				map->mode != KVM_APIC_MODE_X2APIC))
862 			return true;
863 		if (irq->dest_id == X2APIC_BROADCAST)
864 			return true;
865 	} else {
866 		bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
867 		if (irq->dest_id == (x2apic_ipi ?
868 		                     X2APIC_BROADCAST : APIC_BROADCAST))
869 			return true;
870 	}
871 
872 	return false;
873 }
874 
875 /* Return true if the interrupt can be handled by using *bitmap as index mask
876  * for valid destinations in *dst array.
877  * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
878  * Note: we may have zero kvm_lapic destinations when we return true, which
879  * means that the interrupt should be dropped.  In this case, *bitmap would be
880  * zero and *dst undefined.
881  */
882 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
883 		struct kvm_lapic **src, struct kvm_lapic_irq *irq,
884 		struct kvm_apic_map *map, struct kvm_lapic ***dst,
885 		unsigned long *bitmap)
886 {
887 	int i, lowest;
888 
889 	if (irq->shorthand == APIC_DEST_SELF && src) {
890 		*dst = src;
891 		*bitmap = 1;
892 		return true;
893 	} else if (irq->shorthand)
894 		return false;
895 
896 	if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
897 		return false;
898 
899 	if (irq->dest_mode == APIC_DEST_PHYSICAL) {
900 		if (irq->dest_id > map->max_apic_id) {
901 			*bitmap = 0;
902 		} else {
903 			u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1);
904 			*dst = &map->phys_map[dest_id];
905 			*bitmap = 1;
906 		}
907 		return true;
908 	}
909 
910 	*bitmap = 0;
911 	if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
912 				(u16 *)bitmap))
913 		return false;
914 
915 	if (!kvm_lowest_prio_delivery(irq))
916 		return true;
917 
918 	if (!kvm_vector_hashing_enabled()) {
919 		lowest = -1;
920 		for_each_set_bit(i, bitmap, 16) {
921 			if (!(*dst)[i])
922 				continue;
923 			if (lowest < 0)
924 				lowest = i;
925 			else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
926 						(*dst)[lowest]->vcpu) < 0)
927 				lowest = i;
928 		}
929 	} else {
930 		if (!*bitmap)
931 			return true;
932 
933 		lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
934 				bitmap, 16);
935 
936 		if (!(*dst)[lowest]) {
937 			kvm_apic_disabled_lapic_found(kvm);
938 			*bitmap = 0;
939 			return true;
940 		}
941 	}
942 
943 	*bitmap = (lowest >= 0) ? 1 << lowest : 0;
944 
945 	return true;
946 }
947 
948 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
949 		struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
950 {
951 	struct kvm_apic_map *map;
952 	unsigned long bitmap;
953 	struct kvm_lapic **dst = NULL;
954 	int i;
955 	bool ret;
956 
957 	*r = -1;
958 
959 	if (irq->shorthand == APIC_DEST_SELF) {
960 		*r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
961 		return true;
962 	}
963 
964 	rcu_read_lock();
965 	map = rcu_dereference(kvm->arch.apic_map);
966 
967 	ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
968 	if (ret) {
969 		*r = 0;
970 		for_each_set_bit(i, &bitmap, 16) {
971 			if (!dst[i])
972 				continue;
973 			*r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
974 		}
975 	}
976 
977 	rcu_read_unlock();
978 	return ret;
979 }
980 
981 /*
982  * This routine tries to handler interrupts in posted mode, here is how
983  * it deals with different cases:
984  * - For single-destination interrupts, handle it in posted mode
985  * - Else if vector hashing is enabled and it is a lowest-priority
986  *   interrupt, handle it in posted mode and use the following mechanism
987  *   to find the destinaiton vCPU.
988  *	1. For lowest-priority interrupts, store all the possible
989  *	   destination vCPUs in an array.
990  *	2. Use "guest vector % max number of destination vCPUs" to find
991  *	   the right destination vCPU in the array for the lowest-priority
992  *	   interrupt.
993  * - Otherwise, use remapped mode to inject the interrupt.
994  */
995 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
996 			struct kvm_vcpu **dest_vcpu)
997 {
998 	struct kvm_apic_map *map;
999 	unsigned long bitmap;
1000 	struct kvm_lapic **dst = NULL;
1001 	bool ret = false;
1002 
1003 	if (irq->shorthand)
1004 		return false;
1005 
1006 	rcu_read_lock();
1007 	map = rcu_dereference(kvm->arch.apic_map);
1008 
1009 	if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
1010 			hweight16(bitmap) == 1) {
1011 		unsigned long i = find_first_bit(&bitmap, 16);
1012 
1013 		if (dst[i]) {
1014 			*dest_vcpu = dst[i]->vcpu;
1015 			ret = true;
1016 		}
1017 	}
1018 
1019 	rcu_read_unlock();
1020 	return ret;
1021 }
1022 
1023 /*
1024  * Add a pending IRQ into lapic.
1025  * Return 1 if successfully added and 0 if discarded.
1026  */
1027 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
1028 			     int vector, int level, int trig_mode,
1029 			     struct dest_map *dest_map)
1030 {
1031 	int result = 0;
1032 	struct kvm_vcpu *vcpu = apic->vcpu;
1033 
1034 	trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
1035 				  trig_mode, vector);
1036 	switch (delivery_mode) {
1037 	case APIC_DM_LOWEST:
1038 		vcpu->arch.apic_arb_prio++;
1039 		/* fall through */
1040 	case APIC_DM_FIXED:
1041 		if (unlikely(trig_mode && !level))
1042 			break;
1043 
1044 		/* FIXME add logic for vcpu on reset */
1045 		if (unlikely(!apic_enabled(apic)))
1046 			break;
1047 
1048 		result = 1;
1049 
1050 		if (dest_map) {
1051 			__set_bit(vcpu->vcpu_id, dest_map->map);
1052 			dest_map->vectors[vcpu->vcpu_id] = vector;
1053 		}
1054 
1055 		if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
1056 			if (trig_mode)
1057 				kvm_lapic_set_vector(vector,
1058 						     apic->regs + APIC_TMR);
1059 			else
1060 				kvm_lapic_clear_vector(vector,
1061 						       apic->regs + APIC_TMR);
1062 		}
1063 
1064 		if (vcpu->arch.apicv_active)
1065 			kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
1066 		else {
1067 			kvm_lapic_set_irr(vector, apic);
1068 
1069 			kvm_make_request(KVM_REQ_EVENT, vcpu);
1070 			kvm_vcpu_kick(vcpu);
1071 		}
1072 		break;
1073 
1074 	case APIC_DM_REMRD:
1075 		result = 1;
1076 		vcpu->arch.pv.pv_unhalted = 1;
1077 		kvm_make_request(KVM_REQ_EVENT, vcpu);
1078 		kvm_vcpu_kick(vcpu);
1079 		break;
1080 
1081 	case APIC_DM_SMI:
1082 		result = 1;
1083 		kvm_make_request(KVM_REQ_SMI, vcpu);
1084 		kvm_vcpu_kick(vcpu);
1085 		break;
1086 
1087 	case APIC_DM_NMI:
1088 		result = 1;
1089 		kvm_inject_nmi(vcpu);
1090 		kvm_vcpu_kick(vcpu);
1091 		break;
1092 
1093 	case APIC_DM_INIT:
1094 		if (!trig_mode || level) {
1095 			result = 1;
1096 			/* assumes that there are only KVM_APIC_INIT/SIPI */
1097 			apic->pending_events = (1UL << KVM_APIC_INIT);
1098 			/* make sure pending_events is visible before sending
1099 			 * the request */
1100 			smp_wmb();
1101 			kvm_make_request(KVM_REQ_EVENT, vcpu);
1102 			kvm_vcpu_kick(vcpu);
1103 		}
1104 		break;
1105 
1106 	case APIC_DM_STARTUP:
1107 		result = 1;
1108 		apic->sipi_vector = vector;
1109 		/* make sure sipi_vector is visible for the receiver */
1110 		smp_wmb();
1111 		set_bit(KVM_APIC_SIPI, &apic->pending_events);
1112 		kvm_make_request(KVM_REQ_EVENT, vcpu);
1113 		kvm_vcpu_kick(vcpu);
1114 		break;
1115 
1116 	case APIC_DM_EXTINT:
1117 		/*
1118 		 * Should only be called by kvm_apic_local_deliver() with LVT0,
1119 		 * before NMI watchdog was enabled. Already handled by
1120 		 * kvm_apic_accept_pic_intr().
1121 		 */
1122 		break;
1123 
1124 	default:
1125 		printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1126 		       delivery_mode);
1127 		break;
1128 	}
1129 	return result;
1130 }
1131 
1132 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
1133 {
1134 	return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
1135 }
1136 
1137 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1138 {
1139 	return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
1140 }
1141 
1142 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1143 {
1144 	int trigger_mode;
1145 
1146 	/* Eoi the ioapic only if the ioapic doesn't own the vector. */
1147 	if (!kvm_ioapic_handles_vector(apic, vector))
1148 		return;
1149 
1150 	/* Request a KVM exit to inform the userspace IOAPIC. */
1151 	if (irqchip_split(apic->vcpu->kvm)) {
1152 		apic->vcpu->arch.pending_ioapic_eoi = vector;
1153 		kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1154 		return;
1155 	}
1156 
1157 	if (apic_test_vector(vector, apic->regs + APIC_TMR))
1158 		trigger_mode = IOAPIC_LEVEL_TRIG;
1159 	else
1160 		trigger_mode = IOAPIC_EDGE_TRIG;
1161 
1162 	kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
1163 }
1164 
1165 static int apic_set_eoi(struct kvm_lapic *apic)
1166 {
1167 	int vector = apic_find_highest_isr(apic);
1168 
1169 	trace_kvm_eoi(apic, vector);
1170 
1171 	/*
1172 	 * Not every write EOI will has corresponding ISR,
1173 	 * one example is when Kernel check timer on setup_IO_APIC
1174 	 */
1175 	if (vector == -1)
1176 		return vector;
1177 
1178 	apic_clear_isr(vector, apic);
1179 	apic_update_ppr(apic);
1180 
1181 	if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
1182 		kvm_hv_synic_send_eoi(apic->vcpu, vector);
1183 
1184 	kvm_ioapic_send_eoi(apic, vector);
1185 	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1186 	return vector;
1187 }
1188 
1189 /*
1190  * this interface assumes a trap-like exit, which has already finished
1191  * desired side effect including vISR and vPPR update.
1192  */
1193 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1194 {
1195 	struct kvm_lapic *apic = vcpu->arch.apic;
1196 
1197 	trace_kvm_eoi(apic, vector);
1198 
1199 	kvm_ioapic_send_eoi(apic, vector);
1200 	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1201 }
1202 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1203 
1204 static void apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high)
1205 {
1206 	struct kvm_lapic_irq irq;
1207 
1208 	irq.vector = icr_low & APIC_VECTOR_MASK;
1209 	irq.delivery_mode = icr_low & APIC_MODE_MASK;
1210 	irq.dest_mode = icr_low & APIC_DEST_MASK;
1211 	irq.level = (icr_low & APIC_INT_ASSERT) != 0;
1212 	irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1213 	irq.shorthand = icr_low & APIC_SHORT_MASK;
1214 	irq.msi_redir_hint = false;
1215 	if (apic_x2apic_mode(apic))
1216 		irq.dest_id = icr_high;
1217 	else
1218 		irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
1219 
1220 	trace_kvm_apic_ipi(icr_low, irq.dest_id);
1221 
1222 	kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
1223 }
1224 
1225 static u32 apic_get_tmcct(struct kvm_lapic *apic)
1226 {
1227 	ktime_t remaining, now;
1228 	s64 ns;
1229 	u32 tmcct;
1230 
1231 	ASSERT(apic != NULL);
1232 
1233 	/* if initial count is 0, current count should also be 0 */
1234 	if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
1235 		apic->lapic_timer.period == 0)
1236 		return 0;
1237 
1238 	now = ktime_get();
1239 	remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1240 	if (ktime_to_ns(remaining) < 0)
1241 		remaining = 0;
1242 
1243 	ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1244 	tmcct = div64_u64(ns,
1245 			 (APIC_BUS_CYCLE_NS * apic->divide_count));
1246 
1247 	return tmcct;
1248 }
1249 
1250 static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1251 {
1252 	struct kvm_vcpu *vcpu = apic->vcpu;
1253 	struct kvm_run *run = vcpu->run;
1254 
1255 	kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
1256 	run->tpr_access.rip = kvm_rip_read(vcpu);
1257 	run->tpr_access.is_write = write;
1258 }
1259 
1260 static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1261 {
1262 	if (apic->vcpu->arch.tpr_access_reporting)
1263 		__report_tpr_access(apic, write);
1264 }
1265 
1266 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1267 {
1268 	u32 val = 0;
1269 
1270 	if (offset >= LAPIC_MMIO_LENGTH)
1271 		return 0;
1272 
1273 	switch (offset) {
1274 	case APIC_ARBPRI:
1275 		break;
1276 
1277 	case APIC_TMCCT:	/* Timer CCR */
1278 		if (apic_lvtt_tscdeadline(apic))
1279 			return 0;
1280 
1281 		val = apic_get_tmcct(apic);
1282 		break;
1283 	case APIC_PROCPRI:
1284 		apic_update_ppr(apic);
1285 		val = kvm_lapic_get_reg(apic, offset);
1286 		break;
1287 	case APIC_TASKPRI:
1288 		report_tpr_access(apic, false);
1289 		/* fall thru */
1290 	default:
1291 		val = kvm_lapic_get_reg(apic, offset);
1292 		break;
1293 	}
1294 
1295 	return val;
1296 }
1297 
1298 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1299 {
1300 	return container_of(dev, struct kvm_lapic, dev);
1301 }
1302 
1303 #define APIC_REG_MASK(reg)	(1ull << ((reg) >> 4))
1304 #define APIC_REGS_MASK(first, count) \
1305 	(APIC_REG_MASK(first) * ((1ull << (count)) - 1))
1306 
1307 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
1308 		void *data)
1309 {
1310 	unsigned char alignment = offset & 0xf;
1311 	u32 result;
1312 	/* this bitmask has a bit cleared for each reserved register */
1313 	u64 valid_reg_mask =
1314 		APIC_REG_MASK(APIC_ID) |
1315 		APIC_REG_MASK(APIC_LVR) |
1316 		APIC_REG_MASK(APIC_TASKPRI) |
1317 		APIC_REG_MASK(APIC_PROCPRI) |
1318 		APIC_REG_MASK(APIC_LDR) |
1319 		APIC_REG_MASK(APIC_DFR) |
1320 		APIC_REG_MASK(APIC_SPIV) |
1321 		APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) |
1322 		APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) |
1323 		APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) |
1324 		APIC_REG_MASK(APIC_ESR) |
1325 		APIC_REG_MASK(APIC_ICR) |
1326 		APIC_REG_MASK(APIC_ICR2) |
1327 		APIC_REG_MASK(APIC_LVTT) |
1328 		APIC_REG_MASK(APIC_LVTTHMR) |
1329 		APIC_REG_MASK(APIC_LVTPC) |
1330 		APIC_REG_MASK(APIC_LVT0) |
1331 		APIC_REG_MASK(APIC_LVT1) |
1332 		APIC_REG_MASK(APIC_LVTERR) |
1333 		APIC_REG_MASK(APIC_TMICT) |
1334 		APIC_REG_MASK(APIC_TMCCT) |
1335 		APIC_REG_MASK(APIC_TDCR);
1336 
1337 	/* ARBPRI is not valid on x2APIC */
1338 	if (!apic_x2apic_mode(apic))
1339 		valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI);
1340 
1341 	if (offset > 0x3f0 || !(valid_reg_mask & APIC_REG_MASK(offset)))
1342 		return 1;
1343 
1344 	result = __apic_read(apic, offset & ~0xf);
1345 
1346 	trace_kvm_apic_read(offset, result);
1347 
1348 	switch (len) {
1349 	case 1:
1350 	case 2:
1351 	case 4:
1352 		memcpy(data, (char *)&result + alignment, len);
1353 		break;
1354 	default:
1355 		printk(KERN_ERR "Local APIC read with len = %x, "
1356 		       "should be 1,2, or 4 instead\n", len);
1357 		break;
1358 	}
1359 	return 0;
1360 }
1361 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
1362 
1363 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1364 {
1365 	return addr >= apic->base_address &&
1366 		addr < apic->base_address + LAPIC_MMIO_LENGTH;
1367 }
1368 
1369 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1370 			   gpa_t address, int len, void *data)
1371 {
1372 	struct kvm_lapic *apic = to_lapic(this);
1373 	u32 offset = address - apic->base_address;
1374 
1375 	if (!apic_mmio_in_range(apic, address))
1376 		return -EOPNOTSUPP;
1377 
1378 	if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1379 		if (!kvm_check_has_quirk(vcpu->kvm,
1380 					 KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1381 			return -EOPNOTSUPP;
1382 
1383 		memset(data, 0xff, len);
1384 		return 0;
1385 	}
1386 
1387 	kvm_lapic_reg_read(apic, offset, len, data);
1388 
1389 	return 0;
1390 }
1391 
1392 static void update_divide_count(struct kvm_lapic *apic)
1393 {
1394 	u32 tmp1, tmp2, tdcr;
1395 
1396 	tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
1397 	tmp1 = tdcr & 0xf;
1398 	tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
1399 	apic->divide_count = 0x1 << (tmp2 & 0x7);
1400 }
1401 
1402 static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
1403 {
1404 	/*
1405 	 * Do not allow the guest to program periodic timers with small
1406 	 * interval, since the hrtimers are not throttled by the host
1407 	 * scheduler.
1408 	 */
1409 	if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1410 		s64 min_period = min_timer_period_us * 1000LL;
1411 
1412 		if (apic->lapic_timer.period < min_period) {
1413 			pr_info_ratelimited(
1414 			    "kvm: vcpu %i: requested %lld ns "
1415 			    "lapic timer period limited to %lld ns\n",
1416 			    apic->vcpu->vcpu_id,
1417 			    apic->lapic_timer.period, min_period);
1418 			apic->lapic_timer.period = min_period;
1419 		}
1420 	}
1421 }
1422 
1423 static void apic_update_lvtt(struct kvm_lapic *apic)
1424 {
1425 	u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
1426 			apic->lapic_timer.timer_mode_mask;
1427 
1428 	if (apic->lapic_timer.timer_mode != timer_mode) {
1429 		if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
1430 				APIC_LVT_TIMER_TSCDEADLINE)) {
1431 			hrtimer_cancel(&apic->lapic_timer.timer);
1432 			kvm_lapic_set_reg(apic, APIC_TMICT, 0);
1433 			apic->lapic_timer.period = 0;
1434 			apic->lapic_timer.tscdeadline = 0;
1435 		}
1436 		apic->lapic_timer.timer_mode = timer_mode;
1437 		limit_periodic_timer_frequency(apic);
1438 	}
1439 }
1440 
1441 /*
1442  * On APICv, this test will cause a busy wait
1443  * during a higher-priority task.
1444  */
1445 
1446 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1447 {
1448 	struct kvm_lapic *apic = vcpu->arch.apic;
1449 	u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1450 
1451 	if (kvm_apic_hw_enabled(apic)) {
1452 		int vec = reg & APIC_VECTOR_MASK;
1453 		void *bitmap = apic->regs + APIC_ISR;
1454 
1455 		if (vcpu->arch.apicv_active)
1456 			bitmap = apic->regs + APIC_IRR;
1457 
1458 		if (apic_test_vector(vec, bitmap))
1459 			return true;
1460 	}
1461 	return false;
1462 }
1463 
1464 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles)
1465 {
1466 	u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns;
1467 
1468 	/*
1469 	 * If the guest TSC is running at a different ratio than the host, then
1470 	 * convert the delay to nanoseconds to achieve an accurate delay.  Note
1471 	 * that __delay() uses delay_tsc whenever the hardware has TSC, thus
1472 	 * always for VMX enabled hardware.
1473 	 */
1474 	if (vcpu->arch.tsc_scaling_ratio == kvm_default_tsc_scaling_ratio) {
1475 		__delay(min(guest_cycles,
1476 			nsec_to_cycles(vcpu, timer_advance_ns)));
1477 	} else {
1478 		u64 delay_ns = guest_cycles * 1000000ULL;
1479 		do_div(delay_ns, vcpu->arch.virtual_tsc_khz);
1480 		ndelay(min_t(u32, delay_ns, timer_advance_ns));
1481 	}
1482 }
1483 
1484 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
1485 					      s64 advance_expire_delta)
1486 {
1487 	struct kvm_lapic *apic = vcpu->arch.apic;
1488 	u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
1489 	u64 ns;
1490 
1491 	/* Do not adjust for tiny fluctuations or large random spikes. */
1492 	if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX ||
1493 	    abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN)
1494 		return;
1495 
1496 	/* too early */
1497 	if (advance_expire_delta < 0) {
1498 		ns = -advance_expire_delta * 1000000ULL;
1499 		do_div(ns, vcpu->arch.virtual_tsc_khz);
1500 		timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1501 	} else {
1502 	/* too late */
1503 		ns = advance_expire_delta * 1000000ULL;
1504 		do_div(ns, vcpu->arch.virtual_tsc_khz);
1505 		timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1506 	}
1507 
1508 	if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX))
1509 		timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
1510 	apic->lapic_timer.timer_advance_ns = timer_advance_ns;
1511 }
1512 
1513 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1514 {
1515 	struct kvm_lapic *apic = vcpu->arch.apic;
1516 	u64 guest_tsc, tsc_deadline;
1517 
1518 	if (apic->lapic_timer.expired_tscdeadline == 0)
1519 		return;
1520 
1521 	tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1522 	apic->lapic_timer.expired_tscdeadline = 0;
1523 	guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1524 	apic->lapic_timer.advance_expire_delta = guest_tsc - tsc_deadline;
1525 
1526 	if (guest_tsc < tsc_deadline)
1527 		__wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
1528 
1529 	if (lapic_timer_advance_dynamic)
1530 		adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
1531 }
1532 
1533 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1534 {
1535 	if (lapic_timer_int_injected(vcpu))
1536 		__kvm_wait_lapic_expire(vcpu);
1537 }
1538 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire);
1539 
1540 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic)
1541 {
1542 	struct kvm_timer *ktimer = &apic->lapic_timer;
1543 
1544 	kvm_apic_local_deliver(apic, APIC_LVTT);
1545 	if (apic_lvtt_tscdeadline(apic))
1546 		ktimer->tscdeadline = 0;
1547 	if (apic_lvtt_oneshot(apic)) {
1548 		ktimer->tscdeadline = 0;
1549 		ktimer->target_expiration = 0;
1550 	}
1551 }
1552 
1553 static void apic_timer_expired(struct kvm_lapic *apic)
1554 {
1555 	struct kvm_vcpu *vcpu = apic->vcpu;
1556 	struct kvm_timer *ktimer = &apic->lapic_timer;
1557 
1558 	if (atomic_read(&apic->lapic_timer.pending))
1559 		return;
1560 
1561 	if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
1562 		ktimer->expired_tscdeadline = ktimer->tscdeadline;
1563 
1564 	if (kvm_use_posted_timer_interrupt(apic->vcpu)) {
1565 		if (apic->lapic_timer.timer_advance_ns)
1566 			__kvm_wait_lapic_expire(vcpu);
1567 		kvm_apic_inject_pending_timer_irqs(apic);
1568 		return;
1569 	}
1570 
1571 	atomic_inc(&apic->lapic_timer.pending);
1572 	kvm_set_pending_timer(vcpu);
1573 }
1574 
1575 static void start_sw_tscdeadline(struct kvm_lapic *apic)
1576 {
1577 	struct kvm_timer *ktimer = &apic->lapic_timer;
1578 	u64 guest_tsc, tscdeadline = ktimer->tscdeadline;
1579 	u64 ns = 0;
1580 	ktime_t expire;
1581 	struct kvm_vcpu *vcpu = apic->vcpu;
1582 	unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1583 	unsigned long flags;
1584 	ktime_t now;
1585 
1586 	if (unlikely(!tscdeadline || !this_tsc_khz))
1587 		return;
1588 
1589 	local_irq_save(flags);
1590 
1591 	now = ktime_get();
1592 	guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1593 
1594 	ns = (tscdeadline - guest_tsc) * 1000000ULL;
1595 	do_div(ns, this_tsc_khz);
1596 
1597 	if (likely(tscdeadline > guest_tsc) &&
1598 	    likely(ns > apic->lapic_timer.timer_advance_ns)) {
1599 		expire = ktime_add_ns(now, ns);
1600 		expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
1601 		hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD);
1602 	} else
1603 		apic_timer_expired(apic);
1604 
1605 	local_irq_restore(flags);
1606 }
1607 
1608 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
1609 {
1610 	ktime_t now, remaining;
1611 	u64 ns_remaining_old, ns_remaining_new;
1612 
1613 	apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1614 		* APIC_BUS_CYCLE_NS * apic->divide_count;
1615 	limit_periodic_timer_frequency(apic);
1616 
1617 	now = ktime_get();
1618 	remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1619 	if (ktime_to_ns(remaining) < 0)
1620 		remaining = 0;
1621 
1622 	ns_remaining_old = ktime_to_ns(remaining);
1623 	ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
1624 	                                   apic->divide_count, old_divisor);
1625 
1626 	apic->lapic_timer.tscdeadline +=
1627 		nsec_to_cycles(apic->vcpu, ns_remaining_new) -
1628 		nsec_to_cycles(apic->vcpu, ns_remaining_old);
1629 	apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
1630 }
1631 
1632 static bool set_target_expiration(struct kvm_lapic *apic)
1633 {
1634 	ktime_t now;
1635 	u64 tscl = rdtsc();
1636 
1637 	now = ktime_get();
1638 	apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1639 		* APIC_BUS_CYCLE_NS * apic->divide_count;
1640 
1641 	if (!apic->lapic_timer.period) {
1642 		apic->lapic_timer.tscdeadline = 0;
1643 		return false;
1644 	}
1645 
1646 	limit_periodic_timer_frequency(apic);
1647 
1648 	apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1649 		nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1650 	apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);
1651 
1652 	return true;
1653 }
1654 
1655 static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1656 {
1657 	ktime_t now = ktime_get();
1658 	u64 tscl = rdtsc();
1659 	ktime_t delta;
1660 
1661 	/*
1662 	 * Synchronize both deadlines to the same time source or
1663 	 * differences in the periods (caused by differences in the
1664 	 * underlying clocks or numerical approximation errors) will
1665 	 * cause the two to drift apart over time as the errors
1666 	 * accumulate.
1667 	 */
1668 	apic->lapic_timer.target_expiration =
1669 		ktime_add_ns(apic->lapic_timer.target_expiration,
1670 				apic->lapic_timer.period);
1671 	delta = ktime_sub(apic->lapic_timer.target_expiration, now);
1672 	apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1673 		nsec_to_cycles(apic->vcpu, delta);
1674 }
1675 
1676 static void start_sw_period(struct kvm_lapic *apic)
1677 {
1678 	if (!apic->lapic_timer.period)
1679 		return;
1680 
1681 	if (ktime_after(ktime_get(),
1682 			apic->lapic_timer.target_expiration)) {
1683 		apic_timer_expired(apic);
1684 
1685 		if (apic_lvtt_oneshot(apic))
1686 			return;
1687 
1688 		advance_periodic_target_expiration(apic);
1689 	}
1690 
1691 	hrtimer_start(&apic->lapic_timer.timer,
1692 		apic->lapic_timer.target_expiration,
1693 		HRTIMER_MODE_ABS);
1694 }
1695 
1696 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1697 {
1698 	if (!lapic_in_kernel(vcpu))
1699 		return false;
1700 
1701 	return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1702 }
1703 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1704 
1705 static void cancel_hv_timer(struct kvm_lapic *apic)
1706 {
1707 	WARN_ON(preemptible());
1708 	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1709 	kvm_x86_ops->cancel_hv_timer(apic->vcpu);
1710 	apic->lapic_timer.hv_timer_in_use = false;
1711 }
1712 
1713 static bool start_hv_timer(struct kvm_lapic *apic)
1714 {
1715 	struct kvm_timer *ktimer = &apic->lapic_timer;
1716 	struct kvm_vcpu *vcpu = apic->vcpu;
1717 	bool expired;
1718 
1719 	WARN_ON(preemptible());
1720 	if (!kvm_x86_ops->set_hv_timer)
1721 		return false;
1722 
1723 	if (!ktimer->tscdeadline)
1724 		return false;
1725 
1726 	if (kvm_x86_ops->set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
1727 		return false;
1728 
1729 	ktimer->hv_timer_in_use = true;
1730 	hrtimer_cancel(&ktimer->timer);
1731 
1732 	/*
1733 	 * To simplify handling the periodic timer, leave the hv timer running
1734 	 * even if the deadline timer has expired, i.e. rely on the resulting
1735 	 * VM-Exit to recompute the periodic timer's target expiration.
1736 	 */
1737 	if (!apic_lvtt_period(apic)) {
1738 		/*
1739 		 * Cancel the hv timer if the sw timer fired while the hv timer
1740 		 * was being programmed, or if the hv timer itself expired.
1741 		 */
1742 		if (atomic_read(&ktimer->pending)) {
1743 			cancel_hv_timer(apic);
1744 		} else if (expired) {
1745 			apic_timer_expired(apic);
1746 			cancel_hv_timer(apic);
1747 		}
1748 	}
1749 
1750 	trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);
1751 
1752 	return true;
1753 }
1754 
1755 static void start_sw_timer(struct kvm_lapic *apic)
1756 {
1757 	struct kvm_timer *ktimer = &apic->lapic_timer;
1758 
1759 	WARN_ON(preemptible());
1760 	if (apic->lapic_timer.hv_timer_in_use)
1761 		cancel_hv_timer(apic);
1762 	if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1763 		return;
1764 
1765 	if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1766 		start_sw_period(apic);
1767 	else if (apic_lvtt_tscdeadline(apic))
1768 		start_sw_tscdeadline(apic);
1769 	trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
1770 }
1771 
1772 static void restart_apic_timer(struct kvm_lapic *apic)
1773 {
1774 	preempt_disable();
1775 
1776 	if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending))
1777 		goto out;
1778 
1779 	if (!start_hv_timer(apic))
1780 		start_sw_timer(apic);
1781 out:
1782 	preempt_enable();
1783 }
1784 
1785 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1786 {
1787 	struct kvm_lapic *apic = vcpu->arch.apic;
1788 
1789 	preempt_disable();
1790 	/* If the preempt notifier has already run, it also called apic_timer_expired */
1791 	if (!apic->lapic_timer.hv_timer_in_use)
1792 		goto out;
1793 	WARN_ON(swait_active(&vcpu->wq));
1794 	cancel_hv_timer(apic);
1795 	apic_timer_expired(apic);
1796 
1797 	if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1798 		advance_periodic_target_expiration(apic);
1799 		restart_apic_timer(apic);
1800 	}
1801 out:
1802 	preempt_enable();
1803 }
1804 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1805 
1806 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1807 {
1808 	restart_apic_timer(vcpu->arch.apic);
1809 }
1810 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1811 
1812 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1813 {
1814 	struct kvm_lapic *apic = vcpu->arch.apic;
1815 
1816 	preempt_disable();
1817 	/* Possibly the TSC deadline timer is not enabled yet */
1818 	if (apic->lapic_timer.hv_timer_in_use)
1819 		start_sw_timer(apic);
1820 	preempt_enable();
1821 }
1822 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
1823 
1824 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
1825 {
1826 	struct kvm_lapic *apic = vcpu->arch.apic;
1827 
1828 	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1829 	restart_apic_timer(apic);
1830 }
1831 
1832 static void start_apic_timer(struct kvm_lapic *apic)
1833 {
1834 	atomic_set(&apic->lapic_timer.pending, 0);
1835 
1836 	if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1837 	    && !set_target_expiration(apic))
1838 		return;
1839 
1840 	restart_apic_timer(apic);
1841 }
1842 
1843 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1844 {
1845 	bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
1846 
1847 	if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1848 		apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1849 		if (lvt0_in_nmi_mode) {
1850 			atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1851 		} else
1852 			atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1853 	}
1854 }
1855 
1856 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
1857 {
1858 	int ret = 0;
1859 
1860 	trace_kvm_apic_write(reg, val);
1861 
1862 	switch (reg) {
1863 	case APIC_ID:		/* Local APIC ID */
1864 		if (!apic_x2apic_mode(apic))
1865 			kvm_apic_set_xapic_id(apic, val >> 24);
1866 		else
1867 			ret = 1;
1868 		break;
1869 
1870 	case APIC_TASKPRI:
1871 		report_tpr_access(apic, true);
1872 		apic_set_tpr(apic, val & 0xff);
1873 		break;
1874 
1875 	case APIC_EOI:
1876 		apic_set_eoi(apic);
1877 		break;
1878 
1879 	case APIC_LDR:
1880 		if (!apic_x2apic_mode(apic))
1881 			kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
1882 		else
1883 			ret = 1;
1884 		break;
1885 
1886 	case APIC_DFR:
1887 		if (!apic_x2apic_mode(apic)) {
1888 			kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1889 			recalculate_apic_map(apic->vcpu->kvm);
1890 		} else
1891 			ret = 1;
1892 		break;
1893 
1894 	case APIC_SPIV: {
1895 		u32 mask = 0x3ff;
1896 		if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
1897 			mask |= APIC_SPIV_DIRECTED_EOI;
1898 		apic_set_spiv(apic, val & mask);
1899 		if (!(val & APIC_SPIV_APIC_ENABLED)) {
1900 			int i;
1901 			u32 lvt_val;
1902 
1903 			for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
1904 				lvt_val = kvm_lapic_get_reg(apic,
1905 						       APIC_LVTT + 0x10 * i);
1906 				kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
1907 					     lvt_val | APIC_LVT_MASKED);
1908 			}
1909 			apic_update_lvtt(apic);
1910 			atomic_set(&apic->lapic_timer.pending, 0);
1911 
1912 		}
1913 		break;
1914 	}
1915 	case APIC_ICR:
1916 		/* No delay here, so we always clear the pending bit */
1917 		val &= ~(1 << 12);
1918 		apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2));
1919 		kvm_lapic_set_reg(apic, APIC_ICR, val);
1920 		break;
1921 
1922 	case APIC_ICR2:
1923 		if (!apic_x2apic_mode(apic))
1924 			val &= 0xff000000;
1925 		kvm_lapic_set_reg(apic, APIC_ICR2, val);
1926 		break;
1927 
1928 	case APIC_LVT0:
1929 		apic_manage_nmi_watchdog(apic, val);
1930 		/* fall through */
1931 	case APIC_LVTTHMR:
1932 	case APIC_LVTPC:
1933 	case APIC_LVT1:
1934 	case APIC_LVTERR:
1935 		/* TODO: Check vector */
1936 		if (!kvm_apic_sw_enabled(apic))
1937 			val |= APIC_LVT_MASKED;
1938 
1939 		val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
1940 		kvm_lapic_set_reg(apic, reg, val);
1941 
1942 		break;
1943 
1944 	case APIC_LVTT:
1945 		if (!kvm_apic_sw_enabled(apic))
1946 			val |= APIC_LVT_MASKED;
1947 		val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1948 		kvm_lapic_set_reg(apic, APIC_LVTT, val);
1949 		apic_update_lvtt(apic);
1950 		break;
1951 
1952 	case APIC_TMICT:
1953 		if (apic_lvtt_tscdeadline(apic))
1954 			break;
1955 
1956 		hrtimer_cancel(&apic->lapic_timer.timer);
1957 		kvm_lapic_set_reg(apic, APIC_TMICT, val);
1958 		start_apic_timer(apic);
1959 		break;
1960 
1961 	case APIC_TDCR: {
1962 		uint32_t old_divisor = apic->divide_count;
1963 
1964 		kvm_lapic_set_reg(apic, APIC_TDCR, val);
1965 		update_divide_count(apic);
1966 		if (apic->divide_count != old_divisor &&
1967 				apic->lapic_timer.period) {
1968 			hrtimer_cancel(&apic->lapic_timer.timer);
1969 			update_target_expiration(apic, old_divisor);
1970 			restart_apic_timer(apic);
1971 		}
1972 		break;
1973 	}
1974 	case APIC_ESR:
1975 		if (apic_x2apic_mode(apic) && val != 0)
1976 			ret = 1;
1977 		break;
1978 
1979 	case APIC_SELF_IPI:
1980 		if (apic_x2apic_mode(apic)) {
1981 			kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
1982 		} else
1983 			ret = 1;
1984 		break;
1985 	default:
1986 		ret = 1;
1987 		break;
1988 	}
1989 
1990 	return ret;
1991 }
1992 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
1993 
1994 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1995 			    gpa_t address, int len, const void *data)
1996 {
1997 	struct kvm_lapic *apic = to_lapic(this);
1998 	unsigned int offset = address - apic->base_address;
1999 	u32 val;
2000 
2001 	if (!apic_mmio_in_range(apic, address))
2002 		return -EOPNOTSUPP;
2003 
2004 	if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
2005 		if (!kvm_check_has_quirk(vcpu->kvm,
2006 					 KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
2007 			return -EOPNOTSUPP;
2008 
2009 		return 0;
2010 	}
2011 
2012 	/*
2013 	 * APIC register must be aligned on 128-bits boundary.
2014 	 * 32/64/128 bits registers must be accessed thru 32 bits.
2015 	 * Refer SDM 8.4.1
2016 	 */
2017 	if (len != 4 || (offset & 0xf))
2018 		return 0;
2019 
2020 	val = *(u32*)data;
2021 
2022 	kvm_lapic_reg_write(apic, offset & 0xff0, val);
2023 
2024 	return 0;
2025 }
2026 
2027 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
2028 {
2029 	kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
2030 }
2031 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
2032 
2033 /* emulate APIC access in a trap manner */
2034 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
2035 {
2036 	u32 val = 0;
2037 
2038 	/* hw has done the conditional check and inst decode */
2039 	offset &= 0xff0;
2040 
2041 	kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
2042 
2043 	/* TODO: optimize to just emulate side effect w/o one more write */
2044 	kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
2045 }
2046 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
2047 
2048 void kvm_free_lapic(struct kvm_vcpu *vcpu)
2049 {
2050 	struct kvm_lapic *apic = vcpu->arch.apic;
2051 
2052 	if (!vcpu->arch.apic)
2053 		return;
2054 
2055 	hrtimer_cancel(&apic->lapic_timer.timer);
2056 
2057 	if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
2058 		static_key_slow_dec_deferred(&apic_hw_disabled);
2059 
2060 	if (!apic->sw_enabled)
2061 		static_key_slow_dec_deferred(&apic_sw_disabled);
2062 
2063 	if (apic->regs)
2064 		free_page((unsigned long)apic->regs);
2065 
2066 	kfree(apic);
2067 }
2068 
2069 /*
2070  *----------------------------------------------------------------------
2071  * LAPIC interface
2072  *----------------------------------------------------------------------
2073  */
2074 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
2075 {
2076 	struct kvm_lapic *apic = vcpu->arch.apic;
2077 
2078 	if (!lapic_in_kernel(vcpu) ||
2079 		!apic_lvtt_tscdeadline(apic))
2080 		return 0;
2081 
2082 	return apic->lapic_timer.tscdeadline;
2083 }
2084 
2085 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
2086 {
2087 	struct kvm_lapic *apic = vcpu->arch.apic;
2088 
2089 	if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
2090 			apic_lvtt_period(apic))
2091 		return;
2092 
2093 	hrtimer_cancel(&apic->lapic_timer.timer);
2094 	apic->lapic_timer.tscdeadline = data;
2095 	start_apic_timer(apic);
2096 }
2097 
2098 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
2099 {
2100 	struct kvm_lapic *apic = vcpu->arch.apic;
2101 
2102 	apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
2103 		     | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
2104 }
2105 
2106 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
2107 {
2108 	u64 tpr;
2109 
2110 	tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
2111 
2112 	return (tpr & 0xf0) >> 4;
2113 }
2114 
2115 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
2116 {
2117 	u64 old_value = vcpu->arch.apic_base;
2118 	struct kvm_lapic *apic = vcpu->arch.apic;
2119 
2120 	if (!apic)
2121 		value |= MSR_IA32_APICBASE_BSP;
2122 
2123 	vcpu->arch.apic_base = value;
2124 
2125 	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
2126 		kvm_update_cpuid(vcpu);
2127 
2128 	if (!apic)
2129 		return;
2130 
2131 	/* update jump label if enable bit changes */
2132 	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
2133 		if (value & MSR_IA32_APICBASE_ENABLE) {
2134 			kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2135 			static_key_slow_dec_deferred(&apic_hw_disabled);
2136 		} else {
2137 			static_key_slow_inc(&apic_hw_disabled.key);
2138 			recalculate_apic_map(vcpu->kvm);
2139 		}
2140 	}
2141 
2142 	if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
2143 		kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
2144 
2145 	if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
2146 		kvm_x86_ops->set_virtual_apic_mode(vcpu);
2147 
2148 	apic->base_address = apic->vcpu->arch.apic_base &
2149 			     MSR_IA32_APICBASE_BASE;
2150 
2151 	if ((value & MSR_IA32_APICBASE_ENABLE) &&
2152 	     apic->base_address != APIC_DEFAULT_PHYS_BASE)
2153 		pr_warn_once("APIC base relocation is unsupported by KVM");
2154 }
2155 
2156 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
2157 {
2158 	struct kvm_lapic *apic = vcpu->arch.apic;
2159 	int i;
2160 
2161 	if (!apic)
2162 		return;
2163 
2164 	/* Stop the timer in case it's a reset to an active apic */
2165 	hrtimer_cancel(&apic->lapic_timer.timer);
2166 
2167 	if (!init_event) {
2168 		kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
2169 		                         MSR_IA32_APICBASE_ENABLE);
2170 		kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2171 	}
2172 	kvm_apic_set_version(apic->vcpu);
2173 
2174 	for (i = 0; i < KVM_APIC_LVT_NUM; i++)
2175 		kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
2176 	apic_update_lvtt(apic);
2177 	if (kvm_vcpu_is_reset_bsp(vcpu) &&
2178 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
2179 		kvm_lapic_set_reg(apic, APIC_LVT0,
2180 			     SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
2181 	apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2182 
2183 	kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
2184 	apic_set_spiv(apic, 0xff);
2185 	kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
2186 	if (!apic_x2apic_mode(apic))
2187 		kvm_apic_set_ldr(apic, 0);
2188 	kvm_lapic_set_reg(apic, APIC_ESR, 0);
2189 	kvm_lapic_set_reg(apic, APIC_ICR, 0);
2190 	kvm_lapic_set_reg(apic, APIC_ICR2, 0);
2191 	kvm_lapic_set_reg(apic, APIC_TDCR, 0);
2192 	kvm_lapic_set_reg(apic, APIC_TMICT, 0);
2193 	for (i = 0; i < 8; i++) {
2194 		kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
2195 		kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
2196 		kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
2197 	}
2198 	apic->irr_pending = vcpu->arch.apicv_active;
2199 	apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
2200 	apic->highest_isr_cache = -1;
2201 	update_divide_count(apic);
2202 	atomic_set(&apic->lapic_timer.pending, 0);
2203 	if (kvm_vcpu_is_bsp(vcpu))
2204 		kvm_lapic_set_base(vcpu,
2205 				vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
2206 	vcpu->arch.pv_eoi.msr_val = 0;
2207 	apic_update_ppr(apic);
2208 	if (vcpu->arch.apicv_active) {
2209 		kvm_x86_ops->apicv_post_state_restore(vcpu);
2210 		kvm_x86_ops->hwapic_irr_update(vcpu, -1);
2211 		kvm_x86_ops->hwapic_isr_update(vcpu, -1);
2212 	}
2213 
2214 	vcpu->arch.apic_arb_prio = 0;
2215 	vcpu->arch.apic_attention = 0;
2216 }
2217 
2218 /*
2219  *----------------------------------------------------------------------
2220  * timer interface
2221  *----------------------------------------------------------------------
2222  */
2223 
2224 static bool lapic_is_periodic(struct kvm_lapic *apic)
2225 {
2226 	return apic_lvtt_period(apic);
2227 }
2228 
2229 int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2230 {
2231 	struct kvm_lapic *apic = vcpu->arch.apic;
2232 
2233 	if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
2234 		return atomic_read(&apic->lapic_timer.pending);
2235 
2236 	return 0;
2237 }
2238 
2239 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
2240 {
2241 	u32 reg = kvm_lapic_get_reg(apic, lvt_type);
2242 	int vector, mode, trig_mode;
2243 
2244 	if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
2245 		vector = reg & APIC_VECTOR_MASK;
2246 		mode = reg & APIC_MODE_MASK;
2247 		trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
2248 		return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2249 					NULL);
2250 	}
2251 	return 0;
2252 }
2253 
2254 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
2255 {
2256 	struct kvm_lapic *apic = vcpu->arch.apic;
2257 
2258 	if (apic)
2259 		kvm_apic_local_deliver(apic, APIC_LVT0);
2260 }
2261 
2262 static const struct kvm_io_device_ops apic_mmio_ops = {
2263 	.read     = apic_mmio_read,
2264 	.write    = apic_mmio_write,
2265 };
2266 
2267 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2268 {
2269 	struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2270 	struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
2271 
2272 	apic_timer_expired(apic);
2273 
2274 	if (lapic_is_periodic(apic)) {
2275 		advance_periodic_target_expiration(apic);
2276 		hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2277 		return HRTIMER_RESTART;
2278 	} else
2279 		return HRTIMER_NORESTART;
2280 }
2281 
2282 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
2283 {
2284 	struct kvm_lapic *apic;
2285 
2286 	ASSERT(vcpu != NULL);
2287 
2288 	apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT);
2289 	if (!apic)
2290 		goto nomem;
2291 
2292 	vcpu->arch.apic = apic;
2293 
2294 	apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
2295 	if (!apic->regs) {
2296 		printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2297 		       vcpu->vcpu_id);
2298 		goto nomem_free_apic;
2299 	}
2300 	apic->vcpu = vcpu;
2301 
2302 	hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2303 		     HRTIMER_MODE_ABS_HARD);
2304 	apic->lapic_timer.timer.function = apic_timer_fn;
2305 	if (timer_advance_ns == -1) {
2306 		apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
2307 		lapic_timer_advance_dynamic = true;
2308 	} else {
2309 		apic->lapic_timer.timer_advance_ns = timer_advance_ns;
2310 		lapic_timer_advance_dynamic = false;
2311 	}
2312 
2313 	/*
2314 	 * APIC is created enabled. This will prevent kvm_lapic_set_base from
2315 	 * thinking that APIC state has changed.
2316 	 */
2317 	vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2318 	static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
2319 	kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
2320 
2321 	return 0;
2322 nomem_free_apic:
2323 	kfree(apic);
2324 	vcpu->arch.apic = NULL;
2325 nomem:
2326 	return -ENOMEM;
2327 }
2328 
2329 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2330 {
2331 	struct kvm_lapic *apic = vcpu->arch.apic;
2332 	u32 ppr;
2333 
2334 	if (!kvm_apic_hw_enabled(apic))
2335 		return -1;
2336 
2337 	__apic_update_ppr(apic, &ppr);
2338 	return apic_has_interrupt_for_ppr(apic, ppr);
2339 }
2340 
2341 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2342 {
2343 	u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
2344 	int r = 0;
2345 
2346 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
2347 		r = 1;
2348 	if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2349 	    GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2350 		r = 1;
2351 	return r;
2352 }
2353 
2354 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2355 {
2356 	struct kvm_lapic *apic = vcpu->arch.apic;
2357 
2358 	if (atomic_read(&apic->lapic_timer.pending) > 0) {
2359 		kvm_apic_inject_pending_timer_irqs(apic);
2360 		atomic_set(&apic->lapic_timer.pending, 0);
2361 	}
2362 }
2363 
2364 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2365 {
2366 	int vector = kvm_apic_has_interrupt(vcpu);
2367 	struct kvm_lapic *apic = vcpu->arch.apic;
2368 	u32 ppr;
2369 
2370 	if (vector == -1)
2371 		return -1;
2372 
2373 	/*
2374 	 * We get here even with APIC virtualization enabled, if doing
2375 	 * nested virtualization and L1 runs with the "acknowledge interrupt
2376 	 * on exit" mode.  Then we cannot inject the interrupt via RVI,
2377 	 * because the process would deliver it through the IDT.
2378 	 */
2379 
2380 	apic_clear_irr(vector, apic);
2381 	if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
2382 		/*
2383 		 * For auto-EOI interrupts, there might be another pending
2384 		 * interrupt above PPR, so check whether to raise another
2385 		 * KVM_REQ_EVENT.
2386 		 */
2387 		apic_update_ppr(apic);
2388 	} else {
2389 		/*
2390 		 * For normal interrupts, PPR has been raised and there cannot
2391 		 * be a higher-priority pending interrupt---except if there was
2392 		 * a concurrent interrupt injection, but that would have
2393 		 * triggered KVM_REQ_EVENT already.
2394 		 */
2395 		apic_set_isr(vector, apic);
2396 		__apic_update_ppr(apic, &ppr);
2397 	}
2398 
2399 	return vector;
2400 }
2401 
2402 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2403 		struct kvm_lapic_state *s, bool set)
2404 {
2405 	if (apic_x2apic_mode(vcpu->arch.apic)) {
2406 		u32 *id = (u32 *)(s->regs + APIC_ID);
2407 		u32 *ldr = (u32 *)(s->regs + APIC_LDR);
2408 
2409 		if (vcpu->kvm->arch.x2apic_format) {
2410 			if (*id != vcpu->vcpu_id)
2411 				return -EINVAL;
2412 		} else {
2413 			if (set)
2414 				*id >>= 24;
2415 			else
2416 				*id <<= 24;
2417 		}
2418 
2419 		/* In x2APIC mode, the LDR is fixed and based on the id */
2420 		if (set)
2421 			*ldr = kvm_apic_calc_x2apic_ldr(*id);
2422 	}
2423 
2424 	return 0;
2425 }
2426 
2427 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2428 {
2429 	memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2430 	return kvm_apic_state_fixup(vcpu, s, false);
2431 }
2432 
2433 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2434 {
2435 	struct kvm_lapic *apic = vcpu->arch.apic;
2436 	int r;
2437 
2438 
2439 	kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
2440 	/* set SPIV separately to get count of SW disabled APICs right */
2441 	apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
2442 
2443 	r = kvm_apic_state_fixup(vcpu, s, true);
2444 	if (r)
2445 		return r;
2446 	memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s));
2447 
2448 	recalculate_apic_map(vcpu->kvm);
2449 	kvm_apic_set_version(vcpu);
2450 
2451 	apic_update_ppr(apic);
2452 	hrtimer_cancel(&apic->lapic_timer.timer);
2453 	apic_update_lvtt(apic);
2454 	apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2455 	update_divide_count(apic);
2456 	start_apic_timer(apic);
2457 	apic->irr_pending = true;
2458 	apic->isr_count = vcpu->arch.apicv_active ?
2459 				1 : count_vectors(apic->regs + APIC_ISR);
2460 	apic->highest_isr_cache = -1;
2461 	if (vcpu->arch.apicv_active) {
2462 		kvm_x86_ops->apicv_post_state_restore(vcpu);
2463 		kvm_x86_ops->hwapic_irr_update(vcpu,
2464 				apic_find_highest_irr(apic));
2465 		kvm_x86_ops->hwapic_isr_update(vcpu,
2466 				apic_find_highest_isr(apic));
2467 	}
2468 	kvm_make_request(KVM_REQ_EVENT, vcpu);
2469 	if (ioapic_in_kernel(vcpu->kvm))
2470 		kvm_rtc_eoi_tracking_restore_one(vcpu);
2471 
2472 	vcpu->arch.apic_arb_prio = 0;
2473 
2474 	return 0;
2475 }
2476 
2477 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
2478 {
2479 	struct hrtimer *timer;
2480 
2481 	if (!lapic_in_kernel(vcpu) ||
2482 		kvm_can_post_timer_interrupt(vcpu))
2483 		return;
2484 
2485 	timer = &vcpu->arch.apic->lapic_timer.timer;
2486 	if (hrtimer_cancel(timer))
2487 		hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD);
2488 }
2489 
2490 /*
2491  * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2492  *
2493  * Detect whether guest triggered PV EOI since the
2494  * last entry. If yes, set EOI on guests's behalf.
2495  * Clear PV EOI in guest memory in any case.
2496  */
2497 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2498 					struct kvm_lapic *apic)
2499 {
2500 	bool pending;
2501 	int vector;
2502 	/*
2503 	 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2504 	 * and KVM_PV_EOI_ENABLED in guest memory as follows:
2505 	 *
2506 	 * KVM_APIC_PV_EOI_PENDING is unset:
2507 	 * 	-> host disabled PV EOI.
2508 	 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2509 	 * 	-> host enabled PV EOI, guest did not execute EOI yet.
2510 	 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2511 	 * 	-> host enabled PV EOI, guest executed EOI.
2512 	 */
2513 	BUG_ON(!pv_eoi_enabled(vcpu));
2514 	pending = pv_eoi_get_pending(vcpu);
2515 	/*
2516 	 * Clear pending bit in any case: it will be set again on vmentry.
2517 	 * While this might not be ideal from performance point of view,
2518 	 * this makes sure pv eoi is only enabled when we know it's safe.
2519 	 */
2520 	pv_eoi_clr_pending(vcpu);
2521 	if (pending)
2522 		return;
2523 	vector = apic_set_eoi(apic);
2524 	trace_kvm_pv_eoi(apic, vector);
2525 }
2526 
2527 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2528 {
2529 	u32 data;
2530 
2531 	if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2532 		apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2533 
2534 	if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2535 		return;
2536 
2537 	if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2538 				  sizeof(u32)))
2539 		return;
2540 
2541 	apic_set_tpr(vcpu->arch.apic, data & 0xff);
2542 }
2543 
2544 /*
2545  * apic_sync_pv_eoi_to_guest - called before vmentry
2546  *
2547  * Detect whether it's safe to enable PV EOI and
2548  * if yes do so.
2549  */
2550 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2551 					struct kvm_lapic *apic)
2552 {
2553 	if (!pv_eoi_enabled(vcpu) ||
2554 	    /* IRR set or many bits in ISR: could be nested. */
2555 	    apic->irr_pending ||
2556 	    /* Cache not set: could be safe but we don't bother. */
2557 	    apic->highest_isr_cache == -1 ||
2558 	    /* Need EOI to update ioapic. */
2559 	    kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
2560 		/*
2561 		 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2562 		 * so we need not do anything here.
2563 		 */
2564 		return;
2565 	}
2566 
2567 	pv_eoi_set_pending(apic->vcpu);
2568 }
2569 
2570 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2571 {
2572 	u32 data, tpr;
2573 	int max_irr, max_isr;
2574 	struct kvm_lapic *apic = vcpu->arch.apic;
2575 
2576 	apic_sync_pv_eoi_to_guest(vcpu, apic);
2577 
2578 	if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2579 		return;
2580 
2581 	tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
2582 	max_irr = apic_find_highest_irr(apic);
2583 	if (max_irr < 0)
2584 		max_irr = 0;
2585 	max_isr = apic_find_highest_isr(apic);
2586 	if (max_isr < 0)
2587 		max_isr = 0;
2588 	data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2589 
2590 	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2591 				sizeof(u32));
2592 }
2593 
2594 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
2595 {
2596 	if (vapic_addr) {
2597 		if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2598 					&vcpu->arch.apic->vapic_cache,
2599 					vapic_addr, sizeof(u32)))
2600 			return -EINVAL;
2601 		__set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2602 	} else {
2603 		__clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2604 	}
2605 
2606 	vcpu->arch.apic->vapic_addr = vapic_addr;
2607 	return 0;
2608 }
2609 
2610 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2611 {
2612 	struct kvm_lapic *apic = vcpu->arch.apic;
2613 	u32 reg = (msr - APIC_BASE_MSR) << 4;
2614 
2615 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2616 		return 1;
2617 
2618 	if (reg == APIC_ICR2)
2619 		return 1;
2620 
2621 	/* if this is ICR write vector before command */
2622 	if (reg == APIC_ICR)
2623 		kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2624 	return kvm_lapic_reg_write(apic, reg, (u32)data);
2625 }
2626 
2627 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2628 {
2629 	struct kvm_lapic *apic = vcpu->arch.apic;
2630 	u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2631 
2632 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2633 		return 1;
2634 
2635 	if (reg == APIC_DFR || reg == APIC_ICR2)
2636 		return 1;
2637 
2638 	if (kvm_lapic_reg_read(apic, reg, 4, &low))
2639 		return 1;
2640 	if (reg == APIC_ICR)
2641 		kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2642 
2643 	*data = (((u64)high) << 32) | low;
2644 
2645 	return 0;
2646 }
2647 
2648 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2649 {
2650 	struct kvm_lapic *apic = vcpu->arch.apic;
2651 
2652 	if (!lapic_in_kernel(vcpu))
2653 		return 1;
2654 
2655 	/* if this is ICR write vector before command */
2656 	if (reg == APIC_ICR)
2657 		kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2658 	return kvm_lapic_reg_write(apic, reg, (u32)data);
2659 }
2660 
2661 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2662 {
2663 	struct kvm_lapic *apic = vcpu->arch.apic;
2664 	u32 low, high = 0;
2665 
2666 	if (!lapic_in_kernel(vcpu))
2667 		return 1;
2668 
2669 	if (kvm_lapic_reg_read(apic, reg, 4, &low))
2670 		return 1;
2671 	if (reg == APIC_ICR)
2672 		kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2673 
2674 	*data = (((u64)high) << 32) | low;
2675 
2676 	return 0;
2677 }
2678 
2679 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len)
2680 {
2681 	u64 addr = data & ~KVM_MSR_ENABLED;
2682 	struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data;
2683 	unsigned long new_len;
2684 
2685 	if (!IS_ALIGNED(addr, 4))
2686 		return 1;
2687 
2688 	vcpu->arch.pv_eoi.msr_val = data;
2689 	if (!pv_eoi_enabled(vcpu))
2690 		return 0;
2691 
2692 	if (addr == ghc->gpa && len <= ghc->len)
2693 		new_len = ghc->len;
2694 	else
2695 		new_len = len;
2696 
2697 	return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
2698 }
2699 
2700 void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2701 {
2702 	struct kvm_lapic *apic = vcpu->arch.apic;
2703 	u8 sipi_vector;
2704 	unsigned long pe;
2705 
2706 	if (!lapic_in_kernel(vcpu) || !apic->pending_events)
2707 		return;
2708 
2709 	/*
2710 	 * INITs are latched while CPU is in specific states
2711 	 * (SMM, VMX non-root mode, SVM with GIF=0).
2712 	 * Because a CPU cannot be in these states immediately
2713 	 * after it has processed an INIT signal (and thus in
2714 	 * KVM_MP_STATE_INIT_RECEIVED state), just eat SIPIs
2715 	 * and leave the INIT pending.
2716 	 */
2717 	if (is_smm(vcpu) || kvm_x86_ops->apic_init_signal_blocked(vcpu)) {
2718 		WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2719 		if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2720 			clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2721 		return;
2722 	}
2723 
2724 	pe = xchg(&apic->pending_events, 0);
2725 	if (test_bit(KVM_APIC_INIT, &pe)) {
2726 		kvm_vcpu_reset(vcpu, true);
2727 		if (kvm_vcpu_is_bsp(apic->vcpu))
2728 			vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2729 		else
2730 			vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2731 	}
2732 	if (test_bit(KVM_APIC_SIPI, &pe) &&
2733 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2734 		/* evaluate pending_events before reading the vector */
2735 		smp_rmb();
2736 		sipi_vector = apic->sipi_vector;
2737 		kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2738 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2739 	}
2740 }
2741 
2742 void kvm_lapic_init(void)
2743 {
2744 	/* do not patch jump label more than once per second */
2745 	jump_label_rate_limit(&apic_hw_disabled, HZ);
2746 	jump_label_rate_limit(&apic_sw_disabled, HZ);
2747 }
2748 
2749 void kvm_lapic_exit(void)
2750 {
2751 	static_key_deferred_flush(&apic_hw_disabled);
2752 	static_key_deferred_flush(&apic_sw_disabled);
2753 }
2754