xref: /freebsd/sys/amd64/vmm/io/vlapic.c (revision becd9849004aff4ef7b940de459f3d331e90a3a9)
1366f6083SPeter Grehan /*-
2366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
3366f6083SPeter Grehan  * All rights reserved.
4366f6083SPeter Grehan  *
5366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
6366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
7366f6083SPeter Grehan  * are met:
8366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
9366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
10366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
11366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
12366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
13366f6083SPeter Grehan  *
14366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24366f6083SPeter Grehan  * SUCH DAMAGE.
25366f6083SPeter Grehan  *
26366f6083SPeter Grehan  * $FreeBSD$
27366f6083SPeter Grehan  */
28366f6083SPeter Grehan 
29366f6083SPeter Grehan #include <sys/cdefs.h>
30366f6083SPeter Grehan __FBSDID("$FreeBSD$");
31366f6083SPeter Grehan 
32366f6083SPeter Grehan #include <sys/param.h>
33fb03ca4eSNeel Natu #include <sys/lock.h>
34366f6083SPeter Grehan #include <sys/kernel.h>
35366f6083SPeter Grehan #include <sys/malloc.h>
36fb03ca4eSNeel Natu #include <sys/mutex.h>
37366f6083SPeter Grehan #include <sys/systm.h>
38a5615c90SPeter Grehan #include <sys/smp.h>
39366f6083SPeter Grehan 
40366f6083SPeter Grehan #include <machine/clock.h>
412d3a73edSNeel Natu #include <x86/specialreg.h>
4234a6b2d6SJohn Baldwin #include <x86/apicreg.h>
43366f6083SPeter Grehan 
44366f6083SPeter Grehan #include <machine/vmm.h>
45366f6083SPeter Grehan 
4677d8fd9bSNeel Natu #include "vmm_stat.h"
47366f6083SPeter Grehan #include "vmm_lapic.h"
48366f6083SPeter Grehan #include "vmm_ktr.h"
49366f6083SPeter Grehan #include "vlapic.h"
50b5b28fc9SNeel Natu #include "vioapic.h"
51366f6083SPeter Grehan 
52366f6083SPeter Grehan #define	VLAPIC_CTR0(vlapic, format)					\
53513c8d33SNeel Natu 	VCPU_CTR0((vlapic)->vm, (vlapic)->vcpuid, format)
54366f6083SPeter Grehan 
55366f6083SPeter Grehan #define	VLAPIC_CTR1(vlapic, format, p1)					\
56513c8d33SNeel Natu 	VCPU_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1)
57366f6083SPeter Grehan 
581c052192SNeel Natu #define	VLAPIC_CTR2(vlapic, format, p1, p2)				\
591c052192SNeel Natu 	VCPU_CTR2((vlapic)->vm, (vlapic)->vcpuid, format, p1, p2)
601c052192SNeel Natu 
61366f6083SPeter Grehan #define	VLAPIC_CTR_IRR(vlapic, msg)					\
62366f6083SPeter Grehan do {									\
63366f6083SPeter Grehan 	uint32_t *irrptr = &(vlapic)->apic.irr0;			\
64366f6083SPeter Grehan 	irrptr[0] = irrptr[0];	/* silence compiler */			\
65366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]);	\
66366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]);	\
67366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]);	\
68366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]);	\
69366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]);	\
70366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]);	\
71366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]);	\
72366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]);	\
73366f6083SPeter Grehan } while (0)
74366f6083SPeter Grehan 
75366f6083SPeter Grehan #define	VLAPIC_CTR_ISR(vlapic, msg)					\
76366f6083SPeter Grehan do {									\
77366f6083SPeter Grehan 	uint32_t *isrptr = &(vlapic)->apic.isr0;			\
78366f6083SPeter Grehan 	isrptr[0] = isrptr[0];	/* silence compiler */			\
79366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]);	\
80366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]);	\
81366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]);	\
82366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]);	\
83366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]);	\
84366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]);	\
85366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]);	\
86366f6083SPeter Grehan 	VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]);	\
87366f6083SPeter Grehan } while (0)
88366f6083SPeter Grehan 
89366f6083SPeter Grehan static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
90366f6083SPeter Grehan 
91366f6083SPeter Grehan #define	PRIO(x)			((x) >> 4)
92366f6083SPeter Grehan 
93366f6083SPeter Grehan #define VLAPIC_VERSION		(16)
94366f6083SPeter Grehan #define VLAPIC_MAXLVT_ENTRIES	(5)
95366f6083SPeter Grehan 
96a2da7af6SNeel Natu #define	x2apic(vlapic)	(((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
972d3a73edSNeel Natu 
98edf89256SNeel Natu enum boot_state {
99edf89256SNeel Natu 	BS_INIT,
100edf89256SNeel Natu 	BS_SIPI,
101edf89256SNeel Natu 	BS_RUNNING
102edf89256SNeel Natu };
103edf89256SNeel Natu 
104366f6083SPeter Grehan struct vlapic {
105366f6083SPeter Grehan 	struct vm		*vm;
106366f6083SPeter Grehan 	int			vcpuid;
107366f6083SPeter Grehan 
108366f6083SPeter Grehan 	struct LAPIC		apic;
109366f6083SPeter Grehan 
110366f6083SPeter Grehan 	int			 esr_update;
111366f6083SPeter Grehan 
112fb03ca4eSNeel Natu 	struct callout	callout;	/* vlapic timer */
113fb03ca4eSNeel Natu 	struct bintime	timer_fire_bt;	/* callout expiry time */
114fb03ca4eSNeel Natu 	struct bintime	timer_freq_bt;	/* timer frequency */
115fb03ca4eSNeel Natu 	struct bintime	timer_period_bt; /* timer period */
116fb03ca4eSNeel Natu 	struct mtx	timer_mtx;
117366f6083SPeter Grehan 
118366f6083SPeter Grehan 	/*
119366f6083SPeter Grehan 	 * The 'isrvec_stk' is a stack of vectors injected by the local apic.
120366f6083SPeter Grehan 	 * A vector is popped from the stack when the processor does an EOI.
121366f6083SPeter Grehan 	 * The vector on the top of the stack is used to compute the
122366f6083SPeter Grehan 	 * Processor Priority in conjunction with the TPR.
123366f6083SPeter Grehan 	 */
124366f6083SPeter Grehan 	uint8_t			 isrvec_stk[ISRVEC_STK_SIZE];
125366f6083SPeter Grehan 	int			 isrvec_stk_top;
1262d3a73edSNeel Natu 
1272d3a73edSNeel Natu 	uint64_t		msr_apicbase;
128edf89256SNeel Natu 	enum boot_state		boot_state;
129366f6083SPeter Grehan };
130366f6083SPeter Grehan 
131fb03ca4eSNeel Natu /*
132fb03ca4eSNeel Natu  * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
133fb03ca4eSNeel Natu  * vlapic_callout_handler() and vcpu accesses to the following registers:
134fb03ca4eSNeel Natu  * - initial count register aka icr_timer
135fb03ca4eSNeel Natu  * - current count register aka ccr_timer
136fb03ca4eSNeel Natu  * - divide config register aka dcr_timer
137fb03ca4eSNeel Natu  * - timer LVT register
138fb03ca4eSNeel Natu  *
139fb03ca4eSNeel Natu  * Note that the vlapic_callout_handler() does not write to any of these
140fb03ca4eSNeel Natu  * registers so they can be safely read from the vcpu context without locking.
141fb03ca4eSNeel Natu  */
142*becd9849SNeel Natu #define	VLAPIC_TIMER_LOCK(vlapic)	mtx_lock_spin(&((vlapic)->timer_mtx))
143*becd9849SNeel Natu #define	VLAPIC_TIMER_UNLOCK(vlapic)	mtx_unlock_spin(&((vlapic)->timer_mtx))
144fb03ca4eSNeel Natu #define	VLAPIC_TIMER_LOCKED(vlapic)	mtx_owned(&((vlapic)->timer_mtx))
145fb03ca4eSNeel Natu 
1462e25737aSNeel Natu #define VLAPIC_BUS_FREQ	tsc_freq
1472e25737aSNeel Natu 
1482e25737aSNeel Natu static int
1492e25737aSNeel Natu vlapic_timer_divisor(uint32_t dcr)
1502e25737aSNeel Natu {
1512e25737aSNeel Natu 	switch (dcr & 0xB) {
152117e8f37SPeter Grehan 	case APIC_TDCR_1:
153117e8f37SPeter Grehan 		return (1);
1542e25737aSNeel Natu 	case APIC_TDCR_2:
1552e25737aSNeel Natu 		return (2);
1562e25737aSNeel Natu 	case APIC_TDCR_4:
1572e25737aSNeel Natu 		return (4);
1582e25737aSNeel Natu 	case APIC_TDCR_8:
1592e25737aSNeel Natu 		return (8);
1602e25737aSNeel Natu 	case APIC_TDCR_16:
1612e25737aSNeel Natu 		return (16);
1622e25737aSNeel Natu 	case APIC_TDCR_32:
1632e25737aSNeel Natu 		return (32);
1642e25737aSNeel Natu 	case APIC_TDCR_64:
1652e25737aSNeel Natu 		return (64);
1662e25737aSNeel Natu 	case APIC_TDCR_128:
1672e25737aSNeel Natu 		return (128);
1682e25737aSNeel Natu 	default:
1692e25737aSNeel Natu 		panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
1702e25737aSNeel Natu 	}
1712e25737aSNeel Natu }
1722e25737aSNeel Natu 
173366f6083SPeter Grehan static void
174366f6083SPeter Grehan vlapic_mask_lvts(uint32_t *lvts, int num_lvt)
175366f6083SPeter Grehan {
176366f6083SPeter Grehan 	int i;
177366f6083SPeter Grehan 	for (i = 0; i < num_lvt; i++) {
178366f6083SPeter Grehan 		*lvts |= APIC_LVT_M;
179366f6083SPeter Grehan 		lvts += 4;
180366f6083SPeter Grehan 	}
181366f6083SPeter Grehan }
182366f6083SPeter Grehan 
183366f6083SPeter Grehan #if 0
184366f6083SPeter Grehan static inline void
185366f6083SPeter Grehan vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
186366f6083SPeter Grehan {
187366f6083SPeter Grehan 	printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
188366f6083SPeter Grehan 	    *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
189366f6083SPeter Grehan 	    *lvt & APIC_LVTT_M);
190366f6083SPeter Grehan }
191366f6083SPeter Grehan #endif
192366f6083SPeter Grehan 
193fb03ca4eSNeel Natu static uint32_t
194366f6083SPeter Grehan vlapic_get_ccr(struct vlapic *vlapic)
195366f6083SPeter Grehan {
196fb03ca4eSNeel Natu 	struct bintime bt_now, bt_rem;
197fb03ca4eSNeel Natu 	struct LAPIC *lapic;
198fb03ca4eSNeel Natu 	uint32_t ccr;
199fb03ca4eSNeel Natu 
200fb03ca4eSNeel Natu 	ccr = 0;
201fb03ca4eSNeel Natu 	lapic = &vlapic->apic;
202fb03ca4eSNeel Natu 
203fb03ca4eSNeel Natu 	VLAPIC_TIMER_LOCK(vlapic);
204fb03ca4eSNeel Natu 	if (callout_active(&vlapic->callout)) {
205fb03ca4eSNeel Natu 		/*
206fb03ca4eSNeel Natu 		 * If the timer is scheduled to expire in the future then
207fb03ca4eSNeel Natu 		 * compute the value of 'ccr' based on the remaining time.
208fb03ca4eSNeel Natu 		 */
209fb03ca4eSNeel Natu 		binuptime(&bt_now);
210fb03ca4eSNeel Natu 		if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
211fb03ca4eSNeel Natu 			bt_rem = vlapic->timer_fire_bt;
212fb03ca4eSNeel Natu 			bintime_sub(&bt_rem, &bt_now);
213fb03ca4eSNeel Natu 			ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
214fb03ca4eSNeel Natu 			ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
215fb03ca4eSNeel Natu 		}
216fb03ca4eSNeel Natu 	}
217fb03ca4eSNeel Natu 	KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
218fb03ca4eSNeel Natu 	    "icr_timer is %#x", ccr, lapic->icr_timer));
219fb03ca4eSNeel Natu 	VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
220fb03ca4eSNeel Natu 	    ccr, lapic->icr_timer);
221fb03ca4eSNeel Natu 	VLAPIC_TIMER_UNLOCK(vlapic);
222fb03ca4eSNeel Natu 	return (ccr);
223fb03ca4eSNeel Natu }
224fb03ca4eSNeel Natu 
225fb03ca4eSNeel Natu static void
226fb03ca4eSNeel Natu vlapic_set_dcr(struct vlapic *vlapic, uint32_t dcr)
227fb03ca4eSNeel Natu {
228fb03ca4eSNeel Natu 	struct LAPIC *lapic;
229fb03ca4eSNeel Natu 	int divisor;
230fb03ca4eSNeel Natu 
231fb03ca4eSNeel Natu 	lapic = &vlapic->apic;
232fb03ca4eSNeel Natu 	VLAPIC_TIMER_LOCK(vlapic);
233fb03ca4eSNeel Natu 
234fb03ca4eSNeel Natu 	lapic->dcr_timer = dcr;
235fb03ca4eSNeel Natu 	divisor = vlapic_timer_divisor(dcr);
236fb03ca4eSNeel Natu 	VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d", dcr, divisor);
237fb03ca4eSNeel Natu 
238fb03ca4eSNeel Natu 	/*
239fb03ca4eSNeel Natu 	 * Update the timer frequency and the timer period.
240fb03ca4eSNeel Natu 	 *
241fb03ca4eSNeel Natu 	 * XXX changes to the frequency divider will not take effect until
242fb03ca4eSNeel Natu 	 * the timer is reloaded.
243fb03ca4eSNeel Natu 	 */
244fb03ca4eSNeel Natu 	FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
245fb03ca4eSNeel Natu 	vlapic->timer_period_bt = vlapic->timer_freq_bt;
246fb03ca4eSNeel Natu 	bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
247fb03ca4eSNeel Natu 
248fb03ca4eSNeel Natu 	VLAPIC_TIMER_UNLOCK(vlapic);
249366f6083SPeter Grehan }
250366f6083SPeter Grehan 
251366f6083SPeter Grehan static void
252366f6083SPeter Grehan vlapic_update_errors(struct vlapic *vlapic)
253366f6083SPeter Grehan {
254366f6083SPeter Grehan 	struct LAPIC    *lapic = &vlapic->apic;
255366f6083SPeter Grehan 	lapic->esr = 0; // XXX
256366f6083SPeter Grehan }
257366f6083SPeter Grehan 
258366f6083SPeter Grehan static void
259366f6083SPeter Grehan vlapic_init_ipi(struct vlapic *vlapic)
260366f6083SPeter Grehan {
261366f6083SPeter Grehan 	struct LAPIC    *lapic = &vlapic->apic;
262366f6083SPeter Grehan 	lapic->version = VLAPIC_VERSION;
263366f6083SPeter Grehan 	lapic->version |= (VLAPIC_MAXLVT_ENTRIES < MAXLVTSHIFT);
264366f6083SPeter Grehan 	lapic->dfr = 0xffffffff;
265366f6083SPeter Grehan 	lapic->svr = APIC_SVR_VECTOR;
266366f6083SPeter Grehan 	vlapic_mask_lvts(&lapic->lvt_timer, VLAPIC_MAXLVT_ENTRIES+1);
267366f6083SPeter Grehan }
268366f6083SPeter Grehan 
269366f6083SPeter Grehan static int
27003cd0501SNeel Natu vlapic_reset(struct vlapic *vlapic)
271366f6083SPeter Grehan {
272366f6083SPeter Grehan 	struct LAPIC	*lapic = &vlapic->apic;
273366f6083SPeter Grehan 
274366f6083SPeter Grehan 	memset(lapic, 0, sizeof(*lapic));
275366f6083SPeter Grehan 	lapic->apr = vlapic->vcpuid;
276366f6083SPeter Grehan 	vlapic_init_ipi(vlapic);
277fb03ca4eSNeel Natu 	vlapic_set_dcr(vlapic, 0);
278366f6083SPeter Grehan 
279edf89256SNeel Natu 	if (vlapic->vcpuid == 0)
280edf89256SNeel Natu 		vlapic->boot_state = BS_RUNNING;	/* BSP */
281edf89256SNeel Natu 	else
282edf89256SNeel Natu 		vlapic->boot_state = BS_INIT;		/* AP */
283edf89256SNeel Natu 
284366f6083SPeter Grehan 	return 0;
285366f6083SPeter Grehan 
286366f6083SPeter Grehan }
287366f6083SPeter Grehan 
288366f6083SPeter Grehan void
289b5b28fc9SNeel Natu vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
290366f6083SPeter Grehan {
291366f6083SPeter Grehan 	struct LAPIC	*lapic = &vlapic->apic;
292b5b28fc9SNeel Natu 	uint32_t	*irrptr, *tmrptr, mask;
293366f6083SPeter Grehan 	int		idx;
294366f6083SPeter Grehan 
295366f6083SPeter Grehan 	if (vector < 0 || vector >= 256)
296366f6083SPeter Grehan 		panic("vlapic_set_intr_ready: invalid vector %d\n", vector);
297366f6083SPeter Grehan 
2981c052192SNeel Natu 	if (!(lapic->svr & APIC_SVR_ENABLE)) {
2991c052192SNeel Natu 		VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
3001c052192SNeel Natu 		    "interrupt %d", vector);
3011c052192SNeel Natu 		return;
3021c052192SNeel Natu 	}
3031c052192SNeel Natu 
304366f6083SPeter Grehan 	idx = (vector / 32) * 4;
305b5b28fc9SNeel Natu 	mask = 1 << (vector % 32);
306b5b28fc9SNeel Natu 
307366f6083SPeter Grehan 	irrptr = &lapic->irr0;
308b5b28fc9SNeel Natu 	atomic_set_int(&irrptr[idx], mask);
309b5b28fc9SNeel Natu 
310b5b28fc9SNeel Natu 	/*
311b5b28fc9SNeel Natu 	 * Upon acceptance of an interrupt into the IRR the corresponding
312b5b28fc9SNeel Natu 	 * TMR bit is cleared for edge-triggered interrupts and set for
313b5b28fc9SNeel Natu 	 * level-triggered interrupts.
314b5b28fc9SNeel Natu 	 */
315b5b28fc9SNeel Natu 	tmrptr = &lapic->tmr0;
316b5b28fc9SNeel Natu 	if (level)
317b5b28fc9SNeel Natu 		atomic_set_int(&tmrptr[idx], mask);
318b5b28fc9SNeel Natu 	else
319b5b28fc9SNeel Natu 		atomic_clear_int(&tmrptr[idx], mask);
320b5b28fc9SNeel Natu 
321366f6083SPeter Grehan 	VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
322366f6083SPeter Grehan }
323366f6083SPeter Grehan 
324366f6083SPeter Grehan static __inline uint32_t *
325fb03ca4eSNeel Natu vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
326366f6083SPeter Grehan {
327366f6083SPeter Grehan 	struct LAPIC	*lapic = &vlapic->apic;
328366f6083SPeter Grehan 	int 		 i;
329366f6083SPeter Grehan 
330366f6083SPeter Grehan 	if (offset < APIC_OFFSET_TIMER_LVT || offset > APIC_OFFSET_ERROR_LVT) {
331366f6083SPeter Grehan 		panic("vlapic_get_lvt: invalid LVT\n");
332366f6083SPeter Grehan 	}
333366f6083SPeter Grehan 	i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
334366f6083SPeter Grehan 	return ((&lapic->lvt_timer) + i);;
335366f6083SPeter Grehan }
336366f6083SPeter Grehan 
337fb03ca4eSNeel Natu static __inline uint32_t
338fb03ca4eSNeel Natu vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
339fb03ca4eSNeel Natu {
340fb03ca4eSNeel Natu 
341fb03ca4eSNeel Natu 	return (*vlapic_get_lvtptr(vlapic, offset));
342fb03ca4eSNeel Natu }
343fb03ca4eSNeel Natu 
344fb03ca4eSNeel Natu static void
345fb03ca4eSNeel Natu vlapic_set_lvt(struct vlapic *vlapic, uint32_t offset, uint32_t val)
346fb03ca4eSNeel Natu {
347fb03ca4eSNeel Natu 	uint32_t *lvtptr;
348fb03ca4eSNeel Natu 	struct LAPIC *lapic;
349fb03ca4eSNeel Natu 
350fb03ca4eSNeel Natu 	lapic = &vlapic->apic;
351fb03ca4eSNeel Natu 	lvtptr = vlapic_get_lvtptr(vlapic, offset);
352fb03ca4eSNeel Natu 
353fb03ca4eSNeel Natu 	if (offset == APIC_OFFSET_TIMER_LVT)
354fb03ca4eSNeel Natu 		VLAPIC_TIMER_LOCK(vlapic);
355fb03ca4eSNeel Natu 
356fb03ca4eSNeel Natu 	if (!(lapic->svr & APIC_SVR_ENABLE))
357fb03ca4eSNeel Natu 		val |= APIC_LVT_M;
358fb03ca4eSNeel Natu 	*lvtptr = val;
359fb03ca4eSNeel Natu 
360fb03ca4eSNeel Natu 	if (offset == APIC_OFFSET_TIMER_LVT)
361fb03ca4eSNeel Natu 		VLAPIC_TIMER_UNLOCK(vlapic);
362fb03ca4eSNeel Natu }
363fb03ca4eSNeel Natu 
364366f6083SPeter Grehan #if 1
365366f6083SPeter Grehan static void
366366f6083SPeter Grehan dump_isrvec_stk(struct vlapic *vlapic)
367366f6083SPeter Grehan {
368366f6083SPeter Grehan 	int i;
369366f6083SPeter Grehan 	uint32_t *isrptr;
370366f6083SPeter Grehan 
371366f6083SPeter Grehan 	isrptr = &vlapic->apic.isr0;
372366f6083SPeter Grehan 	for (i = 0; i < 8; i++)
373366f6083SPeter Grehan 		printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
374366f6083SPeter Grehan 
375366f6083SPeter Grehan 	for (i = 0; i <= vlapic->isrvec_stk_top; i++)
376366f6083SPeter Grehan 		printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
377366f6083SPeter Grehan }
378366f6083SPeter Grehan #endif
379366f6083SPeter Grehan 
380366f6083SPeter Grehan /*
381366f6083SPeter Grehan  * Algorithm adopted from section "Interrupt, Task and Processor Priority"
382366f6083SPeter Grehan  * in Intel Architecture Manual Vol 3a.
383366f6083SPeter Grehan  */
384366f6083SPeter Grehan static void
385366f6083SPeter Grehan vlapic_update_ppr(struct vlapic *vlapic)
386366f6083SPeter Grehan {
387366f6083SPeter Grehan 	int isrvec, tpr, ppr;
388366f6083SPeter Grehan 
389366f6083SPeter Grehan 	/*
390366f6083SPeter Grehan 	 * Note that the value on the stack at index 0 is always 0.
391366f6083SPeter Grehan 	 *
392366f6083SPeter Grehan 	 * This is a placeholder for the value of ISRV when none of the
393366f6083SPeter Grehan 	 * bits is set in the ISRx registers.
394366f6083SPeter Grehan 	 */
395366f6083SPeter Grehan 	isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
396366f6083SPeter Grehan 	tpr = vlapic->apic.tpr;
397366f6083SPeter Grehan 
398366f6083SPeter Grehan #if 1
399366f6083SPeter Grehan 	{
400366f6083SPeter Grehan 		int i, lastprio, curprio, vector, idx;
401366f6083SPeter Grehan 		uint32_t *isrptr;
402366f6083SPeter Grehan 
403366f6083SPeter Grehan 		if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
404366f6083SPeter Grehan 			panic("isrvec_stk is corrupted: %d", isrvec);
405366f6083SPeter Grehan 
406366f6083SPeter Grehan 		/*
407366f6083SPeter Grehan 		 * Make sure that the priority of the nested interrupts is
408366f6083SPeter Grehan 		 * always increasing.
409366f6083SPeter Grehan 		 */
410366f6083SPeter Grehan 		lastprio = -1;
411366f6083SPeter Grehan 		for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
412366f6083SPeter Grehan 			curprio = PRIO(vlapic->isrvec_stk[i]);
413366f6083SPeter Grehan 			if (curprio <= lastprio) {
414366f6083SPeter Grehan 				dump_isrvec_stk(vlapic);
415366f6083SPeter Grehan 				panic("isrvec_stk does not satisfy invariant");
416366f6083SPeter Grehan 			}
417366f6083SPeter Grehan 			lastprio = curprio;
418366f6083SPeter Grehan 		}
419366f6083SPeter Grehan 
420366f6083SPeter Grehan 		/*
421366f6083SPeter Grehan 		 * Make sure that each bit set in the ISRx registers has a
422366f6083SPeter Grehan 		 * corresponding entry on the isrvec stack.
423366f6083SPeter Grehan 		 */
424366f6083SPeter Grehan 		i = 1;
425366f6083SPeter Grehan 		isrptr = &vlapic->apic.isr0;
426366f6083SPeter Grehan 		for (vector = 0; vector < 256; vector++) {
427366f6083SPeter Grehan 			idx = (vector / 32) * 4;
428366f6083SPeter Grehan 			if (isrptr[idx] & (1 << (vector % 32))) {
429366f6083SPeter Grehan 				if (i > vlapic->isrvec_stk_top ||
430366f6083SPeter Grehan 				    vlapic->isrvec_stk[i] != vector) {
431366f6083SPeter Grehan 					dump_isrvec_stk(vlapic);
432366f6083SPeter Grehan 					panic("ISR and isrvec_stk out of sync");
433366f6083SPeter Grehan 				}
434366f6083SPeter Grehan 				i++;
435366f6083SPeter Grehan 			}
436366f6083SPeter Grehan 		}
437366f6083SPeter Grehan 	}
438366f6083SPeter Grehan #endif
439366f6083SPeter Grehan 
440366f6083SPeter Grehan 	if (PRIO(tpr) >= PRIO(isrvec))
441366f6083SPeter Grehan 		ppr = tpr;
442366f6083SPeter Grehan 	else
443366f6083SPeter Grehan 		ppr = isrvec & 0xf0;
444366f6083SPeter Grehan 
445366f6083SPeter Grehan 	vlapic->apic.ppr = ppr;
446366f6083SPeter Grehan 	VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
447366f6083SPeter Grehan }
448366f6083SPeter Grehan 
449366f6083SPeter Grehan static void
450366f6083SPeter Grehan vlapic_process_eoi(struct vlapic *vlapic)
451366f6083SPeter Grehan {
452366f6083SPeter Grehan 	struct LAPIC	*lapic = &vlapic->apic;
453b5b28fc9SNeel Natu 	uint32_t	*isrptr, *tmrptr;
454b5b28fc9SNeel Natu 	int		i, idx, bitpos, vector;
455366f6083SPeter Grehan 
456366f6083SPeter Grehan 	isrptr = &lapic->isr0;
457b5b28fc9SNeel Natu 	tmrptr = &lapic->tmr0;
458366f6083SPeter Grehan 
459366f6083SPeter Grehan 	/*
460366f6083SPeter Grehan 	 * The x86 architecture reserves the the first 32 vectors for use
461366f6083SPeter Grehan 	 * by the processor.
462366f6083SPeter Grehan 	 */
463366f6083SPeter Grehan 	for (i = 7; i > 0; i--) {
464366f6083SPeter Grehan 		idx = i * 4;
465366f6083SPeter Grehan 		bitpos = fls(isrptr[idx]);
466b5b28fc9SNeel Natu 		if (bitpos-- != 0) {
467366f6083SPeter Grehan 			if (vlapic->isrvec_stk_top <= 0) {
468366f6083SPeter Grehan 				panic("invalid vlapic isrvec_stk_top %d",
469366f6083SPeter Grehan 				      vlapic->isrvec_stk_top);
470366f6083SPeter Grehan 			}
471b5b28fc9SNeel Natu 			isrptr[idx] &= ~(1 << bitpos);
472366f6083SPeter Grehan 			VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
473366f6083SPeter Grehan 			vlapic->isrvec_stk_top--;
474366f6083SPeter Grehan 			vlapic_update_ppr(vlapic);
475b5b28fc9SNeel Natu 			if ((tmrptr[idx] & (1 << bitpos)) != 0) {
476b5b28fc9SNeel Natu 				vector = i * 32 + bitpos;
477b5b28fc9SNeel Natu 				vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
478b5b28fc9SNeel Natu 				    vector);
479b5b28fc9SNeel Natu 			}
480366f6083SPeter Grehan 			return;
481366f6083SPeter Grehan 		}
482366f6083SPeter Grehan 	}
483366f6083SPeter Grehan }
484366f6083SPeter Grehan 
485366f6083SPeter Grehan static __inline int
486fb03ca4eSNeel Natu vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
487366f6083SPeter Grehan {
488fb03ca4eSNeel Natu 
489fb03ca4eSNeel Natu 	return (lvt & mask);
490366f6083SPeter Grehan }
491366f6083SPeter Grehan 
492366f6083SPeter Grehan static __inline int
493366f6083SPeter Grehan vlapic_periodic_timer(struct vlapic *vlapic)
494366f6083SPeter Grehan {
495fb03ca4eSNeel Natu 	uint32_t lvt;
496366f6083SPeter Grehan 
497366f6083SPeter Grehan 	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
498366f6083SPeter Grehan 
499366f6083SPeter Grehan 	return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
500366f6083SPeter Grehan }
501366f6083SPeter Grehan 
50277d8fd9bSNeel Natu static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
50377d8fd9bSNeel Natu 
504366f6083SPeter Grehan static void
505366f6083SPeter Grehan vlapic_fire_timer(struct vlapic *vlapic)
506366f6083SPeter Grehan {
507366f6083SPeter Grehan 	int vector;
508fb03ca4eSNeel Natu 	uint32_t lvt;
509fb03ca4eSNeel Natu 
510fb03ca4eSNeel Natu 	KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
511366f6083SPeter Grehan 
512366f6083SPeter Grehan 	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
513366f6083SPeter Grehan 
514366f6083SPeter Grehan 	if (!vlapic_get_lvt_field(lvt, APIC_LVTT_M)) {
51577d8fd9bSNeel Natu 		vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
516366f6083SPeter Grehan 		vector = vlapic_get_lvt_field(lvt, APIC_LVTT_VECTOR);
517b5b28fc9SNeel Natu 		vlapic_set_intr_ready(vlapic, vector, false);
518fb03ca4eSNeel Natu 		vcpu_notify_event(vlapic->vm, vlapic->vcpuid);
519366f6083SPeter Grehan 	}
520366f6083SPeter Grehan }
521366f6083SPeter Grehan 
522fb03ca4eSNeel Natu static void
523fb03ca4eSNeel Natu vlapic_callout_handler(void *arg)
524fb03ca4eSNeel Natu {
525fb03ca4eSNeel Natu 	struct vlapic *vlapic;
526fb03ca4eSNeel Natu 	struct bintime bt, btnow;
527fb03ca4eSNeel Natu 	sbintime_t rem_sbt;
528fb03ca4eSNeel Natu 
529fb03ca4eSNeel Natu 	vlapic = arg;
530fb03ca4eSNeel Natu 
531fb03ca4eSNeel Natu 	VLAPIC_TIMER_LOCK(vlapic);
532fb03ca4eSNeel Natu 	if (callout_pending(&vlapic->callout))	/* callout was reset */
533fb03ca4eSNeel Natu 		goto done;
534fb03ca4eSNeel Natu 
535fb03ca4eSNeel Natu 	if (!callout_active(&vlapic->callout))	/* callout was stopped */
536fb03ca4eSNeel Natu 		goto done;
537fb03ca4eSNeel Natu 
538fb03ca4eSNeel Natu 	callout_deactivate(&vlapic->callout);
539fb03ca4eSNeel Natu 
540fb03ca4eSNeel Natu 	KASSERT(vlapic->apic.icr_timer != 0, ("vlapic timer is disabled"));
541fb03ca4eSNeel Natu 
542fb03ca4eSNeel Natu 	vlapic_fire_timer(vlapic);
543fb03ca4eSNeel Natu 
544fb03ca4eSNeel Natu 	if (vlapic_periodic_timer(vlapic)) {
545fb03ca4eSNeel Natu 		binuptime(&btnow);
546fb03ca4eSNeel Natu 		KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
547fb03ca4eSNeel Natu 		    ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
548fb03ca4eSNeel Natu 		    btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
549fb03ca4eSNeel Natu 		    vlapic->timer_fire_bt.frac));
550fb03ca4eSNeel Natu 
551fb03ca4eSNeel Natu 		/*
552fb03ca4eSNeel Natu 		 * Compute the delta between when the timer was supposed to
553fb03ca4eSNeel Natu 		 * fire and the present time.
554fb03ca4eSNeel Natu 		 */
555fb03ca4eSNeel Natu 		bt = btnow;
556fb03ca4eSNeel Natu 		bintime_sub(&bt, &vlapic->timer_fire_bt);
557fb03ca4eSNeel Natu 
558fb03ca4eSNeel Natu 		rem_sbt = bttosbt(vlapic->timer_period_bt);
559fb03ca4eSNeel Natu 		if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
560fb03ca4eSNeel Natu 			/*
561fb03ca4eSNeel Natu 			 * Adjust the time until the next countdown downward
562fb03ca4eSNeel Natu 			 * to account for the lost time.
563fb03ca4eSNeel Natu 			 */
564fb03ca4eSNeel Natu 			rem_sbt -= bttosbt(bt);
565fb03ca4eSNeel Natu 		} else {
566fb03ca4eSNeel Natu 			/*
567fb03ca4eSNeel Natu 			 * If the delta is greater than the timer period then
568fb03ca4eSNeel Natu 			 * just reset our time base instead of trying to catch
569fb03ca4eSNeel Natu 			 * up.
570fb03ca4eSNeel Natu 			 */
571fb03ca4eSNeel Natu 			vlapic->timer_fire_bt = btnow;
572fb03ca4eSNeel Natu 			VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
573fb03ca4eSNeel Natu 			    "usecs, period is %lu usecs - resetting time base",
574fb03ca4eSNeel Natu 			    bttosbt(bt) / SBT_1US,
575fb03ca4eSNeel Natu 			    bttosbt(vlapic->timer_period_bt) / SBT_1US);
576fb03ca4eSNeel Natu 		}
577fb03ca4eSNeel Natu 
578fb03ca4eSNeel Natu 		bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
579fb03ca4eSNeel Natu 		callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
580fb03ca4eSNeel Natu 		    vlapic_callout_handler, vlapic, 0);
581fb03ca4eSNeel Natu 	}
582fb03ca4eSNeel Natu done:
583fb03ca4eSNeel Natu 	VLAPIC_TIMER_UNLOCK(vlapic);
584fb03ca4eSNeel Natu }
585fb03ca4eSNeel Natu 
586fb03ca4eSNeel Natu static void
587fb03ca4eSNeel Natu vlapic_set_icr_timer(struct vlapic *vlapic, uint32_t icr_timer)
588fb03ca4eSNeel Natu {
589fb03ca4eSNeel Natu 	struct LAPIC *lapic;
590fb03ca4eSNeel Natu 	sbintime_t sbt;
591fb03ca4eSNeel Natu 
592fb03ca4eSNeel Natu 	VLAPIC_TIMER_LOCK(vlapic);
593fb03ca4eSNeel Natu 
594fb03ca4eSNeel Natu 	lapic = &vlapic->apic;
595fb03ca4eSNeel Natu 	lapic->icr_timer = icr_timer;
596fb03ca4eSNeel Natu 
597fb03ca4eSNeel Natu 	vlapic->timer_period_bt = vlapic->timer_freq_bt;
598fb03ca4eSNeel Natu 	bintime_mul(&vlapic->timer_period_bt, icr_timer);
599fb03ca4eSNeel Natu 
600fb03ca4eSNeel Natu 	if (icr_timer != 0) {
601fb03ca4eSNeel Natu 		binuptime(&vlapic->timer_fire_bt);
602fb03ca4eSNeel Natu 		bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
603fb03ca4eSNeel Natu 
604fb03ca4eSNeel Natu 		sbt = bttosbt(vlapic->timer_period_bt);
605fb03ca4eSNeel Natu 		callout_reset_sbt(&vlapic->callout, sbt, 0,
606fb03ca4eSNeel Natu 		    vlapic_callout_handler, vlapic, 0);
607fb03ca4eSNeel Natu 	} else
608fb03ca4eSNeel Natu 		callout_stop(&vlapic->callout);
609fb03ca4eSNeel Natu 
610fb03ca4eSNeel Natu 	VLAPIC_TIMER_UNLOCK(vlapic);
611fb03ca4eSNeel Natu }
612fb03ca4eSNeel Natu 
6130acb0d84SNeel Natu static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
6140acb0d84SNeel Natu 
615366f6083SPeter Grehan static int
616*becd9849SNeel Natu lapic_process_icr(struct vlapic *vlapic, uint64_t icrval, bool *retu)
617366f6083SPeter Grehan {
618366f6083SPeter Grehan 	int i;
619a5615c90SPeter Grehan 	cpuset_t dmask;
620366f6083SPeter Grehan 	uint32_t dest, vec, mode;
621edf89256SNeel Natu 	struct vlapic *vlapic2;
622edf89256SNeel Natu 	struct vm_exit *vmexit;
623366f6083SPeter Grehan 
624a2da7af6SNeel Natu 	if (x2apic(vlapic))
625366f6083SPeter Grehan 		dest = icrval >> 32;
626a2da7af6SNeel Natu 	else
627a2da7af6SNeel Natu 		dest = icrval >> (32 + 24);
628366f6083SPeter Grehan 	vec = icrval & APIC_VECTOR_MASK;
629366f6083SPeter Grehan 	mode = icrval & APIC_DELMODE_MASK;
630366f6083SPeter Grehan 
631366f6083SPeter Grehan 	if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
632366f6083SPeter Grehan 		switch (icrval & APIC_DEST_MASK) {
633366f6083SPeter Grehan 		case APIC_DEST_DESTFLD:
634a5615c90SPeter Grehan 			CPU_SETOF(dest, &dmask);
635366f6083SPeter Grehan 			break;
636366f6083SPeter Grehan 		case APIC_DEST_SELF:
637a5615c90SPeter Grehan 			CPU_SETOF(vlapic->vcpuid, &dmask);
638366f6083SPeter Grehan 			break;
639366f6083SPeter Grehan 		case APIC_DEST_ALLISELF:
640366f6083SPeter Grehan 			dmask = vm_active_cpus(vlapic->vm);
641366f6083SPeter Grehan 			break;
642366f6083SPeter Grehan 		case APIC_DEST_ALLESELF:
643a5615c90SPeter Grehan 			dmask = vm_active_cpus(vlapic->vm);
644a5615c90SPeter Grehan 			CPU_CLR(vlapic->vcpuid, &dmask);
645366f6083SPeter Grehan 			break;
6461e2751ddSSergey Kandaurov 		default:
6471e2751ddSSergey Kandaurov 			CPU_ZERO(&dmask);	/* satisfy gcc */
6481e2751ddSSergey Kandaurov 			break;
649366f6083SPeter Grehan 		}
650366f6083SPeter Grehan 
65182f2974aSSergey Kandaurov 		while ((i = CPU_FFS(&dmask)) != 0) {
652a5615c90SPeter Grehan 			i--;
653a5615c90SPeter Grehan 			CPU_CLR(i, &dmask);
6540acb0d84SNeel Natu 			if (mode == APIC_DELMODE_FIXED) {
655b5b28fc9SNeel Natu 				lapic_intr_edge(vlapic->vm, i, vec);
6560acb0d84SNeel Natu 				vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
6570acb0d84SNeel Natu 						    IPIS_SENT, i, 1);
6580acb0d84SNeel Natu 			} else
659366f6083SPeter Grehan 				vm_inject_nmi(vlapic->vm, i);
660366f6083SPeter Grehan 		}
661366f6083SPeter Grehan 
662366f6083SPeter Grehan 		return (0);	/* handled completely in the kernel */
663366f6083SPeter Grehan 	}
664366f6083SPeter Grehan 
665edf89256SNeel Natu 	if (mode == APIC_DELMODE_INIT) {
666edf89256SNeel Natu 		if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
667edf89256SNeel Natu 			return (0);
668edf89256SNeel Natu 
669edf89256SNeel Natu 		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
670edf89256SNeel Natu 			vlapic2 = vm_lapic(vlapic->vm, dest);
671edf89256SNeel Natu 
672edf89256SNeel Natu 			/* move from INIT to waiting-for-SIPI state */
673edf89256SNeel Natu 			if (vlapic2->boot_state == BS_INIT) {
674edf89256SNeel Natu 				vlapic2->boot_state = BS_SIPI;
675edf89256SNeel Natu 			}
676edf89256SNeel Natu 
677edf89256SNeel Natu 			return (0);
678edf89256SNeel Natu 		}
679edf89256SNeel Natu 	}
680edf89256SNeel Natu 
681edf89256SNeel Natu 	if (mode == APIC_DELMODE_STARTUP) {
682edf89256SNeel Natu 		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
683edf89256SNeel Natu 			vlapic2 = vm_lapic(vlapic->vm, dest);
684edf89256SNeel Natu 
685edf89256SNeel Natu 			/*
686edf89256SNeel Natu 			 * Ignore SIPIs in any state other than wait-for-SIPI
687edf89256SNeel Natu 			 */
688edf89256SNeel Natu 			if (vlapic2->boot_state != BS_SIPI)
689edf89256SNeel Natu 				return (0);
690edf89256SNeel Natu 
691366f6083SPeter Grehan 			/*
692366f6083SPeter Grehan 			 * XXX this assumes that the startup IPI always succeeds
693366f6083SPeter Grehan 			 */
694edf89256SNeel Natu 			vlapic2->boot_state = BS_RUNNING;
695edf89256SNeel Natu 			vm_activate_cpu(vlapic2->vm, dest);
696edf89256SNeel Natu 
697*becd9849SNeel Natu 			*retu = true;
698*becd9849SNeel Natu 			vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
699*becd9849SNeel Natu 			vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
700*becd9849SNeel Natu 			vmexit->u.spinup_ap.vcpu = dest;
701*becd9849SNeel Natu 			vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
702*becd9849SNeel Natu 
703edf89256SNeel Natu 			return (0);
704edf89256SNeel Natu 		}
705edf89256SNeel Natu 	}
706366f6083SPeter Grehan 
707366f6083SPeter Grehan 	/*
708366f6083SPeter Grehan 	 * This will cause a return to userland.
709366f6083SPeter Grehan 	 */
710366f6083SPeter Grehan 	return (1);
711366f6083SPeter Grehan }
712366f6083SPeter Grehan 
713366f6083SPeter Grehan int
714366f6083SPeter Grehan vlapic_pending_intr(struct vlapic *vlapic)
715366f6083SPeter Grehan {
716366f6083SPeter Grehan 	struct LAPIC	*lapic = &vlapic->apic;
717366f6083SPeter Grehan 	int	  	 idx, i, bitpos, vector;
718366f6083SPeter Grehan 	uint32_t	*irrptr, val;
719366f6083SPeter Grehan 
720366f6083SPeter Grehan 	irrptr = &lapic->irr0;
721366f6083SPeter Grehan 
722366f6083SPeter Grehan 	/*
723366f6083SPeter Grehan 	 * The x86 architecture reserves the the first 32 vectors for use
724366f6083SPeter Grehan 	 * by the processor.
725366f6083SPeter Grehan 	 */
726366f6083SPeter Grehan 	for (i = 7; i > 0; i--) {
727366f6083SPeter Grehan 		idx = i * 4;
728366f6083SPeter Grehan 		val = atomic_load_acq_int(&irrptr[idx]);
729366f6083SPeter Grehan 		bitpos = fls(val);
730366f6083SPeter Grehan 		if (bitpos != 0) {
731366f6083SPeter Grehan 			vector = i * 32 + (bitpos - 1);
732366f6083SPeter Grehan 			if (PRIO(vector) > PRIO(lapic->ppr)) {
733366f6083SPeter Grehan 				VLAPIC_CTR1(vlapic, "pending intr %d", vector);
734366f6083SPeter Grehan 				return (vector);
735366f6083SPeter Grehan 			} else
736366f6083SPeter Grehan 				break;
737366f6083SPeter Grehan 		}
738366f6083SPeter Grehan 	}
739366f6083SPeter Grehan 	return (-1);
740366f6083SPeter Grehan }
741366f6083SPeter Grehan 
742366f6083SPeter Grehan void
743366f6083SPeter Grehan vlapic_intr_accepted(struct vlapic *vlapic, int vector)
744366f6083SPeter Grehan {
745366f6083SPeter Grehan 	struct LAPIC	*lapic = &vlapic->apic;
746366f6083SPeter Grehan 	uint32_t	*irrptr, *isrptr;
747366f6083SPeter Grehan 	int		idx, stk_top;
748366f6083SPeter Grehan 
749366f6083SPeter Grehan 	/*
750366f6083SPeter Grehan 	 * clear the ready bit for vector being accepted in irr
751366f6083SPeter Grehan 	 * and set the vector as in service in isr.
752366f6083SPeter Grehan 	 */
753366f6083SPeter Grehan 	idx = (vector / 32) * 4;
754366f6083SPeter Grehan 
755366f6083SPeter Grehan 	irrptr = &lapic->irr0;
756366f6083SPeter Grehan 	atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
757366f6083SPeter Grehan 	VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
758366f6083SPeter Grehan 
759366f6083SPeter Grehan 	isrptr = &lapic->isr0;
760366f6083SPeter Grehan 	isrptr[idx] |= 1 << (vector % 32);
761366f6083SPeter Grehan 	VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
762366f6083SPeter Grehan 
763366f6083SPeter Grehan 	/*
764366f6083SPeter Grehan 	 * Update the PPR
765366f6083SPeter Grehan 	 */
766366f6083SPeter Grehan 	vlapic->isrvec_stk_top++;
767366f6083SPeter Grehan 
768366f6083SPeter Grehan 	stk_top = vlapic->isrvec_stk_top;
769366f6083SPeter Grehan 	if (stk_top >= ISRVEC_STK_SIZE)
770366f6083SPeter Grehan 		panic("isrvec_stk_top overflow %d", stk_top);
771366f6083SPeter Grehan 
772366f6083SPeter Grehan 	vlapic->isrvec_stk[stk_top] = vector;
773366f6083SPeter Grehan 	vlapic_update_ppr(vlapic);
774366f6083SPeter Grehan }
775366f6083SPeter Grehan 
7761c052192SNeel Natu static void
7771c052192SNeel Natu lapic_set_svr(struct vlapic *vlapic, uint32_t new)
7781c052192SNeel Natu {
7791c052192SNeel Natu 	struct LAPIC *lapic;
7801c052192SNeel Natu 	uint32_t old, changed;
7811c052192SNeel Natu 
7821c052192SNeel Natu 	lapic = &vlapic->apic;
7831c052192SNeel Natu 	old = lapic->svr;
7841c052192SNeel Natu 	changed = old ^ new;
7851c052192SNeel Natu 	if ((changed & APIC_SVR_ENABLE) != 0) {
7861c052192SNeel Natu 		if ((new & APIC_SVR_ENABLE) == 0) {
787fb03ca4eSNeel Natu 			/*
788fb03ca4eSNeel Natu 			 * The apic is now disabled so stop the apic timer.
789fb03ca4eSNeel Natu 			 */
7901c052192SNeel Natu 			VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
791fb03ca4eSNeel Natu 			VLAPIC_TIMER_LOCK(vlapic);
792fb03ca4eSNeel Natu 			callout_stop(&vlapic->callout);
793fb03ca4eSNeel Natu 			VLAPIC_TIMER_UNLOCK(vlapic);
7941c052192SNeel Natu 		} else {
795fb03ca4eSNeel Natu 			/*
796fb03ca4eSNeel Natu 			 * The apic is now enabled so restart the apic timer
797fb03ca4eSNeel Natu 			 * if it is configured in periodic mode.
798fb03ca4eSNeel Natu 			 */
7991c052192SNeel Natu 			VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
800fb03ca4eSNeel Natu 			if (vlapic_periodic_timer(vlapic))
801fb03ca4eSNeel Natu 				vlapic_set_icr_timer(vlapic, lapic->icr_timer);
8021c052192SNeel Natu 		}
8031c052192SNeel Natu 	}
8041c052192SNeel Natu 	lapic->svr = new;
8051c052192SNeel Natu }
8061c052192SNeel Natu 
807366f6083SPeter Grehan int
808*becd9849SNeel Natu vlapic_read(struct vlapic *vlapic, uint64_t offset, uint64_t *data, bool *retu)
809366f6083SPeter Grehan {
810366f6083SPeter Grehan 	struct LAPIC	*lapic = &vlapic->apic;
811366f6083SPeter Grehan 	uint32_t	*reg;
812366f6083SPeter Grehan 	int		 i;
813366f6083SPeter Grehan 
814366f6083SPeter Grehan 	if (offset > sizeof(*lapic)) {
815366f6083SPeter Grehan 		*data = 0;
8161c052192SNeel Natu 		goto done;
817366f6083SPeter Grehan 	}
818366f6083SPeter Grehan 
819366f6083SPeter Grehan 	offset &= ~3;
820366f6083SPeter Grehan 	switch(offset)
821366f6083SPeter Grehan 	{
822366f6083SPeter Grehan 		case APIC_OFFSET_ID:
8232d3a73edSNeel Natu 			if (x2apic(vlapic))
8242d3a73edSNeel Natu 				*data = vlapic->vcpuid;
8252d3a73edSNeel Natu 			else
8262d3a73edSNeel Natu 				*data = vlapic->vcpuid << 24;
827366f6083SPeter Grehan 			break;
828366f6083SPeter Grehan 		case APIC_OFFSET_VER:
829366f6083SPeter Grehan 			*data = lapic->version;
830366f6083SPeter Grehan 			break;
831366f6083SPeter Grehan 		case APIC_OFFSET_TPR:
832366f6083SPeter Grehan 			*data = lapic->tpr;
833366f6083SPeter Grehan 			break;
834366f6083SPeter Grehan 		case APIC_OFFSET_APR:
835366f6083SPeter Grehan 			*data = lapic->apr;
836366f6083SPeter Grehan 			break;
837366f6083SPeter Grehan 		case APIC_OFFSET_PPR:
838366f6083SPeter Grehan 			*data = lapic->ppr;
839366f6083SPeter Grehan 			break;
840366f6083SPeter Grehan 		case APIC_OFFSET_EOI:
841366f6083SPeter Grehan 			*data = lapic->eoi;
842366f6083SPeter Grehan 			break;
843366f6083SPeter Grehan 		case APIC_OFFSET_LDR:
844366f6083SPeter Grehan 			*data = lapic->ldr;
845366f6083SPeter Grehan 			break;
846366f6083SPeter Grehan 		case APIC_OFFSET_DFR:
847366f6083SPeter Grehan 			*data = lapic->dfr;
848366f6083SPeter Grehan 			break;
849366f6083SPeter Grehan 		case APIC_OFFSET_SVR:
850366f6083SPeter Grehan 			*data = lapic->svr;
851366f6083SPeter Grehan 			break;
852366f6083SPeter Grehan 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
853366f6083SPeter Grehan 			i = (offset - APIC_OFFSET_ISR0) >> 2;
854366f6083SPeter Grehan 			reg = &lapic->isr0;
855366f6083SPeter Grehan 			*data = *(reg + i);
856366f6083SPeter Grehan 			break;
857366f6083SPeter Grehan 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
858366f6083SPeter Grehan 			i = (offset - APIC_OFFSET_TMR0) >> 2;
859366f6083SPeter Grehan 			reg = &lapic->tmr0;
860366f6083SPeter Grehan 			*data = *(reg + i);
861366f6083SPeter Grehan 			break;
862366f6083SPeter Grehan 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
863366f6083SPeter Grehan 			i = (offset - APIC_OFFSET_IRR0) >> 2;
864366f6083SPeter Grehan 			reg = &lapic->irr0;
865366f6083SPeter Grehan 			*data = atomic_load_acq_int(reg + i);
866366f6083SPeter Grehan 			break;
867366f6083SPeter Grehan 		case APIC_OFFSET_ESR:
868366f6083SPeter Grehan 			*data = lapic->esr;
869366f6083SPeter Grehan 			break;
870366f6083SPeter Grehan 		case APIC_OFFSET_ICR_LOW:
871366f6083SPeter Grehan 			*data = lapic->icr_lo;
872366f6083SPeter Grehan 			break;
873366f6083SPeter Grehan 		case APIC_OFFSET_ICR_HI:
874366f6083SPeter Grehan 			*data = lapic->icr_hi;
875366f6083SPeter Grehan 			break;
876366f6083SPeter Grehan 		case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
877fb03ca4eSNeel Natu 			*data = vlapic_get_lvt(vlapic, offset);
878366f6083SPeter Grehan 			break;
879366f6083SPeter Grehan 		case APIC_OFFSET_ICR:
880366f6083SPeter Grehan 			*data = lapic->icr_timer;
881366f6083SPeter Grehan 			break;
882366f6083SPeter Grehan 		case APIC_OFFSET_CCR:
883366f6083SPeter Grehan 			*data = vlapic_get_ccr(vlapic);
884366f6083SPeter Grehan 			break;
885366f6083SPeter Grehan 		case APIC_OFFSET_DCR:
886366f6083SPeter Grehan 			*data = lapic->dcr_timer;
887366f6083SPeter Grehan 			break;
888366f6083SPeter Grehan 		case APIC_OFFSET_RRR:
889366f6083SPeter Grehan 		default:
890366f6083SPeter Grehan 			*data = 0;
891366f6083SPeter Grehan 			break;
892366f6083SPeter Grehan 	}
8931c052192SNeel Natu done:
8941c052192SNeel Natu 	VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
895366f6083SPeter Grehan 	return 0;
896366f6083SPeter Grehan }
897366f6083SPeter Grehan 
898366f6083SPeter Grehan int
899*becd9849SNeel Natu vlapic_write(struct vlapic *vlapic, uint64_t offset, uint64_t data, bool *retu)
900366f6083SPeter Grehan {
901366f6083SPeter Grehan 	struct LAPIC	*lapic = &vlapic->apic;
902366f6083SPeter Grehan 	int		retval;
903366f6083SPeter Grehan 
9041c052192SNeel Natu 	VLAPIC_CTR2(vlapic, "vlapic write offset %#x, data %#lx", offset, data);
9051c052192SNeel Natu 
906366f6083SPeter Grehan 	if (offset > sizeof(*lapic)) {
907366f6083SPeter Grehan 		return 0;
908366f6083SPeter Grehan 	}
909366f6083SPeter Grehan 
910366f6083SPeter Grehan 	retval = 0;
911366f6083SPeter Grehan 	offset &= ~3;
912366f6083SPeter Grehan 	switch(offset)
913366f6083SPeter Grehan 	{
914366f6083SPeter Grehan 		case APIC_OFFSET_ID:
915366f6083SPeter Grehan 			break;
916366f6083SPeter Grehan 		case APIC_OFFSET_TPR:
917366f6083SPeter Grehan 			lapic->tpr = data & 0xff;
918366f6083SPeter Grehan 			vlapic_update_ppr(vlapic);
919366f6083SPeter Grehan 			break;
920366f6083SPeter Grehan 		case APIC_OFFSET_EOI:
921366f6083SPeter Grehan 			vlapic_process_eoi(vlapic);
922366f6083SPeter Grehan 			break;
923366f6083SPeter Grehan 		case APIC_OFFSET_LDR:
924366f6083SPeter Grehan 			break;
925366f6083SPeter Grehan 		case APIC_OFFSET_DFR:
926366f6083SPeter Grehan 			break;
927366f6083SPeter Grehan 		case APIC_OFFSET_SVR:
9281c052192SNeel Natu 			lapic_set_svr(vlapic, data);
929366f6083SPeter Grehan 			break;
930366f6083SPeter Grehan 		case APIC_OFFSET_ICR_LOW:
931a2da7af6SNeel Natu 			if (!x2apic(vlapic)) {
932a2da7af6SNeel Natu 				data &= 0xffffffff;
933a2da7af6SNeel Natu 				data |= (uint64_t)lapic->icr_hi << 32;
934a2da7af6SNeel Natu 			}
935*becd9849SNeel Natu 			retval = lapic_process_icr(vlapic, data, retu);
936366f6083SPeter Grehan 			break;
937a2da7af6SNeel Natu 		case APIC_OFFSET_ICR_HI:
938a2da7af6SNeel Natu 			if (!x2apic(vlapic)) {
939a2da7af6SNeel Natu 				retval = 0;
940a2da7af6SNeel Natu 				lapic->icr_hi = data;
941a2da7af6SNeel Natu 			}
942a2da7af6SNeel Natu 			break;
943366f6083SPeter Grehan 		case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
944fb03ca4eSNeel Natu 			vlapic_set_lvt(vlapic, offset, data);
945366f6083SPeter Grehan 			break;
946366f6083SPeter Grehan 		case APIC_OFFSET_ICR:
947fb03ca4eSNeel Natu 			vlapic_set_icr_timer(vlapic, data);
948366f6083SPeter Grehan 			break;
949366f6083SPeter Grehan 
950366f6083SPeter Grehan 		case APIC_OFFSET_DCR:
951fb03ca4eSNeel Natu 			vlapic_set_dcr(vlapic, data);
952366f6083SPeter Grehan 			break;
953366f6083SPeter Grehan 
954366f6083SPeter Grehan 		case APIC_OFFSET_ESR:
955366f6083SPeter Grehan 			vlapic_update_errors(vlapic);
956366f6083SPeter Grehan 			break;
957366f6083SPeter Grehan 		case APIC_OFFSET_VER:
958366f6083SPeter Grehan 		case APIC_OFFSET_APR:
959366f6083SPeter Grehan 		case APIC_OFFSET_PPR:
960366f6083SPeter Grehan 		case APIC_OFFSET_RRR:
961366f6083SPeter Grehan 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
962366f6083SPeter Grehan 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
963366f6083SPeter Grehan 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
964366f6083SPeter Grehan 		case APIC_OFFSET_CCR:
965366f6083SPeter Grehan 		default:
966366f6083SPeter Grehan 			// Read only.
967366f6083SPeter Grehan 			break;
968366f6083SPeter Grehan 	}
969366f6083SPeter Grehan 
970366f6083SPeter Grehan 	return (retval);
971366f6083SPeter Grehan }
972366f6083SPeter Grehan 
973366f6083SPeter Grehan struct vlapic *
974366f6083SPeter Grehan vlapic_init(struct vm *vm, int vcpuid)
975366f6083SPeter Grehan {
976366f6083SPeter Grehan 	struct vlapic 		*vlapic;
977366f6083SPeter Grehan 
978366f6083SPeter Grehan 	vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
979366f6083SPeter Grehan 	vlapic->vm = vm;
980366f6083SPeter Grehan 	vlapic->vcpuid = vcpuid;
9812d3a73edSNeel Natu 
982*becd9849SNeel Natu 	/*
983*becd9849SNeel Natu 	 * If the vlapic is configured in x2apic mode then it will be
984*becd9849SNeel Natu 	 * accessed in the critical section via the MSR emulation code.
985*becd9849SNeel Natu 	 *
986*becd9849SNeel Natu 	 * Therefore the timer mutex must be a spinlock because blockable
987*becd9849SNeel Natu 	 * mutexes cannot be acquired in a critical section.
988*becd9849SNeel Natu 	 */
989*becd9849SNeel Natu 	mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
990fb03ca4eSNeel Natu 	callout_init(&vlapic->callout, 1);
991fb03ca4eSNeel Natu 
992a2da7af6SNeel Natu 	vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
9932d3a73edSNeel Natu 
9942d3a73edSNeel Natu 	if (vcpuid == 0)
9952d3a73edSNeel Natu 		vlapic->msr_apicbase |= APICBASE_BSP;
9962d3a73edSNeel Natu 
99703cd0501SNeel Natu 	vlapic_reset(vlapic);
998366f6083SPeter Grehan 
999366f6083SPeter Grehan 	return (vlapic);
1000366f6083SPeter Grehan }
1001366f6083SPeter Grehan 
1002366f6083SPeter Grehan void
1003366f6083SPeter Grehan vlapic_cleanup(struct vlapic *vlapic)
1004366f6083SPeter Grehan {
100503cd0501SNeel Natu 
1006fb03ca4eSNeel Natu 	callout_drain(&vlapic->callout);
1007366f6083SPeter Grehan 	free(vlapic, M_VLAPIC);
1008366f6083SPeter Grehan }
10092d3a73edSNeel Natu 
10102d3a73edSNeel Natu uint64_t
10112d3a73edSNeel Natu vlapic_get_apicbase(struct vlapic *vlapic)
10122d3a73edSNeel Natu {
10132d3a73edSNeel Natu 
10142d3a73edSNeel Natu 	return (vlapic->msr_apicbase);
10152d3a73edSNeel Natu }
10162d3a73edSNeel Natu 
10172d3a73edSNeel Natu void
10182d3a73edSNeel Natu vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
10192d3a73edSNeel Natu {
1020a2da7af6SNeel Natu 	int err;
1021a2da7af6SNeel Natu 	enum x2apic_state state;
1022a2da7af6SNeel Natu 
1023a2da7af6SNeel Natu 	err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state);
1024a2da7af6SNeel Natu 	if (err)
1025a2da7af6SNeel Natu 		panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
1026a2da7af6SNeel Natu 
1027a2da7af6SNeel Natu 	if (state == X2APIC_DISABLED)
1028a2da7af6SNeel Natu 		val &= ~APICBASE_X2APIC;
10292d3a73edSNeel Natu 
10302d3a73edSNeel Natu 	vlapic->msr_apicbase = val;
10312d3a73edSNeel Natu }
103273820fb0SNeel Natu 
103373820fb0SNeel Natu void
103473820fb0SNeel Natu vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
103573820fb0SNeel Natu {
103673820fb0SNeel Natu 	struct vlapic *vlapic;
103773820fb0SNeel Natu 
103873820fb0SNeel Natu 	vlapic = vm_lapic(vm, vcpuid);
103973820fb0SNeel Natu 
1040485f986aSNeel Natu 	if (state == X2APIC_DISABLED)
104173820fb0SNeel Natu 		vlapic->msr_apicbase &= ~APICBASE_X2APIC;
104273820fb0SNeel Natu }
10431c052192SNeel Natu 
10441c052192SNeel Natu bool
10451c052192SNeel Natu vlapic_enabled(struct vlapic *vlapic)
10461c052192SNeel Natu {
10471c052192SNeel Natu 	struct LAPIC *lapic = &vlapic->apic;
10481c052192SNeel Natu 
10491c052192SNeel Natu 	if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
10501c052192SNeel Natu 	    (lapic->svr & APIC_SVR_ENABLE) != 0)
10511c052192SNeel Natu 		return (true);
10521c052192SNeel Natu 	else
10531c052192SNeel Natu 		return (false);
10541c052192SNeel Natu }
1055