xref: /freebsd/sys/amd64/vmm/io/vlapic.c (revision 864c53ead899f7838cd2e1cca3b485a4a82f5cdc)
1 /*-
2  * Copyright (c) 2011 NetApp, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  * $FreeBSD$
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/lock.h>
34 #include <sys/kernel.h>
35 #include <sys/malloc.h>
36 #include <sys/mutex.h>
37 #include <sys/systm.h>
38 #include <sys/smp.h>
39 
40 #include <x86/specialreg.h>
41 #include <x86/apicreg.h>
42 
43 #include <machine/clock.h>
44 #include <machine/smp.h>
45 
46 #include <machine/vmm.h>
47 
48 #include "vmm_ipi.h"
49 #include "vmm_lapic.h"
50 #include "vmm_ktr.h"
51 #include "vmm_stat.h"
52 
53 #include "vlapic.h"
54 #include "vlapic_priv.h"
55 #include "vioapic.h"
56 
57 #define	PRIO(x)			((x) >> 4)
58 
59 #define VLAPIC_VERSION		(16)
60 
61 #define	x2apic(vlapic)	(((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
62 
63 /*
64  * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
65  * vlapic_callout_handler() and vcpu accesses to:
66  * - timer_freq_bt, timer_period_bt, timer_fire_bt
67  * - timer LVT register
68  */
69 #define	VLAPIC_TIMER_LOCK(vlapic)	mtx_lock_spin(&((vlapic)->timer_mtx))
70 #define	VLAPIC_TIMER_UNLOCK(vlapic)	mtx_unlock_spin(&((vlapic)->timer_mtx))
71 #define	VLAPIC_TIMER_LOCKED(vlapic)	mtx_owned(&((vlapic)->timer_mtx))
72 
73 /*
74  * APIC timer frequency:
75  * - arbitrary but chosen to be in the ballpark of contemporary hardware.
76  * - power-of-two to avoid loss of precision when converted to a bintime.
77  */
78 #define VLAPIC_BUS_FREQ		(128 * 1024 * 1024)
79 
80 static __inline uint32_t
81 vlapic_get_id(struct vlapic *vlapic)
82 {
83 
84 	if (x2apic(vlapic))
85 		return (vlapic->vcpuid);
86 	else
87 		return (vlapic->vcpuid << 24);
88 }
89 
90 static uint32_t
91 x2apic_ldr(struct vlapic *vlapic)
92 {
93 	int apicid;
94 	uint32_t ldr;
95 
96 	apicid = vlapic_get_id(vlapic);
97 	ldr = 1 << (apicid & 0xf);
98 	ldr |= (apicid & 0xffff0) << 12;
99 	return (ldr);
100 }
101 
102 void
103 vlapic_dfr_write_handler(struct vlapic *vlapic)
104 {
105 	struct LAPIC *lapic;
106 
107 	lapic = vlapic->apic_page;
108 	if (x2apic(vlapic)) {
109 		VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
110 		    lapic->dfr);
111 		lapic->dfr = 0;
112 		return;
113 	}
114 
115 	lapic->dfr &= APIC_DFR_MODEL_MASK;
116 	lapic->dfr |= APIC_DFR_RESERVED;
117 
118 	if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
119 		VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
120 	else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
121 		VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
122 	else
123 		VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
124 }
125 
126 void
127 vlapic_ldr_write_handler(struct vlapic *vlapic)
128 {
129 	struct LAPIC *lapic;
130 
131 	lapic = vlapic->apic_page;
132 
133 	/* LDR is read-only in x2apic mode */
134 	if (x2apic(vlapic)) {
135 		VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
136 		    lapic->ldr);
137 		lapic->ldr = x2apic_ldr(vlapic);
138 	} else {
139 		lapic->ldr &= ~APIC_LDR_RESERVED;
140 		VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
141 	}
142 }
143 
144 void
145 vlapic_id_write_handler(struct vlapic *vlapic)
146 {
147 	struct LAPIC *lapic;
148 
149 	/*
150 	 * We don't allow the ID register to be modified so reset it back to
151 	 * its default value.
152 	 */
153 	lapic = vlapic->apic_page;
154 	lapic->id = vlapic_get_id(vlapic);
155 }
156 
157 static int
158 vlapic_timer_divisor(uint32_t dcr)
159 {
160 	switch (dcr & 0xB) {
161 	case APIC_TDCR_1:
162 		return (1);
163 	case APIC_TDCR_2:
164 		return (2);
165 	case APIC_TDCR_4:
166 		return (4);
167 	case APIC_TDCR_8:
168 		return (8);
169 	case APIC_TDCR_16:
170 		return (16);
171 	case APIC_TDCR_32:
172 		return (32);
173 	case APIC_TDCR_64:
174 		return (64);
175 	case APIC_TDCR_128:
176 		return (128);
177 	default:
178 		panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
179 	}
180 }
181 
182 #if 0
183 static inline void
184 vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
185 {
186 	printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
187 	    *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
188 	    *lvt & APIC_LVTT_M);
189 }
190 #endif
191 
192 static uint32_t
193 vlapic_get_ccr(struct vlapic *vlapic)
194 {
195 	struct bintime bt_now, bt_rem;
196 	struct LAPIC *lapic;
197 	uint32_t ccr;
198 
199 	ccr = 0;
200 	lapic = vlapic->apic_page;
201 
202 	VLAPIC_TIMER_LOCK(vlapic);
203 	if (callout_active(&vlapic->callout)) {
204 		/*
205 		 * If the timer is scheduled to expire in the future then
206 		 * compute the value of 'ccr' based on the remaining time.
207 		 */
208 		binuptime(&bt_now);
209 		if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
210 			bt_rem = vlapic->timer_fire_bt;
211 			bintime_sub(&bt_rem, &bt_now);
212 			ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
213 			ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
214 		}
215 	}
216 	KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
217 	    "icr_timer is %#x", ccr, lapic->icr_timer));
218 	VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
219 	    ccr, lapic->icr_timer);
220 	VLAPIC_TIMER_UNLOCK(vlapic);
221 	return (ccr);
222 }
223 
224 void
225 vlapic_dcr_write_handler(struct vlapic *vlapic)
226 {
227 	struct LAPIC *lapic;
228 	int divisor;
229 
230 	lapic = vlapic->apic_page;
231 	VLAPIC_TIMER_LOCK(vlapic);
232 
233 	divisor = vlapic_timer_divisor(lapic->dcr_timer);
234 	VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
235 	    lapic->dcr_timer, divisor);
236 
237 	/*
238 	 * Update the timer frequency and the timer period.
239 	 *
240 	 * XXX changes to the frequency divider will not take effect until
241 	 * the timer is reloaded.
242 	 */
243 	FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
244 	vlapic->timer_period_bt = vlapic->timer_freq_bt;
245 	bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
246 
247 	VLAPIC_TIMER_UNLOCK(vlapic);
248 }
249 
250 void
251 vlapic_esr_write_handler(struct vlapic *vlapic)
252 {
253 	struct LAPIC *lapic;
254 
255 	lapic = vlapic->apic_page;
256 	lapic->esr = vlapic->esr_pending;
257 	vlapic->esr_pending = 0;
258 }
259 
260 int
261 vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
262 {
263 	struct LAPIC *lapic;
264 	uint32_t *irrptr, *tmrptr, mask;
265 	int idx;
266 
267 	KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
268 
269 	lapic = vlapic->apic_page;
270 	if (!(lapic->svr & APIC_SVR_ENABLE)) {
271 		VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
272 		    "interrupt %d", vector);
273 		return (0);
274 	}
275 
276 	if (vector < 16) {
277 		vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
278 		VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
279 		    vector);
280 		return (1);
281 	}
282 
283 	if (vlapic->ops.set_intr_ready)
284 		return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
285 
286 	idx = (vector / 32) * 4;
287 	mask = 1 << (vector % 32);
288 
289 	irrptr = &lapic->irr0;
290 	atomic_set_int(&irrptr[idx], mask);
291 
292 	/*
293 	 * Verify that the trigger-mode of the interrupt matches with
294 	 * the vlapic TMR registers.
295 	 */
296 	tmrptr = &lapic->tmr0;
297 	if ((tmrptr[idx] & mask) != (level ? mask : 0)) {
298 		VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but "
299 		    "interrupt is %s-triggered", idx / 4, tmrptr[idx],
300 		    level ? "level" : "edge");
301 	}
302 
303 	VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
304 	return (1);
305 }
306 
307 static __inline uint32_t *
308 vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
309 {
310 	struct LAPIC	*lapic = vlapic->apic_page;
311 	int 		 i;
312 
313 	switch (offset) {
314 	case APIC_OFFSET_CMCI_LVT:
315 		return (&lapic->lvt_cmci);
316 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
317 		i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
318 		return ((&lapic->lvt_timer) + i);;
319 	default:
320 		panic("vlapic_get_lvt: invalid LVT\n");
321 	}
322 }
323 
324 static __inline int
325 lvt_off_to_idx(uint32_t offset)
326 {
327 	int index;
328 
329 	switch (offset) {
330 	case APIC_OFFSET_CMCI_LVT:
331 		index = APIC_LVT_CMCI;
332 		break;
333 	case APIC_OFFSET_TIMER_LVT:
334 		index = APIC_LVT_TIMER;
335 		break;
336 	case APIC_OFFSET_THERM_LVT:
337 		index = APIC_LVT_THERMAL;
338 		break;
339 	case APIC_OFFSET_PERF_LVT:
340 		index = APIC_LVT_PMC;
341 		break;
342 	case APIC_OFFSET_LINT0_LVT:
343 		index = APIC_LVT_LINT0;
344 		break;
345 	case APIC_OFFSET_LINT1_LVT:
346 		index = APIC_LVT_LINT1;
347 		break;
348 	case APIC_OFFSET_ERROR_LVT:
349 		index = APIC_LVT_ERROR;
350 		break;
351 	default:
352 		index = -1;
353 		break;
354 	}
355 	KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
356 	    "invalid lvt index %d for offset %#x", index, offset));
357 
358 	return (index);
359 }
360 
361 static __inline uint32_t
362 vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
363 {
364 	int idx;
365 	uint32_t val;
366 
367 	idx = lvt_off_to_idx(offset);
368 	val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
369 	return (val);
370 }
371 
372 void
373 vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
374 {
375 	uint32_t *lvtptr, mask, val;
376 	struct LAPIC *lapic;
377 	int idx;
378 
379 	lapic = vlapic->apic_page;
380 	lvtptr = vlapic_get_lvtptr(vlapic, offset);
381 	val = *lvtptr;
382 	idx = lvt_off_to_idx(offset);
383 
384 	if (!(lapic->svr & APIC_SVR_ENABLE))
385 		val |= APIC_LVT_M;
386 	mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
387 	switch (offset) {
388 	case APIC_OFFSET_TIMER_LVT:
389 		mask |= APIC_LVTT_TM;
390 		break;
391 	case APIC_OFFSET_ERROR_LVT:
392 		break;
393 	case APIC_OFFSET_LINT0_LVT:
394 	case APIC_OFFSET_LINT1_LVT:
395 		mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
396 		/* FALLTHROUGH */
397 	default:
398 		mask |= APIC_LVT_DM;
399 		break;
400 	}
401 	val &= mask;
402 	*lvtptr = val;
403 	atomic_store_rel_32(&vlapic->lvt_last[idx], val);
404 }
405 
406 static void
407 vlapic_mask_lvts(struct vlapic *vlapic)
408 {
409 	struct LAPIC *lapic = vlapic->apic_page;
410 
411 	lapic->lvt_cmci |= APIC_LVT_M;
412 	vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
413 
414 	lapic->lvt_timer |= APIC_LVT_M;
415 	vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
416 
417 	lapic->lvt_thermal |= APIC_LVT_M;
418 	vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
419 
420 	lapic->lvt_pcint |= APIC_LVT_M;
421 	vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
422 
423 	lapic->lvt_lint0 |= APIC_LVT_M;
424 	vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
425 
426 	lapic->lvt_lint1 |= APIC_LVT_M;
427 	vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
428 
429 	lapic->lvt_error |= APIC_LVT_M;
430 	vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
431 }
432 
433 static int
434 vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
435 {
436 	uint32_t vec, mode;
437 
438 	if (lvt & APIC_LVT_M)
439 		return (0);
440 
441 	vec = lvt & APIC_LVT_VECTOR;
442 	mode = lvt & APIC_LVT_DM;
443 
444 	switch (mode) {
445 	case APIC_LVT_DM_FIXED:
446 		if (vec < 16) {
447 			vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
448 			return (0);
449 		}
450 		if (vlapic_set_intr_ready(vlapic, vec, false))
451 			vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
452 		break;
453 	case APIC_LVT_DM_NMI:
454 		vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
455 		break;
456 	case APIC_LVT_DM_EXTINT:
457 		vm_inject_extint(vlapic->vm, vlapic->vcpuid);
458 		break;
459 	default:
460 		// Other modes ignored
461 		return (0);
462 	}
463 	return (1);
464 }
465 
466 #if 1
467 static void
468 dump_isrvec_stk(struct vlapic *vlapic)
469 {
470 	int i;
471 	uint32_t *isrptr;
472 
473 	isrptr = &vlapic->apic_page->isr0;
474 	for (i = 0; i < 8; i++)
475 		printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
476 
477 	for (i = 0; i <= vlapic->isrvec_stk_top; i++)
478 		printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
479 }
480 #endif
481 
482 /*
483  * Algorithm adopted from section "Interrupt, Task and Processor Priority"
484  * in Intel Architecture Manual Vol 3a.
485  */
486 static void
487 vlapic_update_ppr(struct vlapic *vlapic)
488 {
489 	int isrvec, tpr, ppr;
490 
491 	/*
492 	 * Note that the value on the stack at index 0 is always 0.
493 	 *
494 	 * This is a placeholder for the value of ISRV when none of the
495 	 * bits is set in the ISRx registers.
496 	 */
497 	isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
498 	tpr = vlapic->apic_page->tpr;
499 
500 #if 1
501 	{
502 		int i, lastprio, curprio, vector, idx;
503 		uint32_t *isrptr;
504 
505 		if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
506 			panic("isrvec_stk is corrupted: %d", isrvec);
507 
508 		/*
509 		 * Make sure that the priority of the nested interrupts is
510 		 * always increasing.
511 		 */
512 		lastprio = -1;
513 		for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
514 			curprio = PRIO(vlapic->isrvec_stk[i]);
515 			if (curprio <= lastprio) {
516 				dump_isrvec_stk(vlapic);
517 				panic("isrvec_stk does not satisfy invariant");
518 			}
519 			lastprio = curprio;
520 		}
521 
522 		/*
523 		 * Make sure that each bit set in the ISRx registers has a
524 		 * corresponding entry on the isrvec stack.
525 		 */
526 		i = 1;
527 		isrptr = &vlapic->apic_page->isr0;
528 		for (vector = 0; vector < 256; vector++) {
529 			idx = (vector / 32) * 4;
530 			if (isrptr[idx] & (1 << (vector % 32))) {
531 				if (i > vlapic->isrvec_stk_top ||
532 				    vlapic->isrvec_stk[i] != vector) {
533 					dump_isrvec_stk(vlapic);
534 					panic("ISR and isrvec_stk out of sync");
535 				}
536 				i++;
537 			}
538 		}
539 	}
540 #endif
541 
542 	if (PRIO(tpr) >= PRIO(isrvec))
543 		ppr = tpr;
544 	else
545 		ppr = isrvec & 0xf0;
546 
547 	vlapic->apic_page->ppr = ppr;
548 	VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
549 }
550 
551 static void
552 vlapic_process_eoi(struct vlapic *vlapic)
553 {
554 	struct LAPIC	*lapic = vlapic->apic_page;
555 	uint32_t	*isrptr, *tmrptr;
556 	int		i, idx, bitpos, vector;
557 
558 	isrptr = &lapic->isr0;
559 	tmrptr = &lapic->tmr0;
560 
561 	/*
562 	 * The x86 architecture reserves the the first 32 vectors for use
563 	 * by the processor.
564 	 */
565 	for (i = 7; i > 0; i--) {
566 		idx = i * 4;
567 		bitpos = fls(isrptr[idx]);
568 		if (bitpos-- != 0) {
569 			if (vlapic->isrvec_stk_top <= 0) {
570 				panic("invalid vlapic isrvec_stk_top %d",
571 				      vlapic->isrvec_stk_top);
572 			}
573 			isrptr[idx] &= ~(1 << bitpos);
574 			VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
575 			vlapic->isrvec_stk_top--;
576 			vlapic_update_ppr(vlapic);
577 			if ((tmrptr[idx] & (1 << bitpos)) != 0) {
578 				vector = i * 32 + bitpos;
579 				vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
580 				    vector);
581 			}
582 			return;
583 		}
584 	}
585 }
586 
587 static __inline int
588 vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
589 {
590 
591 	return (lvt & mask);
592 }
593 
594 static __inline int
595 vlapic_periodic_timer(struct vlapic *vlapic)
596 {
597 	uint32_t lvt;
598 
599 	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
600 
601 	return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
602 }
603 
604 static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
605 
606 void
607 vlapic_set_error(struct vlapic *vlapic, uint32_t mask)
608 {
609 	uint32_t lvt;
610 
611 	vlapic->esr_pending |= mask;
612 	if (vlapic->esr_firing)
613 		return;
614 	vlapic->esr_firing = 1;
615 
616 	// The error LVT always uses the fixed delivery mode.
617 	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
618 	if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
619 		vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
620 	}
621 	vlapic->esr_firing = 0;
622 }
623 
624 static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
625 
626 static void
627 vlapic_fire_timer(struct vlapic *vlapic)
628 {
629 	uint32_t lvt;
630 
631 	KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
632 
633 	// The timer LVT always uses the fixed delivery mode.
634 	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
635 	if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
636 		vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
637 	}
638 }
639 
640 static VMM_STAT(VLAPIC_INTR_CMC,
641     "corrected machine check interrupts generated by vlapic");
642 
643 void
644 vlapic_fire_cmci(struct vlapic *vlapic)
645 {
646 	uint32_t lvt;
647 
648 	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
649 	if (vlapic_fire_lvt(vlapic, lvt)) {
650 		vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
651 	}
652 }
653 
654 static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
655     "lvts triggered");
656 
657 int
658 vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
659 {
660 	uint32_t lvt;
661 
662 	if (vlapic_enabled(vlapic) == false) {
663 		/*
664 		 * When the local APIC is global/hardware disabled,
665 		 * LINT[1:0] pins are configured as INTR and NMI pins,
666 		 * respectively.
667 		*/
668 		switch (vector) {
669 			case APIC_LVT_LINT0:
670 				vm_inject_extint(vlapic->vm, vlapic->vcpuid);
671 				break;
672 			case APIC_LVT_LINT1:
673 				vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
674 				break;
675 			default:
676 				break;
677 		}
678 		return (0);
679 	}
680 
681 	switch (vector) {
682 	case APIC_LVT_LINT0:
683 		lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT);
684 		break;
685 	case APIC_LVT_LINT1:
686 		lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT);
687 		break;
688 	case APIC_LVT_TIMER:
689 		lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
690 		lvt |= APIC_LVT_DM_FIXED;
691 		break;
692 	case APIC_LVT_ERROR:
693 		lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
694 		lvt |= APIC_LVT_DM_FIXED;
695 		break;
696 	case APIC_LVT_PMC:
697 		lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT);
698 		break;
699 	case APIC_LVT_THERMAL:
700 		lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT);
701 		break;
702 	case APIC_LVT_CMCI:
703 		lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
704 		break;
705 	default:
706 		return (EINVAL);
707 	}
708 	if (vlapic_fire_lvt(vlapic, lvt)) {
709 		vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
710 		    LVTS_TRIGGERRED, vector, 1);
711 	}
712 	return (0);
713 }
714 
715 static void
716 vlapic_callout_handler(void *arg)
717 {
718 	struct vlapic *vlapic;
719 	struct bintime bt, btnow;
720 	sbintime_t rem_sbt;
721 
722 	vlapic = arg;
723 
724 	VLAPIC_TIMER_LOCK(vlapic);
725 	if (callout_pending(&vlapic->callout))	/* callout was reset */
726 		goto done;
727 
728 	if (!callout_active(&vlapic->callout))	/* callout was stopped */
729 		goto done;
730 
731 	callout_deactivate(&vlapic->callout);
732 
733 	vlapic_fire_timer(vlapic);
734 
735 	if (vlapic_periodic_timer(vlapic)) {
736 		binuptime(&btnow);
737 		KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
738 		    ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
739 		    btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
740 		    vlapic->timer_fire_bt.frac));
741 
742 		/*
743 		 * Compute the delta between when the timer was supposed to
744 		 * fire and the present time.
745 		 */
746 		bt = btnow;
747 		bintime_sub(&bt, &vlapic->timer_fire_bt);
748 
749 		rem_sbt = bttosbt(vlapic->timer_period_bt);
750 		if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
751 			/*
752 			 * Adjust the time until the next countdown downward
753 			 * to account for the lost time.
754 			 */
755 			rem_sbt -= bttosbt(bt);
756 		} else {
757 			/*
758 			 * If the delta is greater than the timer period then
759 			 * just reset our time base instead of trying to catch
760 			 * up.
761 			 */
762 			vlapic->timer_fire_bt = btnow;
763 			VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
764 			    "usecs, period is %lu usecs - resetting time base",
765 			    bttosbt(bt) / SBT_1US,
766 			    bttosbt(vlapic->timer_period_bt) / SBT_1US);
767 		}
768 
769 		bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
770 		callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
771 		    vlapic_callout_handler, vlapic, 0);
772 	}
773 done:
774 	VLAPIC_TIMER_UNLOCK(vlapic);
775 }
776 
777 void
778 vlapic_icrtmr_write_handler(struct vlapic *vlapic)
779 {
780 	struct LAPIC *lapic;
781 	sbintime_t sbt;
782 	uint32_t icr_timer;
783 
784 	VLAPIC_TIMER_LOCK(vlapic);
785 
786 	lapic = vlapic->apic_page;
787 	icr_timer = lapic->icr_timer;
788 
789 	vlapic->timer_period_bt = vlapic->timer_freq_bt;
790 	bintime_mul(&vlapic->timer_period_bt, icr_timer);
791 
792 	if (icr_timer != 0) {
793 		binuptime(&vlapic->timer_fire_bt);
794 		bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
795 
796 		sbt = bttosbt(vlapic->timer_period_bt);
797 		callout_reset_sbt(&vlapic->callout, sbt, 0,
798 		    vlapic_callout_handler, vlapic, 0);
799 	} else
800 		callout_stop(&vlapic->callout);
801 
802 	VLAPIC_TIMER_UNLOCK(vlapic);
803 }
804 
805 /*
806  * This function populates 'dmask' with the set of vcpus that match the
807  * addressing specified by the (dest, phys, lowprio) tuple.
808  *
809  * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
810  * or xAPIC (8-bit) destination field.
811  */
812 static void
813 vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
814     bool lowprio, bool x2apic_dest)
815 {
816 	struct vlapic *vlapic;
817 	uint32_t dfr, ldr, ldest, cluster;
818 	uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
819 	cpuset_t amask;
820 	int vcpuid;
821 
822 	if ((x2apic_dest && dest == 0xffffffff) ||
823 	    (!x2apic_dest && dest == 0xff)) {
824 		/*
825 		 * Broadcast in both logical and physical modes.
826 		 */
827 		*dmask = vm_active_cpus(vm);
828 		return;
829 	}
830 
831 	if (phys) {
832 		/*
833 		 * Physical mode: destination is APIC ID.
834 		 */
835 		CPU_ZERO(dmask);
836 		vcpuid = vm_apicid2vcpuid(vm, dest);
837 		if (vcpuid < VM_MAXCPU)
838 			CPU_SET(vcpuid, dmask);
839 	} else {
840 		/*
841 		 * In the "Flat Model" the MDA is interpreted as an 8-bit wide
842 		 * bitmask. This model is only avilable in the xAPIC mode.
843 		 */
844 		mda_flat_ldest = dest & 0xff;
845 
846 		/*
847 		 * In the "Cluster Model" the MDA is used to identify a
848 		 * specific cluster and a set of APICs in that cluster.
849 		 */
850 		if (x2apic_dest) {
851 			mda_cluster_id = dest >> 16;
852 			mda_cluster_ldest = dest & 0xffff;
853 		} else {
854 			mda_cluster_id = (dest >> 4) & 0xf;
855 			mda_cluster_ldest = dest & 0xf;
856 		}
857 
858 		/*
859 		 * Logical mode: match each APIC that has a bit set
860 		 * in it's LDR that matches a bit in the ldest.
861 		 */
862 		CPU_ZERO(dmask);
863 		amask = vm_active_cpus(vm);
864 		while ((vcpuid = CPU_FFS(&amask)) != 0) {
865 			vcpuid--;
866 			CPU_CLR(vcpuid, &amask);
867 
868 			vlapic = vm_lapic(vm, vcpuid);
869 			dfr = vlapic->apic_page->dfr;
870 			ldr = vlapic->apic_page->ldr;
871 
872 			if ((dfr & APIC_DFR_MODEL_MASK) ==
873 			    APIC_DFR_MODEL_FLAT) {
874 				ldest = ldr >> 24;
875 				mda_ldest = mda_flat_ldest;
876 			} else if ((dfr & APIC_DFR_MODEL_MASK) ==
877 			    APIC_DFR_MODEL_CLUSTER) {
878 				if (x2apic(vlapic)) {
879 					cluster = ldr >> 16;
880 					ldest = ldr & 0xffff;
881 				} else {
882 					cluster = ldr >> 28;
883 					ldest = (ldr >> 24) & 0xf;
884 				}
885 				if (cluster != mda_cluster_id)
886 					continue;
887 				mda_ldest = mda_cluster_ldest;
888 			} else {
889 				/*
890 				 * Guest has configured a bad logical
891 				 * model for this vcpu - skip it.
892 				 */
893 				VLAPIC_CTR1(vlapic, "vlapic has bad logical "
894 				    "model %x - cannot deliver interrupt", dfr);
895 				continue;
896 			}
897 
898 			if ((mda_ldest & ldest) != 0) {
899 				CPU_SET(vcpuid, dmask);
900 				if (lowprio)
901 					break;
902 			}
903 		}
904 	}
905 }
906 
907 static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
908 
909 int
910 vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
911 {
912 	int i;
913 	bool phys;
914 	cpuset_t dmask;
915 	uint64_t icrval;
916 	uint32_t dest, vec, mode;
917 	struct vlapic *vlapic2;
918 	struct vm_exit *vmexit;
919 	struct LAPIC *lapic;
920 
921 	lapic = vlapic->apic_page;
922 	lapic->icr_lo &= ~APIC_DELSTAT_PEND;
923 	icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
924 
925 	if (x2apic(vlapic))
926 		dest = icrval >> 32;
927 	else
928 		dest = icrval >> (32 + 24);
929 	vec = icrval & APIC_VECTOR_MASK;
930 	mode = icrval & APIC_DELMODE_MASK;
931 
932 	if (mode == APIC_DELMODE_FIXED && vec < 16) {
933 		vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
934 		VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
935 		return (0);
936 	}
937 
938 	VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
939 
940 	if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
941 		switch (icrval & APIC_DEST_MASK) {
942 		case APIC_DEST_DESTFLD:
943 			phys = ((icrval & APIC_DESTMODE_LOG) == 0);
944 			vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
945 			    x2apic(vlapic));
946 			break;
947 		case APIC_DEST_SELF:
948 			CPU_SETOF(vlapic->vcpuid, &dmask);
949 			break;
950 		case APIC_DEST_ALLISELF:
951 			dmask = vm_active_cpus(vlapic->vm);
952 			break;
953 		case APIC_DEST_ALLESELF:
954 			dmask = vm_active_cpus(vlapic->vm);
955 			CPU_CLR(vlapic->vcpuid, &dmask);
956 			break;
957 		default:
958 			CPU_ZERO(&dmask);	/* satisfy gcc */
959 			break;
960 		}
961 
962 		while ((i = CPU_FFS(&dmask)) != 0) {
963 			i--;
964 			CPU_CLR(i, &dmask);
965 			if (mode == APIC_DELMODE_FIXED) {
966 				lapic_intr_edge(vlapic->vm, i, vec);
967 				vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
968 						    IPIS_SENT, i, 1);
969 				VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
970 				    "to vcpuid %d", vec, i);
971 			} else {
972 				vm_inject_nmi(vlapic->vm, i);
973 				VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
974 				    "to vcpuid %d", i);
975 			}
976 		}
977 
978 		return (0);	/* handled completely in the kernel */
979 	}
980 
981 	if (mode == APIC_DELMODE_INIT) {
982 		if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
983 			return (0);
984 
985 		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
986 			vlapic2 = vm_lapic(vlapic->vm, dest);
987 
988 			/* move from INIT to waiting-for-SIPI state */
989 			if (vlapic2->boot_state == BS_INIT) {
990 				vlapic2->boot_state = BS_SIPI;
991 			}
992 
993 			return (0);
994 		}
995 	}
996 
997 	if (mode == APIC_DELMODE_STARTUP) {
998 		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
999 			vlapic2 = vm_lapic(vlapic->vm, dest);
1000 
1001 			/*
1002 			 * Ignore SIPIs in any state other than wait-for-SIPI
1003 			 */
1004 			if (vlapic2->boot_state != BS_SIPI)
1005 				return (0);
1006 
1007 			vlapic2->boot_state = BS_RUNNING;
1008 
1009 			*retu = true;
1010 			vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
1011 			vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
1012 			vmexit->u.spinup_ap.vcpu = dest;
1013 			vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
1014 
1015 			return (0);
1016 		}
1017 	}
1018 
1019 	/*
1020 	 * This will cause a return to userland.
1021 	 */
1022 	return (1);
1023 }
1024 
1025 void
1026 vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val)
1027 {
1028 	int vec;
1029 
1030 	KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode"));
1031 
1032 	vec = val & 0xff;
1033 	lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec);
1034 	vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, IPIS_SENT,
1035 	    vlapic->vcpuid, 1);
1036 	VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec);
1037 }
1038 
1039 int
1040 vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1041 {
1042 	struct LAPIC	*lapic = vlapic->apic_page;
1043 	int	  	 idx, i, bitpos, vector;
1044 	uint32_t	*irrptr, val;
1045 
1046 	if (vlapic->ops.pending_intr)
1047 		return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
1048 
1049 	irrptr = &lapic->irr0;
1050 
1051 	/*
1052 	 * The x86 architecture reserves the the first 32 vectors for use
1053 	 * by the processor.
1054 	 */
1055 	for (i = 7; i > 0; i--) {
1056 		idx = i * 4;
1057 		val = atomic_load_acq_int(&irrptr[idx]);
1058 		bitpos = fls(val);
1059 		if (bitpos != 0) {
1060 			vector = i * 32 + (bitpos - 1);
1061 			if (PRIO(vector) > PRIO(lapic->ppr)) {
1062 				VLAPIC_CTR1(vlapic, "pending intr %d", vector);
1063 				if (vecptr != NULL)
1064 					*vecptr = vector;
1065 				return (1);
1066 			} else
1067 				break;
1068 		}
1069 	}
1070 	return (0);
1071 }
1072 
1073 void
1074 vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1075 {
1076 	struct LAPIC	*lapic = vlapic->apic_page;
1077 	uint32_t	*irrptr, *isrptr;
1078 	int		idx, stk_top;
1079 
1080 	if (vlapic->ops.intr_accepted)
1081 		return ((*vlapic->ops.intr_accepted)(vlapic, vector));
1082 
1083 	/*
1084 	 * clear the ready bit for vector being accepted in irr
1085 	 * and set the vector as in service in isr.
1086 	 */
1087 	idx = (vector / 32) * 4;
1088 
1089 	irrptr = &lapic->irr0;
1090 	atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
1091 	VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
1092 
1093 	isrptr = &lapic->isr0;
1094 	isrptr[idx] |= 1 << (vector % 32);
1095 	VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
1096 
1097 	/*
1098 	 * Update the PPR
1099 	 */
1100 	vlapic->isrvec_stk_top++;
1101 
1102 	stk_top = vlapic->isrvec_stk_top;
1103 	if (stk_top >= ISRVEC_STK_SIZE)
1104 		panic("isrvec_stk_top overflow %d", stk_top);
1105 
1106 	vlapic->isrvec_stk[stk_top] = vector;
1107 	vlapic_update_ppr(vlapic);
1108 }
1109 
1110 void
1111 vlapic_svr_write_handler(struct vlapic *vlapic)
1112 {
1113 	struct LAPIC *lapic;
1114 	uint32_t old, new, changed;
1115 
1116 	lapic = vlapic->apic_page;
1117 
1118 	new = lapic->svr;
1119 	old = vlapic->svr_last;
1120 	vlapic->svr_last = new;
1121 
1122 	changed = old ^ new;
1123 	if ((changed & APIC_SVR_ENABLE) != 0) {
1124 		if ((new & APIC_SVR_ENABLE) == 0) {
1125 			/*
1126 			 * The apic is now disabled so stop the apic timer
1127 			 * and mask all the LVT entries.
1128 			 */
1129 			VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1130 			VLAPIC_TIMER_LOCK(vlapic);
1131 			callout_stop(&vlapic->callout);
1132 			VLAPIC_TIMER_UNLOCK(vlapic);
1133 			vlapic_mask_lvts(vlapic);
1134 		} else {
1135 			/*
1136 			 * The apic is now enabled so restart the apic timer
1137 			 * if it is configured in periodic mode.
1138 			 */
1139 			VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1140 			if (vlapic_periodic_timer(vlapic))
1141 				vlapic_icrtmr_write_handler(vlapic);
1142 		}
1143 	}
1144 }
1145 
1146 int
1147 vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1148     uint64_t *data, bool *retu)
1149 {
1150 	struct LAPIC	*lapic = vlapic->apic_page;
1151 	uint32_t	*reg;
1152 	int		 i;
1153 
1154 	/* Ignore MMIO accesses in x2APIC mode */
1155 	if (x2apic(vlapic) && mmio_access) {
1156 		VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode",
1157 		    offset);
1158 		*data = 0;
1159 		goto done;
1160 	}
1161 
1162 	if (!x2apic(vlapic) && !mmio_access) {
1163 		/*
1164 		 * XXX Generate GP fault for MSR accesses in xAPIC mode
1165 		 */
1166 		VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in "
1167 		    "xAPIC mode", offset);
1168 		*data = 0;
1169 		goto done;
1170 	}
1171 
1172 	if (offset > sizeof(*lapic)) {
1173 		*data = 0;
1174 		goto done;
1175 	}
1176 
1177 	offset &= ~3;
1178 	switch(offset)
1179 	{
1180 		case APIC_OFFSET_ID:
1181 			*data = lapic->id;
1182 			break;
1183 		case APIC_OFFSET_VER:
1184 			*data = lapic->version;
1185 			break;
1186 		case APIC_OFFSET_TPR:
1187 			*data = vlapic_get_tpr(vlapic);
1188 			break;
1189 		case APIC_OFFSET_APR:
1190 			*data = lapic->apr;
1191 			break;
1192 		case APIC_OFFSET_PPR:
1193 			*data = lapic->ppr;
1194 			break;
1195 		case APIC_OFFSET_EOI:
1196 			*data = lapic->eoi;
1197 			break;
1198 		case APIC_OFFSET_LDR:
1199 			*data = lapic->ldr;
1200 			break;
1201 		case APIC_OFFSET_DFR:
1202 			*data = lapic->dfr;
1203 			break;
1204 		case APIC_OFFSET_SVR:
1205 			*data = lapic->svr;
1206 			break;
1207 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1208 			i = (offset - APIC_OFFSET_ISR0) >> 2;
1209 			reg = &lapic->isr0;
1210 			*data = *(reg + i);
1211 			break;
1212 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1213 			i = (offset - APIC_OFFSET_TMR0) >> 2;
1214 			reg = &lapic->tmr0;
1215 			*data = *(reg + i);
1216 			break;
1217 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1218 			i = (offset - APIC_OFFSET_IRR0) >> 2;
1219 			reg = &lapic->irr0;
1220 			*data = atomic_load_acq_int(reg + i);
1221 			break;
1222 		case APIC_OFFSET_ESR:
1223 			*data = lapic->esr;
1224 			break;
1225 		case APIC_OFFSET_ICR_LOW:
1226 			*data = lapic->icr_lo;
1227 			if (x2apic(vlapic))
1228 				*data |= (uint64_t)lapic->icr_hi << 32;
1229 			break;
1230 		case APIC_OFFSET_ICR_HI:
1231 			*data = lapic->icr_hi;
1232 			break;
1233 		case APIC_OFFSET_CMCI_LVT:
1234 		case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1235 			*data = vlapic_get_lvt(vlapic, offset);
1236 #ifdef INVARIANTS
1237 			reg = vlapic_get_lvtptr(vlapic, offset);
1238 			KASSERT(*data == *reg, ("inconsistent lvt value at "
1239 			    "offset %#lx: %#lx/%#x", offset, *data, *reg));
1240 #endif
1241 			break;
1242 		case APIC_OFFSET_TIMER_ICR:
1243 			*data = lapic->icr_timer;
1244 			break;
1245 		case APIC_OFFSET_TIMER_CCR:
1246 			*data = vlapic_get_ccr(vlapic);
1247 			break;
1248 		case APIC_OFFSET_TIMER_DCR:
1249 			*data = lapic->dcr_timer;
1250 			break;
1251 		case APIC_OFFSET_SELF_IPI:
1252 			/*
1253 			 * XXX generate a GP fault if vlapic is in x2apic mode
1254 			 */
1255 			*data = 0;
1256 			break;
1257 		case APIC_OFFSET_RRR:
1258 		default:
1259 			*data = 0;
1260 			break;
1261 	}
1262 done:
1263 	VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1264 	return 0;
1265 }
1266 
1267 int
1268 vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1269     uint64_t data, bool *retu)
1270 {
1271 	struct LAPIC	*lapic = vlapic->apic_page;
1272 	uint32_t	*regptr;
1273 	int		retval;
1274 
1275 	KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1276 	    ("vlapic_write: invalid offset %#lx", offset));
1277 
1278 	VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx",
1279 	    offset, data);
1280 
1281 	if (offset > sizeof(*lapic))
1282 		return (0);
1283 
1284 	/* Ignore MMIO accesses in x2APIC mode */
1285 	if (x2apic(vlapic) && mmio_access) {
1286 		VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx "
1287 		    "in x2APIC mode", data, offset);
1288 		return (0);
1289 	}
1290 
1291 	/*
1292 	 * XXX Generate GP fault for MSR accesses in xAPIC mode
1293 	 */
1294 	if (!x2apic(vlapic) && !mmio_access) {
1295 		VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx "
1296 		    "in xAPIC mode", data, offset);
1297 		return (0);
1298 	}
1299 
1300 	retval = 0;
1301 	switch(offset)
1302 	{
1303 		case APIC_OFFSET_ID:
1304 			lapic->id = data;
1305 			vlapic_id_write_handler(vlapic);
1306 			break;
1307 		case APIC_OFFSET_TPR:
1308 			vlapic_set_tpr(vlapic, data & 0xff);
1309 			break;
1310 		case APIC_OFFSET_EOI:
1311 			vlapic_process_eoi(vlapic);
1312 			break;
1313 		case APIC_OFFSET_LDR:
1314 			lapic->ldr = data;
1315 			vlapic_ldr_write_handler(vlapic);
1316 			break;
1317 		case APIC_OFFSET_DFR:
1318 			lapic->dfr = data;
1319 			vlapic_dfr_write_handler(vlapic);
1320 			break;
1321 		case APIC_OFFSET_SVR:
1322 			lapic->svr = data;
1323 			vlapic_svr_write_handler(vlapic);
1324 			break;
1325 		case APIC_OFFSET_ICR_LOW:
1326 			lapic->icr_lo = data;
1327 			if (x2apic(vlapic))
1328 				lapic->icr_hi = data >> 32;
1329 			retval = vlapic_icrlo_write_handler(vlapic, retu);
1330 			break;
1331 		case APIC_OFFSET_ICR_HI:
1332 			lapic->icr_hi = data;
1333 			break;
1334 		case APIC_OFFSET_CMCI_LVT:
1335 		case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1336 			regptr = vlapic_get_lvtptr(vlapic, offset);
1337 			*regptr = data;
1338 			vlapic_lvt_write_handler(vlapic, offset);
1339 			break;
1340 		case APIC_OFFSET_TIMER_ICR:
1341 			lapic->icr_timer = data;
1342 			vlapic_icrtmr_write_handler(vlapic);
1343 			break;
1344 
1345 		case APIC_OFFSET_TIMER_DCR:
1346 			lapic->dcr_timer = data;
1347 			vlapic_dcr_write_handler(vlapic);
1348 			break;
1349 
1350 		case APIC_OFFSET_ESR:
1351 			vlapic_esr_write_handler(vlapic);
1352 			break;
1353 
1354 		case APIC_OFFSET_SELF_IPI:
1355 			if (x2apic(vlapic))
1356 				vlapic_self_ipi_handler(vlapic, data);
1357 			break;
1358 
1359 		case APIC_OFFSET_VER:
1360 		case APIC_OFFSET_APR:
1361 		case APIC_OFFSET_PPR:
1362 		case APIC_OFFSET_RRR:
1363 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1364 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1365 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1366 		case APIC_OFFSET_TIMER_CCR:
1367 		default:
1368 			// Read only.
1369 			break;
1370 	}
1371 
1372 	return (retval);
1373 }
1374 
1375 static void
1376 vlapic_reset(struct vlapic *vlapic)
1377 {
1378 	struct LAPIC *lapic;
1379 
1380 	lapic = vlapic->apic_page;
1381 	bzero(lapic, sizeof(struct LAPIC));
1382 
1383 	lapic->id = vlapic_get_id(vlapic);
1384 	lapic->version = VLAPIC_VERSION;
1385 	lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
1386 	lapic->dfr = 0xffffffff;
1387 	lapic->svr = APIC_SVR_VECTOR;
1388 	vlapic_mask_lvts(vlapic);
1389 	vlapic_reset_tmr(vlapic);
1390 
1391 	lapic->dcr_timer = 0;
1392 	vlapic_dcr_write_handler(vlapic);
1393 
1394 	if (vlapic->vcpuid == 0)
1395 		vlapic->boot_state = BS_RUNNING;	/* BSP */
1396 	else
1397 		vlapic->boot_state = BS_INIT;		/* AP */
1398 
1399 	vlapic->svr_last = lapic->svr;
1400 }
1401 
1402 void
1403 vlapic_init(struct vlapic *vlapic)
1404 {
1405 	KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
1406 	KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU,
1407 	    ("vlapic_init: vcpuid is not initialized"));
1408 	KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
1409 	    "initialized"));
1410 
1411 	/*
1412 	 * If the vlapic is configured in x2apic mode then it will be
1413 	 * accessed in the critical section via the MSR emulation code.
1414 	 *
1415 	 * Therefore the timer mutex must be a spinlock because blockable
1416 	 * mutexes cannot be acquired in a critical section.
1417 	 */
1418 	mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1419 	callout_init(&vlapic->callout, 1);
1420 
1421 	vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1422 
1423 	if (vlapic->vcpuid == 0)
1424 		vlapic->msr_apicbase |= APICBASE_BSP;
1425 
1426 	vlapic_reset(vlapic);
1427 }
1428 
1429 void
1430 vlapic_cleanup(struct vlapic *vlapic)
1431 {
1432 
1433 	callout_drain(&vlapic->callout);
1434 }
1435 
1436 uint64_t
1437 vlapic_get_apicbase(struct vlapic *vlapic)
1438 {
1439 
1440 	return (vlapic->msr_apicbase);
1441 }
1442 
1443 int
1444 vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1445 {
1446 
1447 	if (vlapic->msr_apicbase != new) {
1448 		VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx "
1449 		    "not supported", vlapic->msr_apicbase, new);
1450 		return (-1);
1451 	}
1452 
1453 	return (0);
1454 }
1455 
1456 void
1457 vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1458 {
1459 	struct vlapic *vlapic;
1460 	struct LAPIC *lapic;
1461 
1462 	vlapic = vm_lapic(vm, vcpuid);
1463 
1464 	if (state == X2APIC_DISABLED)
1465 		vlapic->msr_apicbase &= ~APICBASE_X2APIC;
1466 	else
1467 		vlapic->msr_apicbase |= APICBASE_X2APIC;
1468 
1469 	/*
1470 	 * Reset the local APIC registers whose values are mode-dependent.
1471 	 *
1472 	 * XXX this works because the APIC mode can be changed only at vcpu
1473 	 * initialization time.
1474 	 */
1475 	lapic = vlapic->apic_page;
1476 	lapic->id = vlapic_get_id(vlapic);
1477 	if (x2apic(vlapic)) {
1478 		lapic->ldr = x2apic_ldr(vlapic);
1479 		lapic->dfr = 0;
1480 	} else {
1481 		lapic->ldr = 0;
1482 		lapic->dfr = 0xffffffff;
1483 	}
1484 
1485 	if (state == X2APIC_ENABLED) {
1486 		if (vlapic->ops.enable_x2apic_mode)
1487 			(*vlapic->ops.enable_x2apic_mode)(vlapic);
1488 	}
1489 }
1490 
1491 void
1492 vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
1493     int delmode, int vec)
1494 {
1495 	bool lowprio;
1496 	int vcpuid;
1497 	cpuset_t dmask;
1498 
1499 	if (delmode != IOART_DELFIXED &&
1500 	    delmode != IOART_DELLOPRI &&
1501 	    delmode != IOART_DELEXINT) {
1502 		VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
1503 		return;
1504 	}
1505 	lowprio = (delmode == IOART_DELLOPRI);
1506 
1507 	/*
1508 	 * We don't provide any virtual interrupt redirection hardware so
1509 	 * all interrupts originating from the ioapic or MSI specify the
1510 	 * 'dest' in the legacy xAPIC format.
1511 	 */
1512 	vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
1513 
1514 	while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1515 		vcpuid--;
1516 		CPU_CLR(vcpuid, &dmask);
1517 		if (delmode == IOART_DELEXINT) {
1518 			vm_inject_extint(vm, vcpuid);
1519 		} else {
1520 			lapic_set_intr(vm, vcpuid, vec, level);
1521 		}
1522 	}
1523 }
1524 
1525 void
1526 vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
1527 {
1528 	/*
1529 	 * Post an interrupt to the vcpu currently running on 'hostcpu'.
1530 	 *
1531 	 * This is done by leveraging features like Posted Interrupts (Intel)
1532 	 * Doorbell MSR (AMD AVIC) that avoid a VM exit.
1533 	 *
1534 	 * If neither of these features are available then fallback to
1535 	 * sending an IPI to 'hostcpu'.
1536 	 */
1537 	if (vlapic->ops.post_intr)
1538 		(*vlapic->ops.post_intr)(vlapic, hostcpu);
1539 	else
1540 		ipi_cpu(hostcpu, ipinum);
1541 }
1542 
1543 bool
1544 vlapic_enabled(struct vlapic *vlapic)
1545 {
1546 	struct LAPIC *lapic = vlapic->apic_page;
1547 
1548 	if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1549 	    (lapic->svr & APIC_SVR_ENABLE) != 0)
1550 		return (true);
1551 	else
1552 		return (false);
1553 }
1554 
1555 static void
1556 vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
1557 {
1558 	struct LAPIC *lapic;
1559 	uint32_t *tmrptr, mask;
1560 	int idx;
1561 
1562 	lapic = vlapic->apic_page;
1563 	tmrptr = &lapic->tmr0;
1564 	idx = (vector / 32) * 4;
1565 	mask = 1 << (vector % 32);
1566 	if (level)
1567 		tmrptr[idx] |= mask;
1568 	else
1569 		tmrptr[idx] &= ~mask;
1570 
1571 	if (vlapic->ops.set_tmr != NULL)
1572 		(*vlapic->ops.set_tmr)(vlapic, vector, level);
1573 }
1574 
1575 void
1576 vlapic_reset_tmr(struct vlapic *vlapic)
1577 {
1578 	int vector;
1579 
1580 	VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
1581 
1582 	for (vector = 0; vector <= 255; vector++)
1583 		vlapic_set_tmr(vlapic, vector, false);
1584 }
1585 
1586 void
1587 vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
1588     int delmode, int vector)
1589 {
1590 	cpuset_t dmask;
1591 	bool lowprio;
1592 
1593 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
1594 
1595 	/*
1596 	 * A level trigger is valid only for fixed and lowprio delivery modes.
1597 	 */
1598 	if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1599 		VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
1600 		    "delivery-mode %d", delmode);
1601 		return;
1602 	}
1603 
1604 	lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1605 	vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
1606 
1607 	if (!CPU_ISSET(vlapic->vcpuid, &dmask))
1608 		return;
1609 
1610 	VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
1611 	vlapic_set_tmr(vlapic, vector, true);
1612 }
1613 
1614 void
1615 vlapic_set_tpr(struct vlapic *vlapic, uint8_t val)
1616 {
1617 	struct LAPIC	*lapic = vlapic->apic_page;
1618 
1619 	lapic->tpr = val;
1620 	vlapic_update_ppr(vlapic);
1621 }
1622 
1623 uint8_t
1624 vlapic_get_tpr(struct vlapic *vlapic)
1625 {
1626 	struct LAPIC	*lapic = vlapic->apic_page;
1627 
1628 	return (lapic->tpr);
1629 }
1630