xref: /freebsd/sys/amd64/vmm/io/vlapic.c (revision d2ce15bd43b3a1dcce08eecbff8d5d359946d972)
1 /*-
2  * Copyright (c) 2011 NetApp, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  * $FreeBSD$
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/malloc.h>
35 #include <sys/systm.h>
36 #include <sys/smp.h>
37 
38 #include <machine/clock.h>
39 #include <x86/specialreg.h>
40 #include <x86/apicreg.h>
41 
42 #include <machine/vmm.h>
43 
44 #include "vmm_stat.h"
45 #include "vmm_lapic.h"
46 #include "vmm_ktr.h"
47 #include "vdev.h"
48 #include "vlapic.h"
49 
50 #define	VLAPIC_CTR0(vlapic, format)					\
51 	VMM_CTR0((vlapic)->vm, (vlapic)->vcpuid, format)
52 
53 #define	VLAPIC_CTR1(vlapic, format, p1)					\
54 	VMM_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1)
55 
56 #define	VLAPIC_CTR_IRR(vlapic, msg)					\
57 do {									\
58 	uint32_t *irrptr = &(vlapic)->apic.irr0;			\
59 	irrptr[0] = irrptr[0];	/* silence compiler */			\
60 	VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]);	\
61 	VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]);	\
62 	VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]);	\
63 	VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]);	\
64 	VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]);	\
65 	VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]);	\
66 	VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]);	\
67 	VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]);	\
68 } while (0)
69 
70 #define	VLAPIC_CTR_ISR(vlapic, msg)					\
71 do {									\
72 	uint32_t *isrptr = &(vlapic)->apic.isr0;			\
73 	isrptr[0] = isrptr[0];	/* silence compiler */			\
74 	VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]);	\
75 	VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]);	\
76 	VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]);	\
77 	VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]);	\
78 	VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]);	\
79 	VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]);	\
80 	VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]);	\
81 	VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]);	\
82 } while (0)
83 
84 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
85 
86 #define	PRIO(x)			((x) >> 4)
87 
88 #define VLAPIC_VERSION		(16)
89 #define VLAPIC_MAXLVT_ENTRIES	(5)
90 
91 #define	x2apic(vlapic)	(((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
92 
93 enum boot_state {
94 	BS_INIT,
95 	BS_SIPI,
96 	BS_RUNNING
97 };
98 
99 struct vlapic {
100 	struct vm		*vm;
101 	int			vcpuid;
102 
103 	struct io_region	*mmio;
104 	struct vdev_ops		*ops;
105 	struct LAPIC		 apic;
106 
107 	int			 esr_update;
108 
109 	int			 divisor;
110 	int			 ccr_ticks;
111 
112 	/*
113 	 * The 'isrvec_stk' is a stack of vectors injected by the local apic.
114 	 * A vector is popped from the stack when the processor does an EOI.
115 	 * The vector on the top of the stack is used to compute the
116 	 * Processor Priority in conjunction with the TPR.
117 	 */
118 	uint8_t			 isrvec_stk[ISRVEC_STK_SIZE];
119 	int			 isrvec_stk_top;
120 
121 	uint64_t		msr_apicbase;
122 	enum boot_state		boot_state;
123 };
124 
125 #define VLAPIC_BUS_FREQ	tsc_freq
126 
127 static int
128 vlapic_timer_divisor(uint32_t dcr)
129 {
130 	switch (dcr & 0xB) {
131 	case APIC_TDCR_1:
132 		return (1);
133 	case APIC_TDCR_2:
134 		return (2);
135 	case APIC_TDCR_4:
136 		return (4);
137 	case APIC_TDCR_8:
138 		return (8);
139 	case APIC_TDCR_16:
140 		return (16);
141 	case APIC_TDCR_32:
142 		return (32);
143 	case APIC_TDCR_64:
144 		return (64);
145 	case APIC_TDCR_128:
146 		return (128);
147 	default:
148 		panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
149 	}
150 }
151 
152 static void
153 vlapic_mask_lvts(uint32_t *lvts, int num_lvt)
154 {
155 	int i;
156 	for (i = 0; i < num_lvt; i++) {
157 		*lvts |= APIC_LVT_M;
158 		lvts += 4;
159 	}
160 }
161 
162 #if 0
163 static inline void
164 vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
165 {
166 	printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
167 	    *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
168 	    *lvt & APIC_LVTT_M);
169 }
170 #endif
171 
172 static uint64_t
173 vlapic_get_ccr(struct vlapic *vlapic)
174 {
175 	struct LAPIC    *lapic = &vlapic->apic;
176 	return lapic->ccr_timer;
177 }
178 
179 static void
180 vlapic_update_errors(struct vlapic *vlapic)
181 {
182 	struct LAPIC    *lapic = &vlapic->apic;
183 	lapic->esr = 0; // XXX
184 }
185 
186 static void
187 vlapic_init_ipi(struct vlapic *vlapic)
188 {
189 	struct LAPIC    *lapic = &vlapic->apic;
190 	lapic->version = VLAPIC_VERSION;
191 	lapic->version |= (VLAPIC_MAXLVT_ENTRIES < MAXLVTSHIFT);
192 	lapic->dfr = 0xffffffff;
193 	lapic->svr = APIC_SVR_VECTOR;
194 	vlapic_mask_lvts(&lapic->lvt_timer, VLAPIC_MAXLVT_ENTRIES+1);
195 }
196 
197 static int
198 vlapic_op_reset(void* dev)
199 {
200 	struct vlapic 	*vlapic = (struct vlapic*)dev;
201 	struct LAPIC	*lapic = &vlapic->apic;
202 
203 	memset(lapic, 0, sizeof(*lapic));
204 	lapic->apr = vlapic->vcpuid;
205 	vlapic_init_ipi(vlapic);
206 	vlapic->divisor = vlapic_timer_divisor(lapic->dcr_timer);
207 
208 	if (vlapic->vcpuid == 0)
209 		vlapic->boot_state = BS_RUNNING;	/* BSP */
210 	else
211 		vlapic->boot_state = BS_INIT;		/* AP */
212 
213 	return 0;
214 
215 }
216 
217 static int
218 vlapic_op_init(void* dev)
219 {
220 	struct vlapic *vlapic = (struct vlapic*)dev;
221 	vdev_register_region(vlapic->ops, vlapic, vlapic->mmio);
222 	return vlapic_op_reset(dev);
223 }
224 
225 static int
226 vlapic_op_halt(void* dev)
227 {
228 	struct vlapic *vlapic = (struct vlapic*)dev;
229 	vdev_unregister_region(vlapic, vlapic->mmio);
230 	return 0;
231 
232 }
233 
234 void
235 vlapic_set_intr_ready(struct vlapic *vlapic, int vector)
236 {
237 	struct LAPIC	*lapic = &vlapic->apic;
238 	uint32_t	*irrptr;
239 	int		idx;
240 
241 	if (vector < 0 || vector >= 256)
242 		panic("vlapic_set_intr_ready: invalid vector %d\n", vector);
243 
244 	idx = (vector / 32) * 4;
245 	irrptr = &lapic->irr0;
246 	atomic_set_int(&irrptr[idx], 1 << (vector % 32));
247 	VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
248 }
249 
250 static void
251 vlapic_start_timer(struct vlapic *vlapic, uint32_t elapsed)
252 {
253 	uint32_t icr_timer;
254 
255 	icr_timer = vlapic->apic.icr_timer;
256 
257 	vlapic->ccr_ticks = ticks;
258 	if (elapsed < icr_timer)
259 		vlapic->apic.ccr_timer = icr_timer - elapsed;
260 	else {
261 		/*
262 		 * This can happen when the guest is trying to run its local
263 		 * apic timer higher that the setting of 'hz' in the host.
264 		 *
265 		 * We deal with this by running the guest local apic timer
266 		 * at the rate of the host's 'hz' setting.
267 		 */
268 		vlapic->apic.ccr_timer = 0;
269 	}
270 }
271 
272 static __inline uint32_t *
273 vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
274 {
275 	struct LAPIC	*lapic = &vlapic->apic;
276 	int 		 i;
277 
278 	if (offset < APIC_OFFSET_TIMER_LVT || offset > APIC_OFFSET_ERROR_LVT) {
279 		panic("vlapic_get_lvt: invalid LVT\n");
280 	}
281 	i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
282 	return ((&lapic->lvt_timer) + i);;
283 }
284 
285 #if 1
286 static void
287 dump_isrvec_stk(struct vlapic *vlapic)
288 {
289 	int i;
290 	uint32_t *isrptr;
291 
292 	isrptr = &vlapic->apic.isr0;
293 	for (i = 0; i < 8; i++)
294 		printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
295 
296 	for (i = 0; i <= vlapic->isrvec_stk_top; i++)
297 		printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
298 }
299 #endif
300 
301 /*
302  * Algorithm adopted from section "Interrupt, Task and Processor Priority"
303  * in Intel Architecture Manual Vol 3a.
304  */
305 static void
306 vlapic_update_ppr(struct vlapic *vlapic)
307 {
308 	int isrvec, tpr, ppr;
309 
310 	/*
311 	 * Note that the value on the stack at index 0 is always 0.
312 	 *
313 	 * This is a placeholder for the value of ISRV when none of the
314 	 * bits is set in the ISRx registers.
315 	 */
316 	isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
317 	tpr = vlapic->apic.tpr;
318 
319 #if 1
320 	{
321 		int i, lastprio, curprio, vector, idx;
322 		uint32_t *isrptr;
323 
324 		if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
325 			panic("isrvec_stk is corrupted: %d", isrvec);
326 
327 		/*
328 		 * Make sure that the priority of the nested interrupts is
329 		 * always increasing.
330 		 */
331 		lastprio = -1;
332 		for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
333 			curprio = PRIO(vlapic->isrvec_stk[i]);
334 			if (curprio <= lastprio) {
335 				dump_isrvec_stk(vlapic);
336 				panic("isrvec_stk does not satisfy invariant");
337 			}
338 			lastprio = curprio;
339 		}
340 
341 		/*
342 		 * Make sure that each bit set in the ISRx registers has a
343 		 * corresponding entry on the isrvec stack.
344 		 */
345 		i = 1;
346 		isrptr = &vlapic->apic.isr0;
347 		for (vector = 0; vector < 256; vector++) {
348 			idx = (vector / 32) * 4;
349 			if (isrptr[idx] & (1 << (vector % 32))) {
350 				if (i > vlapic->isrvec_stk_top ||
351 				    vlapic->isrvec_stk[i] != vector) {
352 					dump_isrvec_stk(vlapic);
353 					panic("ISR and isrvec_stk out of sync");
354 				}
355 				i++;
356 			}
357 		}
358 	}
359 #endif
360 
361 	if (PRIO(tpr) >= PRIO(isrvec))
362 		ppr = tpr;
363 	else
364 		ppr = isrvec & 0xf0;
365 
366 	vlapic->apic.ppr = ppr;
367 	VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
368 }
369 
370 static void
371 vlapic_process_eoi(struct vlapic *vlapic)
372 {
373 	struct LAPIC	*lapic = &vlapic->apic;
374 	uint32_t	*isrptr;
375 	int		i, idx, bitpos;
376 
377 	isrptr = &lapic->isr0;
378 
379 	/*
380 	 * The x86 architecture reserves the the first 32 vectors for use
381 	 * by the processor.
382 	 */
383 	for (i = 7; i > 0; i--) {
384 		idx = i * 4;
385 		bitpos = fls(isrptr[idx]);
386 		if (bitpos != 0) {
387 			if (vlapic->isrvec_stk_top <= 0) {
388 				panic("invalid vlapic isrvec_stk_top %d",
389 				      vlapic->isrvec_stk_top);
390 			}
391 			isrptr[idx] &= ~(1 << (bitpos - 1));
392 			VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
393 			vlapic->isrvec_stk_top--;
394 			vlapic_update_ppr(vlapic);
395 			return;
396 		}
397 	}
398 }
399 
400 static __inline int
401 vlapic_get_lvt_field(uint32_t *lvt, uint32_t mask)
402 {
403 	return (*lvt & mask);
404 }
405 
406 static __inline int
407 vlapic_periodic_timer(struct vlapic *vlapic)
408 {
409 	uint32_t *lvt;
410 
411 	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
412 
413 	return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
414 }
415 
416 static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
417 
418 static void
419 vlapic_fire_timer(struct vlapic *vlapic)
420 {
421 	int vector;
422 	uint32_t *lvt;
423 
424 	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
425 
426 	if (!vlapic_get_lvt_field(lvt, APIC_LVTT_M)) {
427 		vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
428 		vector = vlapic_get_lvt_field(lvt,APIC_LVTT_VECTOR);
429 		vlapic_set_intr_ready(vlapic, vector);
430 	}
431 }
432 
433 static int
434 lapic_process_icr(struct vlapic *vlapic, uint64_t icrval)
435 {
436 	int i;
437 	cpuset_t dmask;
438 	uint32_t dest, vec, mode;
439 	struct vlapic *vlapic2;
440 	struct vm_exit *vmexit;
441 
442 	if (x2apic(vlapic))
443 		dest = icrval >> 32;
444 	else
445 		dest = icrval >> (32 + 24);
446 	vec = icrval & APIC_VECTOR_MASK;
447 	mode = icrval & APIC_DELMODE_MASK;
448 
449 	if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
450 		switch (icrval & APIC_DEST_MASK) {
451 		case APIC_DEST_DESTFLD:
452 			CPU_SETOF(dest, &dmask);
453 			break;
454 		case APIC_DEST_SELF:
455 			CPU_SETOF(vlapic->vcpuid, &dmask);
456 			break;
457 		case APIC_DEST_ALLISELF:
458 			dmask = vm_active_cpus(vlapic->vm);
459 			break;
460 		case APIC_DEST_ALLESELF:
461 			dmask = vm_active_cpus(vlapic->vm);
462 			CPU_CLR(vlapic->vcpuid, &dmask);
463 			break;
464 		}
465 
466 		while ((i = cpusetobj_ffs(&dmask)) != 0) {
467 			i--;
468 			CPU_CLR(i, &dmask);
469 			if (mode == APIC_DELMODE_FIXED)
470 				lapic_set_intr(vlapic->vm, i, vec);
471 			else
472 				vm_inject_nmi(vlapic->vm, i);
473 		}
474 
475 		return (0);	/* handled completely in the kernel */
476 	}
477 
478 	if (mode == APIC_DELMODE_INIT) {
479 		if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
480 			return (0);
481 
482 		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
483 			vlapic2 = vm_lapic(vlapic->vm, dest);
484 
485 			/* move from INIT to waiting-for-SIPI state */
486 			if (vlapic2->boot_state == BS_INIT) {
487 				vlapic2->boot_state = BS_SIPI;
488 			}
489 
490 			return (0);
491 		}
492 	}
493 
494 	if (mode == APIC_DELMODE_STARTUP) {
495 		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
496 			vlapic2 = vm_lapic(vlapic->vm, dest);
497 
498 			/*
499 			 * Ignore SIPIs in any state other than wait-for-SIPI
500 			 */
501 			if (vlapic2->boot_state != BS_SIPI)
502 				return (0);
503 
504 			vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
505 			vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
506 			vmexit->u.spinup_ap.vcpu = dest;
507 			vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
508 
509 			/*
510 			 * XXX this assumes that the startup IPI always succeeds
511 			 */
512 			vlapic2->boot_state = BS_RUNNING;
513 			vm_activate_cpu(vlapic2->vm, dest);
514 
515 			return (0);
516 		}
517 	}
518 
519 	/*
520 	 * This will cause a return to userland.
521 	 */
522 	return (1);
523 }
524 
525 int
526 vlapic_pending_intr(struct vlapic *vlapic)
527 {
528 	struct LAPIC	*lapic = &vlapic->apic;
529 	int	  	 idx, i, bitpos, vector;
530 	uint32_t	*irrptr, val;
531 
532 	irrptr = &lapic->irr0;
533 
534 	/*
535 	 * The x86 architecture reserves the the first 32 vectors for use
536 	 * by the processor.
537 	 */
538 	for (i = 7; i > 0; i--) {
539 		idx = i * 4;
540 		val = atomic_load_acq_int(&irrptr[idx]);
541 		bitpos = fls(val);
542 		if (bitpos != 0) {
543 			vector = i * 32 + (bitpos - 1);
544 			if (PRIO(vector) > PRIO(lapic->ppr)) {
545 				VLAPIC_CTR1(vlapic, "pending intr %d", vector);
546 				return (vector);
547 			} else
548 				break;
549 		}
550 	}
551 	VLAPIC_CTR0(vlapic, "no pending intr");
552 	return (-1);
553 }
554 
555 void
556 vlapic_intr_accepted(struct vlapic *vlapic, int vector)
557 {
558 	struct LAPIC	*lapic = &vlapic->apic;
559 	uint32_t	*irrptr, *isrptr;
560 	int		idx, stk_top;
561 
562 	/*
563 	 * clear the ready bit for vector being accepted in irr
564 	 * and set the vector as in service in isr.
565 	 */
566 	idx = (vector / 32) * 4;
567 
568 	irrptr = &lapic->irr0;
569 	atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
570 	VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
571 
572 	isrptr = &lapic->isr0;
573 	isrptr[idx] |= 1 << (vector % 32);
574 	VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
575 
576 	/*
577 	 * Update the PPR
578 	 */
579 	vlapic->isrvec_stk_top++;
580 
581 	stk_top = vlapic->isrvec_stk_top;
582 	if (stk_top >= ISRVEC_STK_SIZE)
583 		panic("isrvec_stk_top overflow %d", stk_top);
584 
585 	vlapic->isrvec_stk[stk_top] = vector;
586 	vlapic_update_ppr(vlapic);
587 }
588 
589 int
590 vlapic_op_mem_read(void* dev, uint64_t gpa, opsize_t size, uint64_t *data)
591 {
592 	struct vlapic 	*vlapic = (struct vlapic*)dev;
593 	struct LAPIC	*lapic = &vlapic->apic;
594 	uint64_t	 offset = gpa & ~(PAGE_SIZE);
595 	uint32_t	*reg;
596 	int		 i;
597 
598 	if (offset > sizeof(*lapic)) {
599 		*data = 0;
600 		return 0;
601 	}
602 
603 	offset &= ~3;
604 	switch(offset)
605 	{
606 		case APIC_OFFSET_ID:
607 			if (x2apic(vlapic))
608 				*data = vlapic->vcpuid;
609 			else
610 				*data = vlapic->vcpuid << 24;
611 			break;
612 		case APIC_OFFSET_VER:
613 			*data = lapic->version;
614 			break;
615 		case APIC_OFFSET_TPR:
616 			*data = lapic->tpr;
617 			break;
618 		case APIC_OFFSET_APR:
619 			*data = lapic->apr;
620 			break;
621 		case APIC_OFFSET_PPR:
622 			*data = lapic->ppr;
623 			break;
624 		case APIC_OFFSET_EOI:
625 			*data = lapic->eoi;
626 			break;
627 		case APIC_OFFSET_LDR:
628 			*data = lapic->ldr;
629 			break;
630 		case APIC_OFFSET_DFR:
631 			*data = lapic->dfr;
632 			break;
633 		case APIC_OFFSET_SVR:
634 			*data = lapic->svr;
635 			break;
636 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
637 			i = (offset - APIC_OFFSET_ISR0) >> 2;
638 			reg = &lapic->isr0;
639 			*data = *(reg + i);
640 			break;
641 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
642 			i = (offset - APIC_OFFSET_TMR0) >> 2;
643 			reg = &lapic->tmr0;
644 			*data = *(reg + i);
645 			break;
646 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
647 			i = (offset - APIC_OFFSET_IRR0) >> 2;
648 			reg = &lapic->irr0;
649 			*data = atomic_load_acq_int(reg + i);
650 			break;
651 		case APIC_OFFSET_ESR:
652 			*data = lapic->esr;
653 			break;
654 		case APIC_OFFSET_ICR_LOW:
655 			*data = lapic->icr_lo;
656 			break;
657 		case APIC_OFFSET_ICR_HI:
658 			*data = lapic->icr_hi;
659 			break;
660 		case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
661 			reg = vlapic_get_lvt(vlapic, offset);
662 			*data = *(reg);
663 			break;
664 		case APIC_OFFSET_ICR:
665 			*data = lapic->icr_timer;
666 			break;
667 		case APIC_OFFSET_CCR:
668 			*data = vlapic_get_ccr(vlapic);
669 			break;
670 		case APIC_OFFSET_DCR:
671 			*data = lapic->dcr_timer;
672 			break;
673 		case APIC_OFFSET_RRR:
674 		default:
675 			*data = 0;
676 			break;
677 	}
678 	return 0;
679 }
680 
681 int
682 vlapic_op_mem_write(void* dev, uint64_t gpa, opsize_t size, uint64_t data)
683 {
684 	struct vlapic 	*vlapic = (struct vlapic*)dev;
685 	struct LAPIC	*lapic = &vlapic->apic;
686 	uint64_t	 offset = gpa & ~(PAGE_SIZE);
687 	uint32_t	*reg;
688 	int		retval;
689 
690 	if (offset > sizeof(*lapic)) {
691 		return 0;
692 	}
693 
694 	retval = 0;
695 	offset &= ~3;
696 	switch(offset)
697 	{
698 		case APIC_OFFSET_ID:
699 			break;
700 		case APIC_OFFSET_TPR:
701 			lapic->tpr = data & 0xff;
702 			vlapic_update_ppr(vlapic);
703 			break;
704 		case APIC_OFFSET_EOI:
705 			vlapic_process_eoi(vlapic);
706 			break;
707 		case APIC_OFFSET_LDR:
708 			break;
709 		case APIC_OFFSET_DFR:
710 			break;
711 		case APIC_OFFSET_SVR:
712 			lapic->svr = data;
713 			break;
714 		case APIC_OFFSET_ICR_LOW:
715 			if (!x2apic(vlapic)) {
716 				data &= 0xffffffff;
717 				data |= (uint64_t)lapic->icr_hi << 32;
718 			}
719 			retval = lapic_process_icr(vlapic, data);
720 			break;
721 		case APIC_OFFSET_ICR_HI:
722 			if (!x2apic(vlapic)) {
723 				retval = 0;
724 				lapic->icr_hi = data;
725 			}
726 			break;
727 		case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
728 			reg = vlapic_get_lvt(vlapic, offset);
729 			if (!(lapic->svr & APIC_SVR_ENABLE)) {
730 				data |= APIC_LVT_M;
731 			}
732 			*reg = data;
733 			// vlapic_dump_lvt(offset, reg);
734 			break;
735 		case APIC_OFFSET_ICR:
736 			lapic->icr_timer = data;
737 			vlapic_start_timer(vlapic, 0);
738 			break;
739 
740 		case APIC_OFFSET_DCR:
741 			lapic->dcr_timer = data;
742 			vlapic->divisor = vlapic_timer_divisor(data);
743 			break;
744 
745 		case APIC_OFFSET_ESR:
746 			vlapic_update_errors(vlapic);
747 			break;
748 		case APIC_OFFSET_VER:
749 		case APIC_OFFSET_APR:
750 		case APIC_OFFSET_PPR:
751 		case APIC_OFFSET_RRR:
752 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
753 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
754 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
755 		case APIC_OFFSET_CCR:
756 		default:
757 			// Read only.
758 			break;
759 	}
760 
761 	return (retval);
762 }
763 
764 int
765 vlapic_timer_tick(struct vlapic *vlapic)
766 {
767 	int curticks, delta, periodic, fired;
768 	uint32_t ccr;
769 	uint32_t decrement, leftover;
770 
771 restart:
772 	curticks = ticks;
773 	delta = curticks - vlapic->ccr_ticks;
774 
775 	/* Local APIC timer is disabled */
776 	if (vlapic->apic.icr_timer == 0)
777 		return (-1);
778 
779 	/* One-shot mode and timer has already counted down to zero */
780 	periodic = vlapic_periodic_timer(vlapic);
781 	if (!periodic && vlapic->apic.ccr_timer == 0)
782 		return (-1);
783 	/*
784 	 * The 'curticks' and 'ccr_ticks' are out of sync by more than
785 	 * 2^31 ticks. We deal with this by restarting the timer.
786 	 */
787 	if (delta < 0) {
788 		vlapic_start_timer(vlapic, 0);
789 		goto restart;
790 	}
791 
792 	fired = 0;
793 	decrement = (VLAPIC_BUS_FREQ / vlapic->divisor) / hz;
794 
795 	vlapic->ccr_ticks = curticks;
796 	ccr = vlapic->apic.ccr_timer;
797 
798 	while (delta-- > 0) {
799 		if (ccr > decrement) {
800 			ccr -= decrement;
801 			continue;
802 		}
803 
804 		/* Trigger the local apic timer interrupt */
805 		vlapic_fire_timer(vlapic);
806 		if (periodic) {
807 			leftover = decrement - ccr;
808 			vlapic_start_timer(vlapic, leftover);
809 			ccr = vlapic->apic.ccr_timer;
810 		} else {
811 			/*
812 			 * One-shot timer has counted down to zero.
813 			 */
814 			ccr = 0;
815 		}
816 		fired = 1;
817 		break;
818 	}
819 
820 	vlapic->apic.ccr_timer = ccr;
821 
822 	if (!fired)
823 		return ((ccr / decrement) + 1);
824 	else
825 		return (0);
826 }
827 
828 struct vdev_ops vlapic_dev_ops = {
829 	.name = "vlapic",
830 	.init = vlapic_op_init,
831 	.reset = vlapic_op_reset,
832 	.halt = vlapic_op_halt,
833 	.memread = vlapic_op_mem_read,
834 	.memwrite = vlapic_op_mem_write,
835 };
836 static struct io_region vlapic_mmio[VM_MAXCPU];
837 
838 struct vlapic *
839 vlapic_init(struct vm *vm, int vcpuid)
840 {
841 	struct vlapic 		*vlapic;
842 
843 	vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
844 	vlapic->vm = vm;
845 	vlapic->vcpuid = vcpuid;
846 
847 	vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
848 
849 	if (vcpuid == 0)
850 		vlapic->msr_apicbase |= APICBASE_BSP;
851 
852 	vlapic->ops = &vlapic_dev_ops;
853 
854 	vlapic->mmio = vlapic_mmio + vcpuid;
855 	vlapic->mmio->base = DEFAULT_APIC_BASE;
856 	vlapic->mmio->len = PAGE_SIZE;
857 	vlapic->mmio->attr = MMIO_READ|MMIO_WRITE;
858 	vlapic->mmio->vcpu = vcpuid;
859 
860 	vdev_register(&vlapic_dev_ops, vlapic);
861 
862 	vlapic_op_init(vlapic);
863 
864 	return (vlapic);
865 }
866 
867 void
868 vlapic_cleanup(struct vlapic *vlapic)
869 {
870 	vlapic_op_halt(vlapic);
871 	vdev_unregister(vlapic);
872 	free(vlapic, M_VLAPIC);
873 }
874 
875 uint64_t
876 vlapic_get_apicbase(struct vlapic *vlapic)
877 {
878 
879 	return (vlapic->msr_apicbase);
880 }
881 
882 void
883 vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
884 {
885 	int err;
886 	enum x2apic_state state;
887 
888 	err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state);
889 	if (err)
890 		panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
891 
892 	if (state == X2APIC_DISABLED)
893 		val &= ~APICBASE_X2APIC;
894 
895 	vlapic->msr_apicbase = val;
896 }
897 
898 void
899 vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
900 {
901 	struct vlapic *vlapic;
902 
903 	vlapic = vm_lapic(vm, vcpuid);
904 
905 	if (state == X2APIC_DISABLED)
906 		vlapic->msr_apicbase &= ~APICBASE_X2APIC;
907 }
908