xref: /freebsd/sys/amd64/vmm/io/vioapic.c (revision 22cf89c938886d14f5796fc49f9f020c23ea8eaf) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5  * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 #include "opt_bhyve_snapshot.h"
32 
33 #include <sys/param.h>
34 #include <sys/queue.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 
41 #include <x86/apicreg.h>
42 #include <machine/vmm.h>
43 #include <machine/vmm_snapshot.h>
44 
45 #include "vmm_ktr.h"
46 #include "vmm_lapic.h"
47 #include "vlapic.h"
48 #include "vioapic.h"
49 
50 #define	IOREGSEL	0x00
51 #define	IOWIN		0x10
52 
53 #define	REDIR_ENTRIES	32
54 #define	RTBL_RO_BITS	((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
55 
56 struct vioapic {
57 	struct vm	*vm;
58 	struct mtx	mtx;
59 	uint32_t	id;
60 	uint32_t	ioregsel;
61 	struct {
62 		uint64_t reg;
63 		int	 acnt;	/* sum of pin asserts (+1) and deasserts (-1) */
64 	} rtbl[REDIR_ENTRIES];
65 };
66 
67 #define	VIOAPIC_LOCK(vioapic)		mtx_lock_spin(&((vioapic)->mtx))
68 #define	VIOAPIC_UNLOCK(vioapic)		mtx_unlock_spin(&((vioapic)->mtx))
69 #define	VIOAPIC_LOCKED(vioapic)		mtx_owned(&((vioapic)->mtx))
70 
71 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
72 
73 #define	VIOAPIC_CTR1(vioapic, fmt, a1)					\
74 	VM_CTR1((vioapic)->vm, fmt, a1)
75 
76 #define	VIOAPIC_CTR2(vioapic, fmt, a1, a2)				\
77 	VM_CTR2((vioapic)->vm, fmt, a1, a2)
78 
79 #define	VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3)				\
80 	VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
81 
82 #define	VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4)			\
83 	VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
84 
85 #ifdef KTR
86 static const char *
87 pinstate_str(bool asserted)
88 {
89 
90 	if (asserted)
91 		return ("asserted");
92 	else
93 		return ("deasserted");
94 }
95 #endif
96 
97 static void
98 vioapic_send_intr(struct vioapic *vioapic, int pin)
99 {
100 	int vector, delmode;
101 	uint32_t low, high, dest;
102 	bool level, phys;
103 
104 	KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
105 	    ("vioapic_set_pinstate: invalid pin number %d", pin));
106 
107 	KASSERT(VIOAPIC_LOCKED(vioapic),
108 	    ("vioapic_set_pinstate: vioapic is not locked"));
109 
110 	low = vioapic->rtbl[pin].reg;
111 	high = vioapic->rtbl[pin].reg >> 32;
112 
113 	if ((low & IOART_INTMASK) == IOART_INTMSET) {
114 		VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
115 		return;
116 	}
117 
118 	phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
119 	delmode = low & IOART_DELMOD;
120 	level = low & IOART_TRGRLVL ? true : false;
121 	if (level) {
122 		if ((low & IOART_REM_IRR) != 0) {
123 			VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending",
124 			    pin);
125 			return;
126 		}
127 		vioapic->rtbl[pin].reg |= IOART_REM_IRR;
128 	}
129 
130 	vector = low & IOART_INTVEC;
131 	dest = high >> APIC_ID_SHIFT;
132 	vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
133 }
134 
135 static void
136 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
137 {
138 	int oldcnt, newcnt;
139 	bool needintr;
140 
141 	KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
142 	    ("vioapic_set_pinstate: invalid pin number %d", pin));
143 
144 	KASSERT(VIOAPIC_LOCKED(vioapic),
145 	    ("vioapic_set_pinstate: vioapic is not locked"));
146 
147 	oldcnt = vioapic->rtbl[pin].acnt;
148 	if (newstate)
149 		vioapic->rtbl[pin].acnt++;
150 	else
151 		vioapic->rtbl[pin].acnt--;
152 	newcnt = vioapic->rtbl[pin].acnt;
153 
154 	if (newcnt < 0) {
155 		VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
156 		    pin, newcnt);
157 	}
158 
159 	needintr = false;
160 	if (oldcnt == 0 && newcnt == 1) {
161 		needintr = true;
162 		VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
163 	} else if (oldcnt == 1 && newcnt == 0) {
164 		VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
165 	} else {
166 		VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
167 		    pin, pinstate_str(newstate), newcnt);
168 	}
169 
170 	if (needintr)
171 		vioapic_send_intr(vioapic, pin);
172 }
173 
174 enum irqstate {
175 	IRQSTATE_ASSERT,
176 	IRQSTATE_DEASSERT,
177 	IRQSTATE_PULSE
178 };
179 
180 static int
181 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
182 {
183 	struct vioapic *vioapic;
184 
185 	if (irq < 0 || irq >= REDIR_ENTRIES)
186 		return (EINVAL);
187 
188 	vioapic = vm_ioapic(vm);
189 
190 	VIOAPIC_LOCK(vioapic);
191 	switch (irqstate) {
192 	case IRQSTATE_ASSERT:
193 		vioapic_set_pinstate(vioapic, irq, true);
194 		break;
195 	case IRQSTATE_DEASSERT:
196 		vioapic_set_pinstate(vioapic, irq, false);
197 		break;
198 	case IRQSTATE_PULSE:
199 		vioapic_set_pinstate(vioapic, irq, true);
200 		vioapic_set_pinstate(vioapic, irq, false);
201 		break;
202 	default:
203 		panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
204 	}
205 	VIOAPIC_UNLOCK(vioapic);
206 
207 	return (0);
208 }
209 
210 int
211 vioapic_assert_irq(struct vm *vm, int irq)
212 {
213 
214 	return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
215 }
216 
217 int
218 vioapic_deassert_irq(struct vm *vm, int irq)
219 {
220 
221 	return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
222 }
223 
224 int
225 vioapic_pulse_irq(struct vm *vm, int irq)
226 {
227 
228 	return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
229 }
230 
231 /*
232  * Reset the vlapic's trigger-mode register to reflect the ioapic pin
233  * configuration.
234  */
235 static void
236 vioapic_update_tmr(struct vcpu *vcpu, void *arg)
237 {
238 	struct vioapic *vioapic;
239 	struct vlapic *vlapic;
240 	uint32_t low, high, dest;
241 	int delmode, pin, vector;
242 	bool level, phys;
243 
244 	vlapic = vm_lapic(vcpu);
245 	vioapic = vm_ioapic(vcpu_vm(vcpu));
246 
247 	VIOAPIC_LOCK(vioapic);
248 	/*
249 	 * Reset all vectors to be edge-triggered.
250 	 */
251 	vlapic_reset_tmr(vlapic);
252 	for (pin = 0; pin < REDIR_ENTRIES; pin++) {
253 		low = vioapic->rtbl[pin].reg;
254 		high = vioapic->rtbl[pin].reg >> 32;
255 
256 		level = low & IOART_TRGRLVL ? true : false;
257 		if (!level)
258 			continue;
259 
260 		/*
261 		 * For a level-triggered 'pin' let the vlapic figure out if
262 		 * an assertion on this 'pin' would result in an interrupt
263 		 * being delivered to it. If yes, then it will modify the
264 		 * TMR bit associated with this vector to level-triggered.
265 		 */
266 		phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
267 		delmode = low & IOART_DELMOD;
268 		vector = low & IOART_INTVEC;
269 		dest = high >> APIC_ID_SHIFT;
270 		vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
271 	}
272 	VIOAPIC_UNLOCK(vioapic);
273 }
274 
275 static uint32_t
276 vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr)
277 {
278 	int regnum, pin, rshift;
279 
280 	regnum = addr & 0xff;
281 	switch (regnum) {
282 	case IOAPIC_ID:
283 		return (vioapic->id);
284 		break;
285 	case IOAPIC_VER:
286 		return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
287 		break;
288 	case IOAPIC_ARB:
289 		return (vioapic->id);
290 		break;
291 	default:
292 		break;
293 	}
294 
295 	/* redirection table entries */
296 	if (regnum >= IOAPIC_REDTBL &&
297 	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
298 		pin = (regnum - IOAPIC_REDTBL) / 2;
299 		if ((regnum - IOAPIC_REDTBL) % 2)
300 			rshift = 32;
301 		else
302 			rshift = 0;
303 
304 		return (vioapic->rtbl[pin].reg >> rshift);
305 	}
306 
307 	return (0);
308 }
309 
310 static void
311 vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr,
312     uint32_t data)
313 {
314 	uint64_t data64, mask64;
315 	uint64_t last, changed;
316 	int regnum, pin, lshift;
317 	cpuset_t allvcpus;
318 
319 	regnum = addr & 0xff;
320 	switch (regnum) {
321 	case IOAPIC_ID:
322 		vioapic->id = data & APIC_ID_MASK;
323 		break;
324 	case IOAPIC_VER:
325 	case IOAPIC_ARB:
326 		/* readonly */
327 		break;
328 	default:
329 		break;
330 	}
331 
332 	/* redirection table entries */
333 	if (regnum >= IOAPIC_REDTBL &&
334 	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
335 		pin = (regnum - IOAPIC_REDTBL) / 2;
336 		if ((regnum - IOAPIC_REDTBL) % 2)
337 			lshift = 32;
338 		else
339 			lshift = 0;
340 
341 		last = vioapic->rtbl[pin].reg;
342 
343 		data64 = (uint64_t)data << lshift;
344 		mask64 = (uint64_t)0xffffffff << lshift;
345 		vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
346 		vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
347 
348 		/*
349 		 * Switching from level to edge triggering will clear the IRR
350 		 * bit. This is what FreeBSD will do in order to EOI an
351 		 * interrupt when the IO-APIC doesn't support targeted EOI (see
352 		 * _ioapic_eoi_source).
353 		 */
354 		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG &&
355 		    (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0)
356 			vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
357 
358 		VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
359 		    pin, vioapic->rtbl[pin].reg);
360 
361 		/*
362 		 * If any fields in the redirection table entry (except mask
363 		 * or polarity) have changed then rendezvous all the vcpus
364 		 * to update their vlapic trigger-mode registers.
365 		 */
366 		changed = last ^ vioapic->rtbl[pin].reg;
367 		if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
368 			VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
369 			    "vlapic trigger-mode register", pin);
370 			VIOAPIC_UNLOCK(vioapic);
371 			allvcpus = vm_active_cpus(vioapic->vm);
372 			(void)vm_smp_rendezvous(vcpu, allvcpus,
373 			    vioapic_update_tmr, NULL);
374 			VIOAPIC_LOCK(vioapic);
375 		}
376 
377 		/*
378 		 * Generate an interrupt if the following conditions are met:
379 		 * - pin trigger mode is level
380 		 * - pin level is asserted
381 		 */
382 		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL &&
383 		    (vioapic->rtbl[pin].acnt > 0)) {
384 			VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
385 			    "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
386 			vioapic_send_intr(vioapic, pin);
387 		}
388 	}
389 }
390 
391 static int
392 vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa,
393     uint64_t *data, int size, bool doread)
394 {
395 	uint64_t offset;
396 
397 	offset = gpa - VIOAPIC_BASE;
398 
399 	/*
400 	 * The IOAPIC specification allows 32-bit wide accesses to the
401 	 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
402 	 */
403 	if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
404 		if (doread)
405 			*data = 0;
406 		return (0);
407 	}
408 
409 	VIOAPIC_LOCK(vioapic);
410 	if (offset == IOREGSEL) {
411 		if (doread)
412 			*data = vioapic->ioregsel;
413 		else
414 			vioapic->ioregsel = *data;
415 	} else {
416 		if (doread) {
417 			*data = vioapic_read(vioapic, vcpu,
418 			    vioapic->ioregsel);
419 		} else {
420 			vioapic_write(vioapic, vcpu, vioapic->ioregsel,
421 			    *data);
422 		}
423 	}
424 	VIOAPIC_UNLOCK(vioapic);
425 
426 	return (0);
427 }
428 
429 int
430 vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval,
431     int size, void *arg)
432 {
433 	int error;
434 	struct vioapic *vioapic;
435 
436 	vioapic = vm_ioapic(vcpu_vm(vcpu));
437 	error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true);
438 	return (error);
439 }
440 
441 int
442 vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval,
443     int size, void *arg)
444 {
445 	int error;
446 	struct vioapic *vioapic;
447 
448 	vioapic = vm_ioapic(vcpu_vm(vcpu));
449 	error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false);
450 	return (error);
451 }
452 
453 void
454 vioapic_process_eoi(struct vm *vm, int vector)
455 {
456 	struct vioapic *vioapic;
457 	int pin;
458 
459 	KASSERT(vector >= 0 && vector < 256,
460 	    ("vioapic_process_eoi: invalid vector %d", vector));
461 
462 	vioapic = vm_ioapic(vm);
463 	VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
464 
465 	/*
466 	 * XXX keep track of the pins associated with this vector instead
467 	 * of iterating on every single pin each time.
468 	 */
469 	VIOAPIC_LOCK(vioapic);
470 	for (pin = 0; pin < REDIR_ENTRIES; pin++) {
471 		if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
472 			continue;
473 		if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
474 			continue;
475 		vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
476 		if (vioapic->rtbl[pin].acnt > 0) {
477 			VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
478 			    "acnt %d", pin, vioapic->rtbl[pin].acnt);
479 			vioapic_send_intr(vioapic, pin);
480 		}
481 	}
482 	VIOAPIC_UNLOCK(vioapic);
483 }
484 
485 struct vioapic *
486 vioapic_init(struct vm *vm)
487 {
488 	int i;
489 	struct vioapic *vioapic;
490 
491 	vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
492 
493 	vioapic->vm = vm;
494 	mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
495 
496 	/* Initialize all redirection entries to mask all interrupts */
497 	for (i = 0; i < REDIR_ENTRIES; i++)
498 		vioapic->rtbl[i].reg = 0x0001000000010000UL;
499 
500 	return (vioapic);
501 }
502 
503 void
504 vioapic_cleanup(struct vioapic *vioapic)
505 {
506 
507 	mtx_destroy(&vioapic->mtx);
508 	free(vioapic, M_VIOAPIC);
509 }
510 
511 int
512 vioapic_pincount(struct vm *vm)
513 {
514 
515 	return (REDIR_ENTRIES);
516 }
517 
518 #ifdef BHYVE_SNAPSHOT
519 int
520 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta)
521 {
522 	int ret;
523 	int i;
524 
525 	SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done);
526 
527 	for (i = 0; i < nitems(vioapic->rtbl); i++) {
528 		SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done);
529 		SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done);
530 	}
531 
532 done:
533 	return (ret);
534 }
535 #endif
536