xref: /freebsd/sys/amd64/vmm/io/vioapic.c (revision 6be3386466ab79a84b48429ae66244f21526d3df) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5  * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34 
35 #include "opt_bhyve_snapshot.h"
36 
37 #include <sys/param.h>
38 #include <sys/queue.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/systm.h>
42 #include <sys/kernel.h>
43 #include <sys/malloc.h>
44 
45 #include <x86/apicreg.h>
46 #include <machine/vmm.h>
47 #include <machine/vmm_snapshot.h>
48 
49 #include "vmm_ktr.h"
50 #include "vmm_lapic.h"
51 #include "vlapic.h"
52 #include "vioapic.h"
53 
54 #define	IOREGSEL	0x00
55 #define	IOWIN		0x10
56 
57 #define	REDIR_ENTRIES	32
58 #define	RTBL_RO_BITS	((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
59 
60 struct vioapic {
61 	struct vm	*vm;
62 	struct mtx	mtx;
63 	uint32_t	id;
64 	uint32_t	ioregsel;
65 	struct {
66 		uint64_t reg;
67 		int	 acnt;	/* sum of pin asserts (+1) and deasserts (-1) */
68 	} rtbl[REDIR_ENTRIES];
69 };
70 
71 #define	VIOAPIC_LOCK(vioapic)		mtx_lock_spin(&((vioapic)->mtx))
72 #define	VIOAPIC_UNLOCK(vioapic)		mtx_unlock_spin(&((vioapic)->mtx))
73 #define	VIOAPIC_LOCKED(vioapic)		mtx_owned(&((vioapic)->mtx))
74 
75 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
76 
77 #define	VIOAPIC_CTR1(vioapic, fmt, a1)					\
78 	VM_CTR1((vioapic)->vm, fmt, a1)
79 
80 #define	VIOAPIC_CTR2(vioapic, fmt, a1, a2)				\
81 	VM_CTR2((vioapic)->vm, fmt, a1, a2)
82 
83 #define	VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3)				\
84 	VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
85 
86 #define	VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4)			\
87 	VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
88 
89 #ifdef KTR
90 static const char *
91 pinstate_str(bool asserted)
92 {
93 
94 	if (asserted)
95 		return ("asserted");
96 	else
97 		return ("deasserted");
98 }
99 #endif
100 
101 static void
102 vioapic_send_intr(struct vioapic *vioapic, int pin)
103 {
104 	int vector, delmode;
105 	uint32_t low, high, dest;
106 	bool level, phys;
107 
108 	KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
109 	    ("vioapic_set_pinstate: invalid pin number %d", pin));
110 
111 	KASSERT(VIOAPIC_LOCKED(vioapic),
112 	    ("vioapic_set_pinstate: vioapic is not locked"));
113 
114 	low = vioapic->rtbl[pin].reg;
115 	high = vioapic->rtbl[pin].reg >> 32;
116 
117 	if ((low & IOART_INTMASK) == IOART_INTMSET) {
118 		VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
119 		return;
120 	}
121 
122 	phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
123 	delmode = low & IOART_DELMOD;
124 	level = low & IOART_TRGRLVL ? true : false;
125 	if (level) {
126 		if ((low & IOART_REM_IRR) != 0) {
127 			VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending",
128 			    pin);
129 			return;
130 		}
131 		vioapic->rtbl[pin].reg |= IOART_REM_IRR;
132 	}
133 
134 	vector = low & IOART_INTVEC;
135 	dest = high >> APIC_ID_SHIFT;
136 	vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
137 }
138 
139 static void
140 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
141 {
142 	int oldcnt, newcnt;
143 	bool needintr;
144 
145 	KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
146 	    ("vioapic_set_pinstate: invalid pin number %d", pin));
147 
148 	KASSERT(VIOAPIC_LOCKED(vioapic),
149 	    ("vioapic_set_pinstate: vioapic is not locked"));
150 
151 	oldcnt = vioapic->rtbl[pin].acnt;
152 	if (newstate)
153 		vioapic->rtbl[pin].acnt++;
154 	else
155 		vioapic->rtbl[pin].acnt--;
156 	newcnt = vioapic->rtbl[pin].acnt;
157 
158 	if (newcnt < 0) {
159 		VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
160 		    pin, newcnt);
161 	}
162 
163 	needintr = false;
164 	if (oldcnt == 0 && newcnt == 1) {
165 		needintr = true;
166 		VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
167 	} else if (oldcnt == 1 && newcnt == 0) {
168 		VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
169 	} else {
170 		VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
171 		    pin, pinstate_str(newstate), newcnt);
172 	}
173 
174 	if (needintr)
175 		vioapic_send_intr(vioapic, pin);
176 }
177 
178 enum irqstate {
179 	IRQSTATE_ASSERT,
180 	IRQSTATE_DEASSERT,
181 	IRQSTATE_PULSE
182 };
183 
184 static int
185 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
186 {
187 	struct vioapic *vioapic;
188 
189 	if (irq < 0 || irq >= REDIR_ENTRIES)
190 		return (EINVAL);
191 
192 	vioapic = vm_ioapic(vm);
193 
194 	VIOAPIC_LOCK(vioapic);
195 	switch (irqstate) {
196 	case IRQSTATE_ASSERT:
197 		vioapic_set_pinstate(vioapic, irq, true);
198 		break;
199 	case IRQSTATE_DEASSERT:
200 		vioapic_set_pinstate(vioapic, irq, false);
201 		break;
202 	case IRQSTATE_PULSE:
203 		vioapic_set_pinstate(vioapic, irq, true);
204 		vioapic_set_pinstate(vioapic, irq, false);
205 		break;
206 	default:
207 		panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
208 	}
209 	VIOAPIC_UNLOCK(vioapic);
210 
211 	return (0);
212 }
213 
214 int
215 vioapic_assert_irq(struct vm *vm, int irq)
216 {
217 
218 	return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
219 }
220 
221 int
222 vioapic_deassert_irq(struct vm *vm, int irq)
223 {
224 
225 	return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
226 }
227 
228 int
229 vioapic_pulse_irq(struct vm *vm, int irq)
230 {
231 
232 	return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
233 }
234 
235 /*
236  * Reset the vlapic's trigger-mode register to reflect the ioapic pin
237  * configuration.
238  */
239 static void
240 vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg)
241 {
242 	struct vioapic *vioapic;
243 	struct vlapic *vlapic;
244 	uint32_t low, high, dest;
245 	int delmode, pin, vector;
246 	bool level, phys;
247 
248 	vlapic = vm_lapic(vm, vcpuid);
249 	vioapic = vm_ioapic(vm);
250 
251 	VIOAPIC_LOCK(vioapic);
252 	/*
253 	 * Reset all vectors to be edge-triggered.
254 	 */
255 	vlapic_reset_tmr(vlapic);
256 	for (pin = 0; pin < REDIR_ENTRIES; pin++) {
257 		low = vioapic->rtbl[pin].reg;
258 		high = vioapic->rtbl[pin].reg >> 32;
259 
260 		level = low & IOART_TRGRLVL ? true : false;
261 		if (!level)
262 			continue;
263 
264 		/*
265 		 * For a level-triggered 'pin' let the vlapic figure out if
266 		 * an assertion on this 'pin' would result in an interrupt
267 		 * being delivered to it. If yes, then it will modify the
268 		 * TMR bit associated with this vector to level-triggered.
269 		 */
270 		phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
271 		delmode = low & IOART_DELMOD;
272 		vector = low & IOART_INTVEC;
273 		dest = high >> APIC_ID_SHIFT;
274 		vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
275 	}
276 	VIOAPIC_UNLOCK(vioapic);
277 }
278 
279 static uint32_t
280 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr)
281 {
282 	int regnum, pin, rshift;
283 
284 	regnum = addr & 0xff;
285 	switch (regnum) {
286 	case IOAPIC_ID:
287 		return (vioapic->id);
288 		break;
289 	case IOAPIC_VER:
290 		return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
291 		break;
292 	case IOAPIC_ARB:
293 		return (vioapic->id);
294 		break;
295 	default:
296 		break;
297 	}
298 
299 	/* redirection table entries */
300 	if (regnum >= IOAPIC_REDTBL &&
301 	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
302 		pin = (regnum - IOAPIC_REDTBL) / 2;
303 		if ((regnum - IOAPIC_REDTBL) % 2)
304 			rshift = 32;
305 		else
306 			rshift = 0;
307 
308 		return (vioapic->rtbl[pin].reg >> rshift);
309 	}
310 
311 	return (0);
312 }
313 
314 static void
315 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data)
316 {
317 	uint64_t data64, mask64;
318 	uint64_t last, changed;
319 	int regnum, pin, lshift;
320 	cpuset_t allvcpus;
321 
322 	regnum = addr & 0xff;
323 	switch (regnum) {
324 	case IOAPIC_ID:
325 		vioapic->id = data & APIC_ID_MASK;
326 		break;
327 	case IOAPIC_VER:
328 	case IOAPIC_ARB:
329 		/* readonly */
330 		break;
331 	default:
332 		break;
333 	}
334 
335 	/* redirection table entries */
336 	if (regnum >= IOAPIC_REDTBL &&
337 	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
338 		pin = (regnum - IOAPIC_REDTBL) / 2;
339 		if ((regnum - IOAPIC_REDTBL) % 2)
340 			lshift = 32;
341 		else
342 			lshift = 0;
343 
344 		last = vioapic->rtbl[pin].reg;
345 
346 		data64 = (uint64_t)data << lshift;
347 		mask64 = (uint64_t)0xffffffff << lshift;
348 		vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
349 		vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
350 
351 		/*
352 		 * Switching from level to edge triggering will clear the IRR
353 		 * bit. This is what FreeBSD will do in order to EOI an
354 		 * interrupt when the IO-APIC doesn't support targeted EOI (see
355 		 * _ioapic_eoi_source).
356 		 */
357 		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG &&
358 		    (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0)
359 			vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
360 
361 		VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
362 		    pin, vioapic->rtbl[pin].reg);
363 
364 		/*
365 		 * If any fields in the redirection table entry (except mask
366 		 * or polarity) have changed then rendezvous all the vcpus
367 		 * to update their vlapic trigger-mode registers.
368 		 */
369 		changed = last ^ vioapic->rtbl[pin].reg;
370 		if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
371 			VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
372 			    "vlapic trigger-mode register", pin);
373 			VIOAPIC_UNLOCK(vioapic);
374 			allvcpus = vm_active_cpus(vioapic->vm);
375 			(void)vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus,
376 			    vioapic_update_tmr, NULL);
377 			VIOAPIC_LOCK(vioapic);
378 		}
379 
380 		/*
381 		 * Generate an interrupt if the following conditions are met:
382 		 * - pin trigger mode is level
383 		 * - pin level is asserted
384 		 */
385 		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL &&
386 		    (vioapic->rtbl[pin].acnt > 0)) {
387 			VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
388 			    "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
389 			vioapic_send_intr(vioapic, pin);
390 		}
391 	}
392 }
393 
394 static int
395 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa,
396     uint64_t *data, int size, bool doread)
397 {
398 	uint64_t offset;
399 
400 	offset = gpa - VIOAPIC_BASE;
401 
402 	/*
403 	 * The IOAPIC specification allows 32-bit wide accesses to the
404 	 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
405 	 */
406 	if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
407 		if (doread)
408 			*data = 0;
409 		return (0);
410 	}
411 
412 	VIOAPIC_LOCK(vioapic);
413 	if (offset == IOREGSEL) {
414 		if (doread)
415 			*data = vioapic->ioregsel;
416 		else
417 			vioapic->ioregsel = *data;
418 	} else {
419 		if (doread) {
420 			*data = vioapic_read(vioapic, vcpuid,
421 			    vioapic->ioregsel);
422 		} else {
423 			vioapic_write(vioapic, vcpuid, vioapic->ioregsel,
424 			    *data);
425 		}
426 	}
427 	VIOAPIC_UNLOCK(vioapic);
428 
429 	return (0);
430 }
431 
432 int
433 vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval,
434     int size, void *arg)
435 {
436 	int error;
437 	struct vioapic *vioapic;
438 
439 	vioapic = vm_ioapic(vm);
440 	error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true);
441 	return (error);
442 }
443 
444 int
445 vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval,
446     int size, void *arg)
447 {
448 	int error;
449 	struct vioapic *vioapic;
450 
451 	vioapic = vm_ioapic(vm);
452 	error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false);
453 	return (error);
454 }
455 
456 void
457 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector)
458 {
459 	struct vioapic *vioapic;
460 	int pin;
461 
462 	KASSERT(vector >= 0 && vector < 256,
463 	    ("vioapic_process_eoi: invalid vector %d", vector));
464 
465 	vioapic = vm_ioapic(vm);
466 	VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
467 
468 	/*
469 	 * XXX keep track of the pins associated with this vector instead
470 	 * of iterating on every single pin each time.
471 	 */
472 	VIOAPIC_LOCK(vioapic);
473 	for (pin = 0; pin < REDIR_ENTRIES; pin++) {
474 		if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
475 			continue;
476 		if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
477 			continue;
478 		vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
479 		if (vioapic->rtbl[pin].acnt > 0) {
480 			VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
481 			    "acnt %d", pin, vioapic->rtbl[pin].acnt);
482 			vioapic_send_intr(vioapic, pin);
483 		}
484 	}
485 	VIOAPIC_UNLOCK(vioapic);
486 }
487 
488 struct vioapic *
489 vioapic_init(struct vm *vm)
490 {
491 	int i;
492 	struct vioapic *vioapic;
493 
494 	vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
495 
496 	vioapic->vm = vm;
497 	mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
498 
499 	/* Initialize all redirection entries to mask all interrupts */
500 	for (i = 0; i < REDIR_ENTRIES; i++)
501 		vioapic->rtbl[i].reg = 0x0001000000010000UL;
502 
503 	return (vioapic);
504 }
505 
506 void
507 vioapic_cleanup(struct vioapic *vioapic)
508 {
509 
510 	free(vioapic, M_VIOAPIC);
511 }
512 
513 int
514 vioapic_pincount(struct vm *vm)
515 {
516 
517 	return (REDIR_ENTRIES);
518 }
519 
520 #ifdef BHYVE_SNAPSHOT
521 int
522 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta)
523 {
524 	int ret;
525 	int i;
526 
527 	SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done);
528 
529 	for (i = 0; i < nitems(vioapic->rtbl); i++) {
530 		SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done);
531 		SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done);
532 	}
533 
534 done:
535 	return (ret);
536 }
537 #endif
538