1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #include "opt_bhyve_snapshot.h"
32
33 #include <sys/param.h>
34 #include <sys/queue.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40
41 #include <x86/apicreg.h>
42 #include <machine/vmm.h>
43 #include <machine/vmm_snapshot.h>
44
45 #include <dev/vmm/vmm_ktr.h>
46
47 #include "vmm_lapic.h"
48 #include "vlapic.h"
49 #include "vioapic.h"
50
51 #define IOREGSEL 0x00
52 #define IOWIN 0x10
53
54 #define REDIR_ENTRIES 32
55 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
56
57 struct vioapic {
58 struct vm *vm;
59 struct mtx mtx;
60 uint32_t id;
61 uint32_t ioregsel;
62 struct {
63 uint64_t reg;
64 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */
65 } rtbl[REDIR_ENTRIES];
66 };
67
68 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx))
69 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx))
70 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx))
71
72 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
73
74 #define VIOAPIC_CTR1(vioapic, fmt, a1) \
75 VM_CTR1((vioapic)->vm, fmt, a1)
76
77 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \
78 VM_CTR2((vioapic)->vm, fmt, a1, a2)
79
80 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \
81 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
82
83 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \
84 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
85
86 #ifdef KTR
87 static const char *
pinstate_str(bool asserted)88 pinstate_str(bool asserted)
89 {
90
91 if (asserted)
92 return ("asserted");
93 else
94 return ("deasserted");
95 }
96 #endif
97
98 static void
vioapic_send_intr(struct vioapic * vioapic,int pin)99 vioapic_send_intr(struct vioapic *vioapic, int pin)
100 {
101 int vector, delmode;
102 uint32_t low, high, dest;
103 bool level, phys;
104
105 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
106 ("vioapic_set_pinstate: invalid pin number %d", pin));
107
108 KASSERT(VIOAPIC_LOCKED(vioapic),
109 ("vioapic_set_pinstate: vioapic is not locked"));
110
111 low = vioapic->rtbl[pin].reg;
112 high = vioapic->rtbl[pin].reg >> 32;
113
114 if ((low & IOART_INTMASK) == IOART_INTMSET) {
115 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
116 return;
117 }
118
119 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
120 delmode = low & IOART_DELMOD;
121 level = low & IOART_TRGRLVL ? true : false;
122 if (level) {
123 if ((low & IOART_REM_IRR) != 0) {
124 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending",
125 pin);
126 return;
127 }
128 vioapic->rtbl[pin].reg |= IOART_REM_IRR;
129 }
130
131 vector = low & IOART_INTVEC;
132 dest = high >> APIC_ID_SHIFT;
133 /*
134 * Ideally we'd just call lapic_intr_msi() here with the
135 * constructed MSI instead of interpreting it for ourselves.
136 * But until/unless we support emulated IOMMUs with interrupt
137 * remapping, interpretation is simple. We just need to mask
138 * in the Extended Destination ID bits for the 15-bit
139 * enlightenment (http://david.woodhou.se/ExtDestId.pdf)
140 */
141 dest |= ((high & APIC_EXT_ID_MASK) >> APIC_EXT_ID_SHIFT) << 8;
142 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
143 }
144
145 static void
vioapic_set_pinstate(struct vioapic * vioapic,int pin,bool newstate)146 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
147 {
148 int oldcnt, newcnt;
149 bool needintr;
150
151 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
152 ("vioapic_set_pinstate: invalid pin number %d", pin));
153
154 KASSERT(VIOAPIC_LOCKED(vioapic),
155 ("vioapic_set_pinstate: vioapic is not locked"));
156
157 oldcnt = vioapic->rtbl[pin].acnt;
158 if (newstate)
159 vioapic->rtbl[pin].acnt++;
160 else
161 vioapic->rtbl[pin].acnt--;
162 newcnt = vioapic->rtbl[pin].acnt;
163
164 if (newcnt < 0) {
165 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
166 pin, newcnt);
167 }
168
169 needintr = false;
170 if (oldcnt == 0 && newcnt == 1) {
171 needintr = true;
172 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
173 } else if (oldcnt == 1 && newcnt == 0) {
174 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
175 } else {
176 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
177 pin, pinstate_str(newstate), newcnt);
178 }
179
180 if (needintr)
181 vioapic_send_intr(vioapic, pin);
182 }
183
184 enum irqstate {
185 IRQSTATE_ASSERT,
186 IRQSTATE_DEASSERT,
187 IRQSTATE_PULSE
188 };
189
190 static int
vioapic_set_irqstate(struct vm * vm,int irq,enum irqstate irqstate)191 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
192 {
193 struct vioapic *vioapic;
194
195 if (irq < 0 || irq >= REDIR_ENTRIES)
196 return (EINVAL);
197
198 vioapic = vm_ioapic(vm);
199
200 VIOAPIC_LOCK(vioapic);
201 switch (irqstate) {
202 case IRQSTATE_ASSERT:
203 vioapic_set_pinstate(vioapic, irq, true);
204 break;
205 case IRQSTATE_DEASSERT:
206 vioapic_set_pinstate(vioapic, irq, false);
207 break;
208 case IRQSTATE_PULSE:
209 vioapic_set_pinstate(vioapic, irq, true);
210 vioapic_set_pinstate(vioapic, irq, false);
211 break;
212 default:
213 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
214 }
215 VIOAPIC_UNLOCK(vioapic);
216
217 return (0);
218 }
219
220 int
vioapic_assert_irq(struct vm * vm,int irq)221 vioapic_assert_irq(struct vm *vm, int irq)
222 {
223
224 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
225 }
226
227 int
vioapic_deassert_irq(struct vm * vm,int irq)228 vioapic_deassert_irq(struct vm *vm, int irq)
229 {
230
231 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
232 }
233
234 int
vioapic_pulse_irq(struct vm * vm,int irq)235 vioapic_pulse_irq(struct vm *vm, int irq)
236 {
237
238 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
239 }
240
241 /*
242 * Reset the vlapic's trigger-mode register to reflect the ioapic pin
243 * configuration.
244 */
245 static void
vioapic_update_tmr(struct vcpu * vcpu,void * arg)246 vioapic_update_tmr(struct vcpu *vcpu, void *arg)
247 {
248 struct vioapic *vioapic;
249 struct vlapic *vlapic;
250 uint32_t low, high, dest;
251 int delmode, pin, vector;
252 bool level, phys;
253
254 vlapic = vm_lapic(vcpu);
255 vioapic = vm_ioapic(vcpu_vm(vcpu));
256
257 VIOAPIC_LOCK(vioapic);
258 /*
259 * Reset all vectors to be edge-triggered.
260 */
261 vlapic_reset_tmr(vlapic);
262 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
263 low = vioapic->rtbl[pin].reg;
264 high = vioapic->rtbl[pin].reg >> 32;
265
266 level = low & IOART_TRGRLVL ? true : false;
267 if (!level)
268 continue;
269
270 /*
271 * For a level-triggered 'pin' let the vlapic figure out if
272 * an assertion on this 'pin' would result in an interrupt
273 * being delivered to it. If yes, then it will modify the
274 * TMR bit associated with this vector to level-triggered.
275 */
276 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
277 delmode = low & IOART_DELMOD;
278 vector = low & IOART_INTVEC;
279 dest = high >> APIC_ID_SHIFT;
280 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
281 }
282 VIOAPIC_UNLOCK(vioapic);
283 }
284
285 static uint32_t
vioapic_read(struct vioapic * vioapic,struct vcpu * vcpu,uint32_t addr)286 vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr)
287 {
288 int regnum, pin, rshift;
289
290 regnum = addr & 0xff;
291 switch (regnum) {
292 case IOAPIC_ID:
293 return (vioapic->id);
294 break;
295 case IOAPIC_VER:
296 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
297 break;
298 case IOAPIC_ARB:
299 return (vioapic->id);
300 break;
301 default:
302 break;
303 }
304
305 /* redirection table entries */
306 if (regnum >= IOAPIC_REDTBL &&
307 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
308 pin = (regnum - IOAPIC_REDTBL) / 2;
309 if ((regnum - IOAPIC_REDTBL) % 2)
310 rshift = 32;
311 else
312 rshift = 0;
313
314 return (vioapic->rtbl[pin].reg >> rshift);
315 }
316
317 return (0);
318 }
319
320 static void
vioapic_write(struct vioapic * vioapic,struct vcpu * vcpu,uint32_t addr,uint32_t data)321 vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr,
322 uint32_t data)
323 {
324 uint64_t data64, mask64;
325 uint64_t last, changed;
326 int regnum, pin, lshift;
327 cpuset_t allvcpus;
328
329 regnum = addr & 0xff;
330 switch (regnum) {
331 case IOAPIC_ID:
332 vioapic->id = data & APIC_ID_MASK;
333 break;
334 case IOAPIC_VER:
335 case IOAPIC_ARB:
336 /* readonly */
337 break;
338 default:
339 break;
340 }
341
342 /* redirection table entries */
343 if (regnum >= IOAPIC_REDTBL &&
344 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
345 pin = (regnum - IOAPIC_REDTBL) / 2;
346 if ((regnum - IOAPIC_REDTBL) % 2)
347 lshift = 32;
348 else
349 lshift = 0;
350
351 last = vioapic->rtbl[pin].reg;
352
353 data64 = (uint64_t)data << lshift;
354 mask64 = (uint64_t)0xffffffff << lshift;
355 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
356 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
357
358 /*
359 * Switching from level to edge triggering will clear the IRR
360 * bit. This is what FreeBSD will do in order to EOI an
361 * interrupt when the IO-APIC doesn't support targeted EOI (see
362 * _ioapic_eoi_source).
363 */
364 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG &&
365 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0)
366 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
367
368 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
369 pin, vioapic->rtbl[pin].reg);
370
371 /*
372 * If any fields in the redirection table entry (except mask
373 * or polarity) have changed then rendezvous all the vcpus
374 * to update their vlapic trigger-mode registers.
375 */
376 changed = last ^ vioapic->rtbl[pin].reg;
377 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
378 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
379 "vlapic trigger-mode register", pin);
380 VIOAPIC_UNLOCK(vioapic);
381 allvcpus = vm_active_cpus(vioapic->vm);
382 (void)vm_smp_rendezvous(vcpu, allvcpus,
383 vioapic_update_tmr, NULL);
384 VIOAPIC_LOCK(vioapic);
385 }
386
387 /*
388 * Generate an interrupt if the following conditions are met:
389 * - pin trigger mode is level
390 * - pin level is asserted
391 */
392 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL &&
393 (vioapic->rtbl[pin].acnt > 0)) {
394 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
395 "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
396 vioapic_send_intr(vioapic, pin);
397 }
398 }
399 }
400
401 static int
vioapic_mmio_rw(struct vioapic * vioapic,struct vcpu * vcpu,uint64_t gpa,uint64_t * data,int size,bool doread)402 vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa,
403 uint64_t *data, int size, bool doread)
404 {
405 uint64_t offset;
406
407 offset = gpa - VIOAPIC_BASE;
408
409 /*
410 * The IOAPIC specification allows 32-bit wide accesses to the
411 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
412 */
413 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
414 if (doread)
415 *data = 0;
416 return (0);
417 }
418
419 VIOAPIC_LOCK(vioapic);
420 if (offset == IOREGSEL) {
421 if (doread)
422 *data = vioapic->ioregsel;
423 else
424 vioapic->ioregsel = *data;
425 } else {
426 if (doread) {
427 *data = vioapic_read(vioapic, vcpu,
428 vioapic->ioregsel);
429 } else {
430 vioapic_write(vioapic, vcpu, vioapic->ioregsel,
431 *data);
432 }
433 }
434 VIOAPIC_UNLOCK(vioapic);
435
436 return (0);
437 }
438
439 int
vioapic_mmio_read(struct vcpu * vcpu,uint64_t gpa,uint64_t * rval,int size,void * arg)440 vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval,
441 int size, void *arg)
442 {
443 int error;
444 struct vioapic *vioapic;
445
446 vioapic = vm_ioapic(vcpu_vm(vcpu));
447 error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true);
448 return (error);
449 }
450
451 int
vioapic_mmio_write(struct vcpu * vcpu,uint64_t gpa,uint64_t wval,int size,void * arg)452 vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval,
453 int size, void *arg)
454 {
455 int error;
456 struct vioapic *vioapic;
457
458 vioapic = vm_ioapic(vcpu_vm(vcpu));
459 error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false);
460 return (error);
461 }
462
463 void
vioapic_process_eoi(struct vm * vm,int vector)464 vioapic_process_eoi(struct vm *vm, int vector)
465 {
466 struct vioapic *vioapic;
467 int pin;
468
469 KASSERT(vector >= 0 && vector < 256,
470 ("vioapic_process_eoi: invalid vector %d", vector));
471
472 vioapic = vm_ioapic(vm);
473 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
474
475 /*
476 * XXX keep track of the pins associated with this vector instead
477 * of iterating on every single pin each time.
478 */
479 VIOAPIC_LOCK(vioapic);
480 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
481 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
482 continue;
483 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
484 continue;
485 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
486 if (vioapic->rtbl[pin].acnt > 0) {
487 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
488 "acnt %d", pin, vioapic->rtbl[pin].acnt);
489 vioapic_send_intr(vioapic, pin);
490 }
491 }
492 VIOAPIC_UNLOCK(vioapic);
493 }
494
495 struct vioapic *
vioapic_init(struct vm * vm)496 vioapic_init(struct vm *vm)
497 {
498 int i;
499 struct vioapic *vioapic;
500
501 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
502
503 vioapic->vm = vm;
504 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
505
506 /* Initialize all redirection entries to mask all interrupts */
507 for (i = 0; i < REDIR_ENTRIES; i++)
508 vioapic->rtbl[i].reg = 0x0001000000010000UL;
509
510 return (vioapic);
511 }
512
513 void
vioapic_cleanup(struct vioapic * vioapic)514 vioapic_cleanup(struct vioapic *vioapic)
515 {
516
517 mtx_destroy(&vioapic->mtx);
518 free(vioapic, M_VIOAPIC);
519 }
520
521 int
vioapic_pincount(struct vm * vm)522 vioapic_pincount(struct vm *vm)
523 {
524
525 return (REDIR_ENTRIES);
526 }
527
528 #ifdef BHYVE_SNAPSHOT
529 int
vioapic_snapshot(struct vioapic * vioapic,struct vm_snapshot_meta * meta)530 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta)
531 {
532 int ret;
533 int i;
534
535 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done);
536
537 for (i = 0; i < nitems(vioapic->rtbl); i++) {
538 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done);
539 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done);
540 }
541
542 done:
543 return (ret);
544 }
545 #endif
546