1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29 /*
30 * This file and its contents are supplied under the terms of the
31 * Common Development and Distribution License ("CDDL"), version 1.0.
32 * You may only use this file in accordance with the terms of version
33 * 1.0 of the CDDL.
34 *
35 * A full copy of the text of the CDDL should have accompanied this
36 * source. A copy of the CDDL is also available via the Internet at
37 * http://www.illumos.org/license/CDDL.
38 */
39 /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
40
41 /*
42 * Copyright 2014 Pluribus Networks Inc.
43 * Copyright 2017 Joyent, Inc.
44 * Copyright 2021 Oxide Computer Company
45 */
46
47 #include <sys/cdefs.h>
48
49 #include <sys/param.h>
50 #include <sys/queue.h>
51 #include <sys/mutex.h>
52 #include <sys/systm.h>
53 #include <sys/kernel.h>
54 #include <sys/kmem.h>
55 #include <sys/cpuset.h>
56
57 #include <x86/apicreg.h>
58 #include <machine/vmm.h>
59 #include <sys/vmm_data.h>
60
61 #include "vmm_lapic.h"
62 #include "vlapic.h"
63 #include "vioapic.h"
64
65 #define IOREGSEL 0x00
66 #define IOWIN 0x10
67
68 #define REDIR_ENTRIES 32
69 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
70
71 struct ioapic_stats {
72 uint64_t is_interrupts;
73 uint64_t is_saturate_low;
74 uint64_t is_saturate_high;
75 };
76
77 struct vioapic {
78 struct vm *vm;
79 kmutex_t lock;
80 uint32_t id;
81 uint32_t ioregsel;
82 struct {
83 uint64_t reg;
84 /*
85 * The sum of pin asserts (+1) and deasserts (-1) are tracked in
86 * 'acnt'. It is clamped to prevent overflow or underflow
87 * should emulation consumers feed it an invalid set of
88 * transitions.
89 */
90 uint_t acnt;
91 } rtbl[REDIR_ENTRIES];
92 struct ioapic_stats stats;
93 };
94
95 #define VIOAPIC_LOCK(vioapic) mutex_enter(&((vioapic)->lock))
96 #define VIOAPIC_UNLOCK(vioapic) mutex_exit(&((vioapic)->lock))
97 #define VIOAPIC_LOCKED(vioapic) MUTEX_HELD(&((vioapic)->lock))
98
99
100 static void
vioapic_send_intr(struct vioapic * vioapic,int pin)101 vioapic_send_intr(struct vioapic *vioapic, int pin)
102 {
103 int vector, delmode;
104 uint32_t low, high, dest;
105 bool level, phys;
106
107 VERIFY(pin >= 0 && pin < REDIR_ENTRIES);
108 ASSERT(VIOAPIC_LOCKED(vioapic));
109
110 low = vioapic->rtbl[pin].reg;
111 high = vioapic->rtbl[pin].reg >> 32;
112
113 if ((low & IOART_INTMASK) == IOART_INTMSET) {
114 /* Pin is masked */
115 return;
116 }
117
118 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
119 delmode = low & IOART_DELMOD;
120 level = low & IOART_TRGRLVL ? true : false;
121 if (level) {
122 if ((low & IOART_REM_IRR) != 0) {
123 /* IRR already pending */
124 return;
125 }
126 vioapic->rtbl[pin].reg |= IOART_REM_IRR;
127 }
128
129 vector = low & IOART_INTVEC;
130 dest = high >> APIC_ID_SHIFT;
131 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
132 vioapic->stats.is_interrupts++;
133 }
134
135 static int
vioapic_set_pinstate(struct vioapic * vioapic,int pin,bool newstate)136 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
137 {
138 uint_t oldcnt, newcnt;
139 bool needintr = false;
140 int err = 0;
141
142 VERIFY(pin >= 0 && pin < REDIR_ENTRIES);
143 ASSERT(VIOAPIC_LOCKED(vioapic));
144
145 oldcnt = newcnt = vioapic->rtbl[pin].acnt;
146 if (newstate) {
147 if (newcnt != UINT_MAX) {
148 newcnt++;
149 } else {
150 err = E2BIG;
151 DTRACE_PROBE2(vioapic__sat_high,
152 struct vioapic *, vioapic, int, pin);
153 vioapic->stats.is_saturate_high++;
154 }
155 } else {
156 if (newcnt != 0) {
157 newcnt--;
158 } else {
159 err = ERANGE;
160 DTRACE_PROBE2(vioapic__sat_low,
161 struct vioapic *, vioapic, int, pin);
162 vioapic->stats.is_saturate_low++;
163 }
164 }
165 vioapic->rtbl[pin].acnt = newcnt;
166
167 if (oldcnt == 0 && newcnt == 1) {
168 needintr = true;
169 DTRACE_PROBE2(vioapic__assert, struct vioapic *, vioapic,
170 int, pin);
171 } else if (oldcnt == 1 && newcnt == 0) {
172 DTRACE_PROBE2(vioapic__deassert, struct vioapic *, vioapic,
173 int, pin);
174 }
175
176 if (needintr) {
177 vioapic_send_intr(vioapic, pin);
178 }
179 return (err);
180 }
181
182 enum irqstate {
183 IRQSTATE_ASSERT,
184 IRQSTATE_DEASSERT,
185 IRQSTATE_PULSE
186 };
187
188 static int
vioapic_set_irqstate(struct vm * vm,int irq,enum irqstate irqstate)189 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
190 {
191 struct vioapic *vioapic;
192 int err = 0;
193
194 if (irq < 0 || irq >= REDIR_ENTRIES)
195 return (EINVAL);
196
197 vioapic = vm_ioapic(vm);
198
199 VIOAPIC_LOCK(vioapic);
200 switch (irqstate) {
201 case IRQSTATE_ASSERT:
202 err = vioapic_set_pinstate(vioapic, irq, true);
203 break;
204 case IRQSTATE_DEASSERT:
205 err = vioapic_set_pinstate(vioapic, irq, false);
206 break;
207 case IRQSTATE_PULSE:
208 err = vioapic_set_pinstate(vioapic, irq, true);
209 if (err == 0) {
210 err = vioapic_set_pinstate(vioapic, irq, false);
211 }
212 break;
213 default:
214 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
215 }
216 VIOAPIC_UNLOCK(vioapic);
217
218 return (err);
219 }
220
221 int
vioapic_assert_irq(struct vm * vm,int irq)222 vioapic_assert_irq(struct vm *vm, int irq)
223 {
224
225 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
226 }
227
228 int
vioapic_deassert_irq(struct vm * vm,int irq)229 vioapic_deassert_irq(struct vm *vm, int irq)
230 {
231
232 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
233 }
234
235 int
vioapic_pulse_irq(struct vm * vm,int irq)236 vioapic_pulse_irq(struct vm *vm, int irq)
237 {
238
239 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
240 }
241
242 static uint32_t
vioapic_read(struct vioapic * vioapic,int vcpuid,uint32_t addr)243 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr)
244 {
245 int regnum, pin, rshift;
246
247 regnum = addr & 0xff;
248 switch (regnum) {
249 case IOAPIC_ID:
250 return (vioapic->id);
251 break;
252 case IOAPIC_VER:
253 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
254 break;
255 case IOAPIC_ARB:
256 return (vioapic->id);
257 break;
258 default:
259 break;
260 }
261
262 /* redirection table entries */
263 if (regnum >= IOAPIC_REDTBL &&
264 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
265 pin = (regnum - IOAPIC_REDTBL) / 2;
266 if ((regnum - IOAPIC_REDTBL) % 2)
267 rshift = 32;
268 else
269 rshift = 0;
270
271 return (vioapic->rtbl[pin].reg >> rshift);
272 }
273
274 return (0);
275 }
276
277 static void
vioapic_write(struct vioapic * vioapic,int vcpuid,uint32_t addr,uint32_t data)278 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data)
279 {
280 uint64_t data64, mask64;
281 int regnum, pin, lshift;
282
283 regnum = addr & 0xff;
284 switch (regnum) {
285 case IOAPIC_ID:
286 vioapic->id = data & APIC_ID_MASK;
287 break;
288 case IOAPIC_VER:
289 case IOAPIC_ARB:
290 /* readonly */
291 break;
292 default:
293 break;
294 }
295
296 /* redirection table entries */
297 if (regnum >= IOAPIC_REDTBL &&
298 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
299 pin = (regnum - IOAPIC_REDTBL) / 2;
300 if ((regnum - IOAPIC_REDTBL) % 2)
301 lshift = 32;
302 else
303 lshift = 0;
304
305 data64 = (uint64_t)data << lshift;
306 mask64 = (uint64_t)0xffffffff << lshift;
307 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
308 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
309
310 /*
311 * Switching from level to edge triggering will clear the IRR
312 * bit. This is what FreeBSD will do in order to EOI an
313 * interrupt when the IO-APIC doesn't support targeted EOI (see
314 * _ioapic_eoi_source).
315 */
316 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG &&
317 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0)
318 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
319
320 /*
321 * Generate an interrupt if the following conditions are met:
322 * - pin trigger mode is level
323 * - pin level is asserted
324 */
325 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL &&
326 (vioapic->rtbl[pin].acnt > 0)) {
327 vioapic_send_intr(vioapic, pin);
328 }
329 }
330 }
331
332 static int
vioapic_mmio_rw(struct vioapic * vioapic,int vcpuid,uint64_t gpa,uint64_t * data,int size,bool doread)333 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa,
334 uint64_t *data, int size, bool doread)
335 {
336 uint64_t offset;
337
338 offset = gpa - VIOAPIC_BASE;
339
340 /*
341 * The IOAPIC specification allows 32-bit wide accesses to the
342 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
343 */
344 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
345 if (doread)
346 *data = 0;
347 return (0);
348 }
349
350 VIOAPIC_LOCK(vioapic);
351 if (offset == IOREGSEL) {
352 if (doread)
353 *data = vioapic->ioregsel;
354 else
355 vioapic->ioregsel = *data;
356 } else {
357 if (doread) {
358 *data = vioapic_read(vioapic, vcpuid,
359 vioapic->ioregsel);
360 } else {
361 vioapic_write(vioapic, vcpuid, vioapic->ioregsel,
362 *data);
363 }
364 }
365 VIOAPIC_UNLOCK(vioapic);
366
367 return (0);
368 }
369
370 int
vioapic_mmio_read(struct vm * vm,int vcpuid,uint64_t gpa,uint64_t * rval,int size)371 vioapic_mmio_read(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t *rval,
372 int size)
373 {
374 int error;
375 struct vioapic *vioapic;
376
377 vioapic = vm_ioapic(vm);
378 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true);
379 return (error);
380 }
381
382 int
vioapic_mmio_write(struct vm * vm,int vcpuid,uint64_t gpa,uint64_t wval,int size)383 vioapic_mmio_write(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t wval,
384 int size)
385 {
386 int error;
387 struct vioapic *vioapic;
388
389 vioapic = vm_ioapic(vm);
390 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false);
391 return (error);
392 }
393
394 void
vioapic_process_eoi(struct vm * vm,int vcpuid,int vector)395 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector)
396 {
397 struct vioapic *vioapic;
398 int pin;
399
400 KASSERT(vector >= 0 && vector < 256,
401 ("vioapic_process_eoi: invalid vector %d", vector));
402
403 vioapic = vm_ioapic(vm);
404
405 /*
406 * XXX keep track of the pins associated with this vector instead
407 * of iterating on every single pin each time.
408 */
409 VIOAPIC_LOCK(vioapic);
410 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
411 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
412 continue;
413 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
414 continue;
415 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
416 if (vioapic->rtbl[pin].acnt > 0) {
417 /* Pin asserted at EOI */
418 vioapic_send_intr(vioapic, pin);
419 }
420 }
421 VIOAPIC_UNLOCK(vioapic);
422 }
423
424 struct vioapic *
vioapic_init(struct vm * vm)425 vioapic_init(struct vm *vm)
426 {
427 int i;
428 struct vioapic *vioapic;
429
430 vioapic = kmem_zalloc(sizeof (struct vioapic), KM_SLEEP);
431
432 vioapic->vm = vm;
433 mutex_init(&vioapic->lock, NULL, MUTEX_ADAPTIVE, NULL);
434
435 /* Initialize all redirection entries to mask all interrupts */
436 for (i = 0; i < REDIR_ENTRIES; i++)
437 vioapic->rtbl[i].reg = 0x0001000000010000UL;
438
439 return (vioapic);
440 }
441
442 void
vioapic_cleanup(struct vioapic * vioapic)443 vioapic_cleanup(struct vioapic *vioapic)
444 {
445 mutex_destroy(&vioapic->lock);
446 kmem_free(vioapic, sizeof (*vioapic));
447 }
448
449 int
vioapic_pincount(struct vm * vm)450 vioapic_pincount(struct vm *vm)
451 {
452
453 return (REDIR_ENTRIES);
454 }
455
456 static int
vioapic_data_read(void * datap,const vmm_data_req_t * req)457 vioapic_data_read(void *datap, const vmm_data_req_t *req)
458 {
459 VERIFY3U(req->vdr_class, ==, VDC_IOAPIC);
460 VERIFY3U(req->vdr_version, ==, 1);
461 VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_ioapic_v1));
462
463 struct vioapic *vioapic = datap;
464 struct vdi_ioapic_v1 *out = req->vdr_data;
465
466 VIOAPIC_LOCK(vioapic);
467 out->vi_id = vioapic->id;
468 out->vi_reg_sel = vioapic->ioregsel;
469 for (uint_t i = 0; i < REDIR_ENTRIES; i++) {
470 out->vi_pin_reg[i] = vioapic->rtbl[i].reg;
471 out->vi_pin_level[i] = vioapic->rtbl[i].acnt;
472 }
473 VIOAPIC_UNLOCK(vioapic);
474
475 return (0);
476 }
477
478 static int
vioapic_data_write(void * datap,const vmm_data_req_t * req)479 vioapic_data_write(void *datap, const vmm_data_req_t *req)
480 {
481 VERIFY3U(req->vdr_class, ==, VDC_IOAPIC);
482 VERIFY3U(req->vdr_version, ==, 1);
483 VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_ioapic_v1));
484
485 struct vioapic *vioapic = datap;
486 const struct vdi_ioapic_v1 *src = req->vdr_data;
487
488 VIOAPIC_LOCK(vioapic);
489 vioapic->id = src->vi_id;
490 vioapic->ioregsel = src->vi_reg_sel;
491 for (uint_t i = 0; i < REDIR_ENTRIES; i++) {
492 vioapic->rtbl[i].reg = src->vi_pin_reg[i] & ~RTBL_RO_BITS;
493 vioapic->rtbl[i].acnt = src->vi_pin_level[i];
494 }
495 VIOAPIC_UNLOCK(vioapic);
496
497 return (0);
498 }
499
500 static const vmm_data_version_entry_t ioapic_v1 = {
501 .vdve_class = VDC_IOAPIC,
502 .vdve_version = 1,
503 .vdve_len_expect = sizeof (struct vdi_ioapic_v1),
504 .vdve_readf = vioapic_data_read,
505 .vdve_writef = vioapic_data_write,
506 };
507 VMM_DATA_VERSION(ioapic_v1);
508