1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/queue.h> 37 #include <sys/lock.h> 38 #include <sys/mutex.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/malloc.h> 42 43 #include <x86/apicreg.h> 44 #include <machine/vmm.h> 45 46 #include "vmm_ktr.h" 47 #include "vmm_lapic.h" 48 #include "vlapic.h" 49 #include "vioapic.h" 50 51 #define IOREGSEL 0x00 52 #define IOWIN 0x10 53 54 #define REDIR_ENTRIES 32 55 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 56 57 struct vioapic { 58 struct vm *vm; 59 struct mtx mtx; 60 uint32_t id; 61 uint32_t ioregsel; 62 struct { 63 uint64_t reg; 64 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 65 } rtbl[REDIR_ENTRIES]; 66 }; 67 68 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 69 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 70 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 71 72 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 73 74 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 75 VM_CTR1((vioapic)->vm, fmt, a1) 76 77 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 78 VM_CTR2((vioapic)->vm, fmt, a1, a2) 79 80 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 81 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 82 83 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 84 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 85 86 #ifdef KTR 87 static const char * 88 pinstate_str(bool asserted) 89 { 90 91 if (asserted) 92 return ("asserted"); 93 else 94 return ("deasserted"); 95 } 96 #endif 97 98 static void 99 vioapic_send_intr(struct vioapic *vioapic, int pin) 100 { 101 int vector, delmode; 102 uint32_t low, high, dest; 103 bool level, phys; 104 105 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 106 ("vioapic_set_pinstate: invalid pin number %d", pin)); 107 108 KASSERT(VIOAPIC_LOCKED(vioapic), 109 ("vioapic_set_pinstate: vioapic is not locked")); 110 111 low = vioapic->rtbl[pin].reg; 112 high = vioapic->rtbl[pin].reg >> 32; 113 114 if ((low & IOART_INTMASK) == IOART_INTMSET) { 115 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 116 return; 117 } 118 119 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 120 delmode = low & IOART_DELMOD; 121 level = low & IOART_TRGRLVL ? true : false; 122 if (level) 123 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 124 125 vector = low & IOART_INTVEC; 126 dest = high >> APIC_ID_SHIFT; 127 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 128 } 129 130 static void 131 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 132 { 133 int oldcnt, newcnt; 134 bool needintr; 135 136 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 137 ("vioapic_set_pinstate: invalid pin number %d", pin)); 138 139 KASSERT(VIOAPIC_LOCKED(vioapic), 140 ("vioapic_set_pinstate: vioapic is not locked")); 141 142 oldcnt = vioapic->rtbl[pin].acnt; 143 if (newstate) 144 vioapic->rtbl[pin].acnt++; 145 else 146 vioapic->rtbl[pin].acnt--; 147 newcnt = vioapic->rtbl[pin].acnt; 148 149 if (newcnt < 0) { 150 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 151 pin, newcnt); 152 } 153 154 needintr = false; 155 if (oldcnt == 0 && newcnt == 1) { 156 needintr = true; 157 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 158 } else if (oldcnt == 1 && newcnt == 0) { 159 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 160 } else { 161 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 162 pin, pinstate_str(newstate), newcnt); 163 } 164 165 if (needintr) 166 vioapic_send_intr(vioapic, pin); 167 } 168 169 enum irqstate { 170 IRQSTATE_ASSERT, 171 IRQSTATE_DEASSERT, 172 IRQSTATE_PULSE 173 }; 174 175 static int 176 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 177 { 178 struct vioapic *vioapic; 179 180 if (irq < 0 || irq >= REDIR_ENTRIES) 181 return (EINVAL); 182 183 vioapic = vm_ioapic(vm); 184 185 VIOAPIC_LOCK(vioapic); 186 switch (irqstate) { 187 case IRQSTATE_ASSERT: 188 vioapic_set_pinstate(vioapic, irq, true); 189 break; 190 case IRQSTATE_DEASSERT: 191 vioapic_set_pinstate(vioapic, irq, false); 192 break; 193 case IRQSTATE_PULSE: 194 vioapic_set_pinstate(vioapic, irq, true); 195 vioapic_set_pinstate(vioapic, irq, false); 196 break; 197 default: 198 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 199 } 200 VIOAPIC_UNLOCK(vioapic); 201 202 return (0); 203 } 204 205 int 206 vioapic_assert_irq(struct vm *vm, int irq) 207 { 208 209 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 210 } 211 212 int 213 vioapic_deassert_irq(struct vm *vm, int irq) 214 { 215 216 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 217 } 218 219 int 220 vioapic_pulse_irq(struct vm *vm, int irq) 221 { 222 223 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 224 } 225 226 /* 227 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 228 * configuration. 229 */ 230 static void 231 vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg) 232 { 233 struct vioapic *vioapic; 234 struct vlapic *vlapic; 235 uint32_t low, high, dest; 236 int delmode, pin, vector; 237 bool level, phys; 238 239 vlapic = vm_lapic(vm, vcpuid); 240 vioapic = vm_ioapic(vm); 241 242 VIOAPIC_LOCK(vioapic); 243 /* 244 * Reset all vectors to be edge-triggered. 245 */ 246 vlapic_reset_tmr(vlapic); 247 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 248 low = vioapic->rtbl[pin].reg; 249 high = vioapic->rtbl[pin].reg >> 32; 250 251 level = low & IOART_TRGRLVL ? true : false; 252 if (!level) 253 continue; 254 255 /* 256 * For a level-triggered 'pin' let the vlapic figure out if 257 * an assertion on this 'pin' would result in an interrupt 258 * being delivered to it. If yes, then it will modify the 259 * TMR bit associated with this vector to level-triggered. 260 */ 261 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 262 delmode = low & IOART_DELMOD; 263 vector = low & IOART_INTVEC; 264 dest = high >> APIC_ID_SHIFT; 265 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 266 } 267 VIOAPIC_UNLOCK(vioapic); 268 } 269 270 static uint32_t 271 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) 272 { 273 int regnum, pin, rshift; 274 275 regnum = addr & 0xff; 276 switch (regnum) { 277 case IOAPIC_ID: 278 return (vioapic->id); 279 break; 280 case IOAPIC_VER: 281 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 282 break; 283 case IOAPIC_ARB: 284 return (vioapic->id); 285 break; 286 default: 287 break; 288 } 289 290 /* redirection table entries */ 291 if (regnum >= IOAPIC_REDTBL && 292 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 293 pin = (regnum - IOAPIC_REDTBL) / 2; 294 if ((regnum - IOAPIC_REDTBL) % 2) 295 rshift = 32; 296 else 297 rshift = 0; 298 299 return (vioapic->rtbl[pin].reg >> rshift); 300 } 301 302 return (0); 303 } 304 305 static void 306 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) 307 { 308 uint64_t data64, mask64; 309 uint64_t last, changed; 310 int regnum, pin, lshift; 311 cpuset_t allvcpus; 312 313 regnum = addr & 0xff; 314 switch (regnum) { 315 case IOAPIC_ID: 316 vioapic->id = data & APIC_ID_MASK; 317 break; 318 case IOAPIC_VER: 319 case IOAPIC_ARB: 320 /* readonly */ 321 break; 322 default: 323 break; 324 } 325 326 /* redirection table entries */ 327 if (regnum >= IOAPIC_REDTBL && 328 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 329 pin = (regnum - IOAPIC_REDTBL) / 2; 330 if ((regnum - IOAPIC_REDTBL) % 2) 331 lshift = 32; 332 else 333 lshift = 0; 334 335 last = vioapic->rtbl[pin].reg; 336 337 data64 = (uint64_t)data << lshift; 338 mask64 = (uint64_t)0xffffffff << lshift; 339 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 340 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 341 342 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 343 pin, vioapic->rtbl[pin].reg); 344 345 /* 346 * If any fields in the redirection table entry (except mask 347 * or polarity) have changed then rendezvous all the vcpus 348 * to update their vlapic trigger-mode registers. 349 */ 350 changed = last ^ vioapic->rtbl[pin].reg; 351 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 352 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 353 "vlapic trigger-mode register", pin); 354 VIOAPIC_UNLOCK(vioapic); 355 allvcpus = vm_active_cpus(vioapic->vm); 356 vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus, 357 vioapic_update_tmr, NULL); 358 VIOAPIC_LOCK(vioapic); 359 } 360 361 /* 362 * Generate an interrupt if the following conditions are met: 363 * - pin is not masked 364 * - previous interrupt has been EOIed 365 * - pin level is asserted 366 */ 367 if ((vioapic->rtbl[pin].reg & IOART_INTMASK) == IOART_INTMCLR && 368 (vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0 && 369 (vioapic->rtbl[pin].acnt > 0)) { 370 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 371 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 372 vioapic_send_intr(vioapic, pin); 373 } 374 } 375 } 376 377 static int 378 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa, 379 uint64_t *data, int size, bool doread) 380 { 381 uint64_t offset; 382 383 offset = gpa - VIOAPIC_BASE; 384 385 /* 386 * The IOAPIC specification allows 32-bit wide accesses to the 387 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 388 */ 389 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 390 if (doread) 391 *data = 0; 392 return (0); 393 } 394 395 VIOAPIC_LOCK(vioapic); 396 if (offset == IOREGSEL) { 397 if (doread) 398 *data = vioapic->ioregsel; 399 else 400 vioapic->ioregsel = *data; 401 } else { 402 if (doread) { 403 *data = vioapic_read(vioapic, vcpuid, 404 vioapic->ioregsel); 405 } else { 406 vioapic_write(vioapic, vcpuid, vioapic->ioregsel, 407 *data); 408 } 409 } 410 VIOAPIC_UNLOCK(vioapic); 411 412 return (0); 413 } 414 415 int 416 vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval, 417 int size, void *arg) 418 { 419 int error; 420 struct vioapic *vioapic; 421 422 vioapic = vm_ioapic(vm); 423 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true); 424 return (error); 425 } 426 427 int 428 vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval, 429 int size, void *arg) 430 { 431 int error; 432 struct vioapic *vioapic; 433 434 vioapic = vm_ioapic(vm); 435 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false); 436 return (error); 437 } 438 439 void 440 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector) 441 { 442 struct vioapic *vioapic; 443 int pin; 444 445 KASSERT(vector >= 0 && vector < 256, 446 ("vioapic_process_eoi: invalid vector %d", vector)); 447 448 vioapic = vm_ioapic(vm); 449 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 450 451 /* 452 * XXX keep track of the pins associated with this vector instead 453 * of iterating on every single pin each time. 454 */ 455 VIOAPIC_LOCK(vioapic); 456 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 457 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 458 continue; 459 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 460 continue; 461 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 462 if (vioapic->rtbl[pin].acnt > 0) { 463 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 464 "acnt %d", pin, vioapic->rtbl[pin].acnt); 465 vioapic_send_intr(vioapic, pin); 466 } 467 } 468 VIOAPIC_UNLOCK(vioapic); 469 } 470 471 struct vioapic * 472 vioapic_init(struct vm *vm) 473 { 474 int i; 475 struct vioapic *vioapic; 476 477 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 478 479 vioapic->vm = vm; 480 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 481 482 /* Initialize all redirection entries to mask all interrupts */ 483 for (i = 0; i < REDIR_ENTRIES; i++) 484 vioapic->rtbl[i].reg = 0x0001000000010000UL; 485 486 return (vioapic); 487 } 488 489 void 490 vioapic_cleanup(struct vioapic *vioapic) 491 { 492 493 free(vioapic, M_VIOAPIC); 494 } 495 496 int 497 vioapic_pincount(struct vm *vm) 498 { 499 500 return (REDIR_ENTRIES); 501 } 502