1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_bhyve_snapshot.h" 36 37 #include <sys/param.h> 38 #include <sys/queue.h> 39 #include <sys/lock.h> 40 #include <sys/mutex.h> 41 #include <sys/systm.h> 42 #include <sys/kernel.h> 43 #include <sys/malloc.h> 44 45 #include <x86/apicreg.h> 46 #include <machine/vmm.h> 47 #include <machine/vmm_snapshot.h> 48 49 #include "vmm_ktr.h" 50 #include "vmm_lapic.h" 51 #include "vlapic.h" 52 #include "vioapic.h" 53 54 #define IOREGSEL 0x00 55 #define IOWIN 0x10 56 57 #define REDIR_ENTRIES 32 58 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 59 60 struct vioapic { 61 struct vm *vm; 62 struct mtx mtx; 63 uint32_t id; 64 uint32_t ioregsel; 65 struct { 66 uint64_t reg; 67 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 68 } rtbl[REDIR_ENTRIES]; 69 }; 70 71 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 72 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 73 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 74 75 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 76 77 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 78 VM_CTR1((vioapic)->vm, fmt, a1) 79 80 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 81 VM_CTR2((vioapic)->vm, fmt, a1, a2) 82 83 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 84 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 85 86 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 87 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 88 89 #ifdef KTR 90 static const char * 91 pinstate_str(bool asserted) 92 { 93 94 if (asserted) 95 return ("asserted"); 96 else 97 return ("deasserted"); 98 } 99 #endif 100 101 static void 102 vioapic_send_intr(struct vioapic *vioapic, int pin) 103 { 104 int vector, delmode; 105 uint32_t low, high, dest; 106 bool level, phys; 107 108 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 109 ("vioapic_set_pinstate: invalid pin number %d", pin)); 110 111 KASSERT(VIOAPIC_LOCKED(vioapic), 112 ("vioapic_set_pinstate: vioapic is not locked")); 113 114 low = vioapic->rtbl[pin].reg; 115 high = vioapic->rtbl[pin].reg >> 32; 116 117 if ((low & IOART_INTMASK) == IOART_INTMSET) { 118 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 119 return; 120 } 121 122 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 123 delmode = low & IOART_DELMOD; 124 level = low & IOART_TRGRLVL ? true : false; 125 if (level) { 126 if ((low & IOART_REM_IRR) != 0) { 127 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending", 128 pin); 129 return; 130 } 131 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 132 } 133 134 vector = low & IOART_INTVEC; 135 dest = high >> APIC_ID_SHIFT; 136 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 137 } 138 139 static void 140 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 141 { 142 int oldcnt, newcnt; 143 bool needintr; 144 145 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 146 ("vioapic_set_pinstate: invalid pin number %d", pin)); 147 148 KASSERT(VIOAPIC_LOCKED(vioapic), 149 ("vioapic_set_pinstate: vioapic is not locked")); 150 151 oldcnt = vioapic->rtbl[pin].acnt; 152 if (newstate) 153 vioapic->rtbl[pin].acnt++; 154 else 155 vioapic->rtbl[pin].acnt--; 156 newcnt = vioapic->rtbl[pin].acnt; 157 158 if (newcnt < 0) { 159 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 160 pin, newcnt); 161 } 162 163 needintr = false; 164 if (oldcnt == 0 && newcnt == 1) { 165 needintr = true; 166 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 167 } else if (oldcnt == 1 && newcnt == 0) { 168 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 169 } else { 170 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 171 pin, pinstate_str(newstate), newcnt); 172 } 173 174 if (needintr) 175 vioapic_send_intr(vioapic, pin); 176 } 177 178 enum irqstate { 179 IRQSTATE_ASSERT, 180 IRQSTATE_DEASSERT, 181 IRQSTATE_PULSE 182 }; 183 184 static int 185 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 186 { 187 struct vioapic *vioapic; 188 189 if (irq < 0 || irq >= REDIR_ENTRIES) 190 return (EINVAL); 191 192 vioapic = vm_ioapic(vm); 193 194 VIOAPIC_LOCK(vioapic); 195 switch (irqstate) { 196 case IRQSTATE_ASSERT: 197 vioapic_set_pinstate(vioapic, irq, true); 198 break; 199 case IRQSTATE_DEASSERT: 200 vioapic_set_pinstate(vioapic, irq, false); 201 break; 202 case IRQSTATE_PULSE: 203 vioapic_set_pinstate(vioapic, irq, true); 204 vioapic_set_pinstate(vioapic, irq, false); 205 break; 206 default: 207 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 208 } 209 VIOAPIC_UNLOCK(vioapic); 210 211 return (0); 212 } 213 214 int 215 vioapic_assert_irq(struct vm *vm, int irq) 216 { 217 218 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 219 } 220 221 int 222 vioapic_deassert_irq(struct vm *vm, int irq) 223 { 224 225 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 226 } 227 228 int 229 vioapic_pulse_irq(struct vm *vm, int irq) 230 { 231 232 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 233 } 234 235 /* 236 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 237 * configuration. 238 */ 239 static void 240 vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg) 241 { 242 struct vioapic *vioapic; 243 struct vlapic *vlapic; 244 uint32_t low, high, dest; 245 int delmode, pin, vector; 246 bool level, phys; 247 248 vlapic = vm_lapic(vm, vcpuid); 249 vioapic = vm_ioapic(vm); 250 251 VIOAPIC_LOCK(vioapic); 252 /* 253 * Reset all vectors to be edge-triggered. 254 */ 255 vlapic_reset_tmr(vlapic); 256 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 257 low = vioapic->rtbl[pin].reg; 258 high = vioapic->rtbl[pin].reg >> 32; 259 260 level = low & IOART_TRGRLVL ? true : false; 261 if (!level) 262 continue; 263 264 /* 265 * For a level-triggered 'pin' let the vlapic figure out if 266 * an assertion on this 'pin' would result in an interrupt 267 * being delivered to it. If yes, then it will modify the 268 * TMR bit associated with this vector to level-triggered. 269 */ 270 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 271 delmode = low & IOART_DELMOD; 272 vector = low & IOART_INTVEC; 273 dest = high >> APIC_ID_SHIFT; 274 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 275 } 276 VIOAPIC_UNLOCK(vioapic); 277 } 278 279 static uint32_t 280 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) 281 { 282 int regnum, pin, rshift; 283 284 regnum = addr & 0xff; 285 switch (regnum) { 286 case IOAPIC_ID: 287 return (vioapic->id); 288 break; 289 case IOAPIC_VER: 290 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 291 break; 292 case IOAPIC_ARB: 293 return (vioapic->id); 294 break; 295 default: 296 break; 297 } 298 299 /* redirection table entries */ 300 if (regnum >= IOAPIC_REDTBL && 301 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 302 pin = (regnum - IOAPIC_REDTBL) / 2; 303 if ((regnum - IOAPIC_REDTBL) % 2) 304 rshift = 32; 305 else 306 rshift = 0; 307 308 return (vioapic->rtbl[pin].reg >> rshift); 309 } 310 311 return (0); 312 } 313 314 static void 315 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) 316 { 317 uint64_t data64, mask64; 318 uint64_t last, changed; 319 int regnum, pin, lshift; 320 cpuset_t allvcpus; 321 322 regnum = addr & 0xff; 323 switch (regnum) { 324 case IOAPIC_ID: 325 vioapic->id = data & APIC_ID_MASK; 326 break; 327 case IOAPIC_VER: 328 case IOAPIC_ARB: 329 /* readonly */ 330 break; 331 default: 332 break; 333 } 334 335 /* redirection table entries */ 336 if (regnum >= IOAPIC_REDTBL && 337 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 338 pin = (regnum - IOAPIC_REDTBL) / 2; 339 if ((regnum - IOAPIC_REDTBL) % 2) 340 lshift = 32; 341 else 342 lshift = 0; 343 344 last = vioapic->rtbl[pin].reg; 345 346 data64 = (uint64_t)data << lshift; 347 mask64 = (uint64_t)0xffffffff << lshift; 348 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 349 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 350 351 /* 352 * Switching from level to edge triggering will clear the IRR 353 * bit. This is what FreeBSD will do in order to EOI an 354 * interrupt when the IO-APIC doesn't support targeted EOI (see 355 * _ioapic_eoi_source). 356 */ 357 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 358 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 359 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 360 361 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 362 pin, vioapic->rtbl[pin].reg); 363 364 /* 365 * If any fields in the redirection table entry (except mask 366 * or polarity) have changed then rendezvous all the vcpus 367 * to update their vlapic trigger-mode registers. 368 */ 369 changed = last ^ vioapic->rtbl[pin].reg; 370 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 371 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 372 "vlapic trigger-mode register", pin); 373 VIOAPIC_UNLOCK(vioapic); 374 allvcpus = vm_active_cpus(vioapic->vm); 375 (void)vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus, 376 vioapic_update_tmr, NULL); 377 VIOAPIC_LOCK(vioapic); 378 } 379 380 /* 381 * Generate an interrupt if the following conditions are met: 382 * - pin trigger mode is level 383 * - pin level is asserted 384 */ 385 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 386 (vioapic->rtbl[pin].acnt > 0)) { 387 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 388 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 389 vioapic_send_intr(vioapic, pin); 390 } 391 } 392 } 393 394 static int 395 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa, 396 uint64_t *data, int size, bool doread) 397 { 398 uint64_t offset; 399 400 offset = gpa - VIOAPIC_BASE; 401 402 /* 403 * The IOAPIC specification allows 32-bit wide accesses to the 404 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 405 */ 406 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 407 if (doread) 408 *data = 0; 409 return (0); 410 } 411 412 VIOAPIC_LOCK(vioapic); 413 if (offset == IOREGSEL) { 414 if (doread) 415 *data = vioapic->ioregsel; 416 else 417 vioapic->ioregsel = *data; 418 } else { 419 if (doread) { 420 *data = vioapic_read(vioapic, vcpuid, 421 vioapic->ioregsel); 422 } else { 423 vioapic_write(vioapic, vcpuid, vioapic->ioregsel, 424 *data); 425 } 426 } 427 VIOAPIC_UNLOCK(vioapic); 428 429 return (0); 430 } 431 432 int 433 vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval, 434 int size, void *arg) 435 { 436 int error; 437 struct vioapic *vioapic; 438 439 vioapic = vm_ioapic(vm); 440 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true); 441 return (error); 442 } 443 444 int 445 vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval, 446 int size, void *arg) 447 { 448 int error; 449 struct vioapic *vioapic; 450 451 vioapic = vm_ioapic(vm); 452 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false); 453 return (error); 454 } 455 456 void 457 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector) 458 { 459 struct vioapic *vioapic; 460 int pin; 461 462 KASSERT(vector >= 0 && vector < 256, 463 ("vioapic_process_eoi: invalid vector %d", vector)); 464 465 vioapic = vm_ioapic(vm); 466 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 467 468 /* 469 * XXX keep track of the pins associated with this vector instead 470 * of iterating on every single pin each time. 471 */ 472 VIOAPIC_LOCK(vioapic); 473 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 474 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 475 continue; 476 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 477 continue; 478 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 479 if (vioapic->rtbl[pin].acnt > 0) { 480 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 481 "acnt %d", pin, vioapic->rtbl[pin].acnt); 482 vioapic_send_intr(vioapic, pin); 483 } 484 } 485 VIOAPIC_UNLOCK(vioapic); 486 } 487 488 struct vioapic * 489 vioapic_init(struct vm *vm) 490 { 491 int i; 492 struct vioapic *vioapic; 493 494 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 495 496 vioapic->vm = vm; 497 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 498 499 /* Initialize all redirection entries to mask all interrupts */ 500 for (i = 0; i < REDIR_ENTRIES; i++) 501 vioapic->rtbl[i].reg = 0x0001000000010000UL; 502 503 return (vioapic); 504 } 505 506 void 507 vioapic_cleanup(struct vioapic *vioapic) 508 { 509 510 free(vioapic, M_VIOAPIC); 511 } 512 513 int 514 vioapic_pincount(struct vm *vm) 515 { 516 517 return (REDIR_ENTRIES); 518 } 519 520 #ifdef BHYVE_SNAPSHOT 521 int 522 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta) 523 { 524 int ret; 525 int i; 526 527 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done); 528 529 for (i = 0; i < nitems(vioapic->rtbl); i++) { 530 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done); 531 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done); 532 } 533 534 done: 535 return (ret); 536 } 537 #endif 538