1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "opt_bhyve_snapshot.h" 32 33 #include <sys/param.h> 34 #include <sys/queue.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 41 #include <x86/apicreg.h> 42 #include <machine/vmm.h> 43 #include <machine/vmm_snapshot.h> 44 45 #include <dev/vmm/vmm_ktr.h> 46 47 #include "vmm_lapic.h" 48 #include "vlapic.h" 49 #include "vioapic.h" 50 51 #define IOREGSEL 0x00 52 #define IOWIN 0x10 53 54 #define REDIR_ENTRIES 32 55 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 56 57 struct vioapic { 58 struct vm *vm; 59 struct mtx mtx; 60 uint32_t id; 61 uint32_t ioregsel; 62 struct { 63 uint64_t reg; 64 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 65 } rtbl[REDIR_ENTRIES]; 66 }; 67 68 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 69 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 70 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 71 72 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 73 74 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 75 VM_CTR1((vioapic)->vm, fmt, a1) 76 77 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 78 VM_CTR2((vioapic)->vm, fmt, a1, a2) 79 80 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 81 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 82 83 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 84 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 85 86 #ifdef KTR 87 static const char * 88 pinstate_str(bool asserted) 89 { 90 91 if (asserted) 92 return ("asserted"); 93 else 94 return ("deasserted"); 95 } 96 #endif 97 98 static void 99 vioapic_send_intr(struct vioapic *vioapic, int pin) 100 { 101 int vector, delmode; 102 uint32_t low, high, dest; 103 bool level, phys; 104 105 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 106 ("vioapic_set_pinstate: invalid pin number %d", pin)); 107 108 KASSERT(VIOAPIC_LOCKED(vioapic), 109 ("vioapic_set_pinstate: vioapic is not locked")); 110 111 low = vioapic->rtbl[pin].reg; 112 high = vioapic->rtbl[pin].reg >> 32; 113 114 if ((low & IOART_INTMASK) == IOART_INTMSET) { 115 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 116 return; 117 } 118 119 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 120 delmode = low & IOART_DELMOD; 121 level = low & IOART_TRGRLVL ? true : false; 122 if (level) { 123 if ((low & IOART_REM_IRR) != 0) { 124 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending", 125 pin); 126 return; 127 } 128 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 129 } 130 131 vector = low & IOART_INTVEC; 132 dest = high >> APIC_ID_SHIFT; 133 /* 134 * Ideally we'd just call lapic_intr_msi() here with the 135 * constructed MSI instead of interpreting it for ourselves. 136 * But until/unless we support emulated IOMMUs with interrupt 137 * remapping, interpretation is simple. We just need to mask 138 * in the Extended Destination ID bits for the 15-bit 139 * enlightenment (http://david.woodhou.se/ExtDestId.pdf) 140 */ 141 dest |= ((high & APIC_EXT_ID_MASK) >> APIC_EXT_ID_SHIFT) << 8; 142 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 143 } 144 145 static void 146 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 147 { 148 int oldcnt, newcnt; 149 bool needintr; 150 151 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 152 ("vioapic_set_pinstate: invalid pin number %d", pin)); 153 154 KASSERT(VIOAPIC_LOCKED(vioapic), 155 ("vioapic_set_pinstate: vioapic is not locked")); 156 157 oldcnt = vioapic->rtbl[pin].acnt; 158 if (newstate) 159 vioapic->rtbl[pin].acnt++; 160 else 161 vioapic->rtbl[pin].acnt--; 162 newcnt = vioapic->rtbl[pin].acnt; 163 164 if (newcnt < 0) { 165 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 166 pin, newcnt); 167 } 168 169 needintr = false; 170 if (oldcnt == 0 && newcnt == 1) { 171 needintr = true; 172 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 173 } else if (oldcnt == 1 && newcnt == 0) { 174 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 175 } else { 176 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 177 pin, pinstate_str(newstate), newcnt); 178 } 179 180 if (needintr) 181 vioapic_send_intr(vioapic, pin); 182 } 183 184 enum irqstate { 185 IRQSTATE_ASSERT, 186 IRQSTATE_DEASSERT, 187 IRQSTATE_PULSE 188 }; 189 190 static int 191 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 192 { 193 struct vioapic *vioapic; 194 195 if (irq < 0 || irq >= REDIR_ENTRIES) 196 return (EINVAL); 197 198 vioapic = vm_ioapic(vm); 199 200 VIOAPIC_LOCK(vioapic); 201 switch (irqstate) { 202 case IRQSTATE_ASSERT: 203 vioapic_set_pinstate(vioapic, irq, true); 204 break; 205 case IRQSTATE_DEASSERT: 206 vioapic_set_pinstate(vioapic, irq, false); 207 break; 208 case IRQSTATE_PULSE: 209 vioapic_set_pinstate(vioapic, irq, true); 210 vioapic_set_pinstate(vioapic, irq, false); 211 break; 212 default: 213 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 214 } 215 VIOAPIC_UNLOCK(vioapic); 216 217 return (0); 218 } 219 220 int 221 vioapic_assert_irq(struct vm *vm, int irq) 222 { 223 224 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 225 } 226 227 int 228 vioapic_deassert_irq(struct vm *vm, int irq) 229 { 230 231 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 232 } 233 234 int 235 vioapic_pulse_irq(struct vm *vm, int irq) 236 { 237 238 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 239 } 240 241 /* 242 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 243 * configuration. 244 */ 245 static void 246 vioapic_update_tmr(struct vcpu *vcpu, void *arg) 247 { 248 struct vioapic *vioapic; 249 struct vlapic *vlapic; 250 uint32_t low, high, dest; 251 int delmode, pin, vector; 252 bool level, phys; 253 254 vlapic = vm_lapic(vcpu); 255 vioapic = vm_ioapic(vcpu_vm(vcpu)); 256 257 VIOAPIC_LOCK(vioapic); 258 /* 259 * Reset all vectors to be edge-triggered. 260 */ 261 vlapic_reset_tmr(vlapic); 262 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 263 low = vioapic->rtbl[pin].reg; 264 high = vioapic->rtbl[pin].reg >> 32; 265 266 level = low & IOART_TRGRLVL ? true : false; 267 if (!level) 268 continue; 269 270 /* 271 * For a level-triggered 'pin' let the vlapic figure out if 272 * an assertion on this 'pin' would result in an interrupt 273 * being delivered to it. If yes, then it will modify the 274 * TMR bit associated with this vector to level-triggered. 275 */ 276 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 277 delmode = low & IOART_DELMOD; 278 vector = low & IOART_INTVEC; 279 dest = high >> APIC_ID_SHIFT; 280 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 281 } 282 VIOAPIC_UNLOCK(vioapic); 283 } 284 285 static uint32_t 286 vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr) 287 { 288 int regnum, pin, rshift; 289 290 regnum = addr & 0xff; 291 switch (regnum) { 292 case IOAPIC_ID: 293 return (vioapic->id); 294 break; 295 case IOAPIC_VER: 296 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 297 break; 298 case IOAPIC_ARB: 299 return (vioapic->id); 300 break; 301 default: 302 break; 303 } 304 305 /* redirection table entries */ 306 if (regnum >= IOAPIC_REDTBL && 307 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 308 pin = (regnum - IOAPIC_REDTBL) / 2; 309 if ((regnum - IOAPIC_REDTBL) % 2) 310 rshift = 32; 311 else 312 rshift = 0; 313 314 return (vioapic->rtbl[pin].reg >> rshift); 315 } 316 317 return (0); 318 } 319 320 static void 321 vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr, 322 uint32_t data) 323 { 324 uint64_t data64, mask64; 325 uint64_t last, changed; 326 int regnum, pin, lshift; 327 cpuset_t allvcpus; 328 329 regnum = addr & 0xff; 330 switch (regnum) { 331 case IOAPIC_ID: 332 vioapic->id = data & APIC_ID_MASK; 333 break; 334 case IOAPIC_VER: 335 case IOAPIC_ARB: 336 /* readonly */ 337 break; 338 default: 339 break; 340 } 341 342 /* redirection table entries */ 343 if (regnum >= IOAPIC_REDTBL && 344 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 345 pin = (regnum - IOAPIC_REDTBL) / 2; 346 if ((regnum - IOAPIC_REDTBL) % 2) 347 lshift = 32; 348 else 349 lshift = 0; 350 351 last = vioapic->rtbl[pin].reg; 352 353 data64 = (uint64_t)data << lshift; 354 mask64 = (uint64_t)0xffffffff << lshift; 355 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 356 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 357 358 /* 359 * Switching from level to edge triggering will clear the IRR 360 * bit. This is what FreeBSD will do in order to EOI an 361 * interrupt when the IO-APIC doesn't support targeted EOI (see 362 * _ioapic_eoi_source). 363 */ 364 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 365 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 366 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 367 368 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 369 pin, vioapic->rtbl[pin].reg); 370 371 /* 372 * If any fields in the redirection table entry (except mask 373 * or polarity) have changed then rendezvous all the vcpus 374 * to update their vlapic trigger-mode registers. 375 */ 376 changed = last ^ vioapic->rtbl[pin].reg; 377 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 378 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 379 "vlapic trigger-mode register", pin); 380 VIOAPIC_UNLOCK(vioapic); 381 allvcpus = vm_active_cpus(vioapic->vm); 382 (void)vm_smp_rendezvous(vcpu, allvcpus, 383 vioapic_update_tmr, NULL); 384 VIOAPIC_LOCK(vioapic); 385 } 386 387 /* 388 * Generate an interrupt if the following conditions are met: 389 * - pin trigger mode is level 390 * - pin level is asserted 391 */ 392 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 393 (vioapic->rtbl[pin].acnt > 0)) { 394 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 395 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 396 vioapic_send_intr(vioapic, pin); 397 } 398 } 399 } 400 401 static int 402 vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa, 403 uint64_t *data, int size, bool doread) 404 { 405 uint64_t offset; 406 407 offset = gpa - VIOAPIC_BASE; 408 409 /* 410 * The IOAPIC specification allows 32-bit wide accesses to the 411 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 412 */ 413 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 414 if (doread) 415 *data = 0; 416 return (0); 417 } 418 419 VIOAPIC_LOCK(vioapic); 420 if (offset == IOREGSEL) { 421 if (doread) 422 *data = vioapic->ioregsel; 423 else 424 vioapic->ioregsel = *data; 425 } else { 426 if (doread) { 427 *data = vioapic_read(vioapic, vcpu, 428 vioapic->ioregsel); 429 } else { 430 vioapic_write(vioapic, vcpu, vioapic->ioregsel, 431 *data); 432 } 433 } 434 VIOAPIC_UNLOCK(vioapic); 435 436 return (0); 437 } 438 439 int 440 vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, 441 int size, void *arg) 442 { 443 int error; 444 struct vioapic *vioapic; 445 446 vioapic = vm_ioapic(vcpu_vm(vcpu)); 447 error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true); 448 return (error); 449 } 450 451 int 452 vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, 453 int size, void *arg) 454 { 455 int error; 456 struct vioapic *vioapic; 457 458 vioapic = vm_ioapic(vcpu_vm(vcpu)); 459 error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false); 460 return (error); 461 } 462 463 void 464 vioapic_process_eoi(struct vm *vm, int vector) 465 { 466 struct vioapic *vioapic; 467 int pin; 468 469 KASSERT(vector >= 0 && vector < 256, 470 ("vioapic_process_eoi: invalid vector %d", vector)); 471 472 vioapic = vm_ioapic(vm); 473 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 474 475 /* 476 * XXX keep track of the pins associated with this vector instead 477 * of iterating on every single pin each time. 478 */ 479 VIOAPIC_LOCK(vioapic); 480 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 481 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 482 continue; 483 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 484 continue; 485 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 486 if (vioapic->rtbl[pin].acnt > 0) { 487 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 488 "acnt %d", pin, vioapic->rtbl[pin].acnt); 489 vioapic_send_intr(vioapic, pin); 490 } 491 } 492 VIOAPIC_UNLOCK(vioapic); 493 } 494 495 struct vioapic * 496 vioapic_init(struct vm *vm) 497 { 498 int i; 499 struct vioapic *vioapic; 500 501 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 502 503 vioapic->vm = vm; 504 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 505 506 /* Initialize all redirection entries to mask all interrupts */ 507 for (i = 0; i < REDIR_ENTRIES; i++) 508 vioapic->rtbl[i].reg = 0x0001000000010000UL; 509 510 return (vioapic); 511 } 512 513 void 514 vioapic_cleanup(struct vioapic *vioapic) 515 { 516 517 mtx_destroy(&vioapic->mtx); 518 free(vioapic, M_VIOAPIC); 519 } 520 521 int 522 vioapic_pincount(struct vm *vm) 523 { 524 525 return (REDIR_ENTRIES); 526 } 527 528 #ifdef BHYVE_SNAPSHOT 529 int 530 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta) 531 { 532 int ret; 533 int i; 534 535 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done); 536 537 for (i = 0; i < nitems(vioapic->rtbl); i++) { 538 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done); 539 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done); 540 } 541 542 done: 543 return (ret); 544 } 545 #endif 546