1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "opt_bhyve_snapshot.h" 32 33 #include <sys/param.h> 34 #include <sys/queue.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 41 #include <x86/apicreg.h> 42 #include <machine/vmm.h> 43 #include <machine/vmm_snapshot.h> 44 45 #include <dev/vmm/vmm_ktr.h> 46 47 #include "vmm_lapic.h" 48 #include "vlapic.h" 49 #include "vioapic.h" 50 51 #define IOREGSEL 0x00 52 #define IOWIN 0x10 53 54 #define REDIR_ENTRIES 32 55 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 56 57 struct vioapic { 58 struct vm *vm; 59 struct mtx mtx; 60 uint32_t id; 61 uint32_t ioregsel; 62 struct { 63 uint64_t reg; 64 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 65 } rtbl[REDIR_ENTRIES]; 66 }; 67 68 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 69 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 70 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 71 72 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 73 74 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 75 VM_CTR1((vioapic)->vm, fmt, a1) 76 77 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 78 VM_CTR2((vioapic)->vm, fmt, a1, a2) 79 80 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 81 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 82 83 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 84 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 85 86 #ifdef KTR 87 static const char * 88 pinstate_str(bool asserted) 89 { 90 91 if (asserted) 92 return ("asserted"); 93 else 94 return ("deasserted"); 95 } 96 #endif 97 98 static void 99 vioapic_send_intr(struct vioapic *vioapic, int pin) 100 { 101 int vector, delmode; 102 uint32_t low, high, dest; 103 bool level, phys; 104 105 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 106 ("vioapic_set_pinstate: invalid pin number %d", pin)); 107 108 KASSERT(VIOAPIC_LOCKED(vioapic), 109 ("vioapic_set_pinstate: vioapic is not locked")); 110 111 low = vioapic->rtbl[pin].reg; 112 high = vioapic->rtbl[pin].reg >> 32; 113 114 if ((low & IOART_INTMASK) == IOART_INTMSET) { 115 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 116 return; 117 } 118 119 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 120 delmode = low & IOART_DELMOD; 121 level = low & IOART_TRGRLVL ? true : false; 122 if (level) { 123 if ((low & IOART_REM_IRR) != 0) { 124 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending", 125 pin); 126 return; 127 } 128 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 129 } 130 131 vector = low & IOART_INTVEC; 132 dest = high >> APIC_ID_SHIFT; 133 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 134 } 135 136 static void 137 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 138 { 139 int oldcnt, newcnt; 140 bool needintr; 141 142 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 143 ("vioapic_set_pinstate: invalid pin number %d", pin)); 144 145 KASSERT(VIOAPIC_LOCKED(vioapic), 146 ("vioapic_set_pinstate: vioapic is not locked")); 147 148 oldcnt = vioapic->rtbl[pin].acnt; 149 if (newstate) 150 vioapic->rtbl[pin].acnt++; 151 else 152 vioapic->rtbl[pin].acnt--; 153 newcnt = vioapic->rtbl[pin].acnt; 154 155 if (newcnt < 0) { 156 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 157 pin, newcnt); 158 } 159 160 needintr = false; 161 if (oldcnt == 0 && newcnt == 1) { 162 needintr = true; 163 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 164 } else if (oldcnt == 1 && newcnt == 0) { 165 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 166 } else { 167 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 168 pin, pinstate_str(newstate), newcnt); 169 } 170 171 if (needintr) 172 vioapic_send_intr(vioapic, pin); 173 } 174 175 enum irqstate { 176 IRQSTATE_ASSERT, 177 IRQSTATE_DEASSERT, 178 IRQSTATE_PULSE 179 }; 180 181 static int 182 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 183 { 184 struct vioapic *vioapic; 185 186 if (irq < 0 || irq >= REDIR_ENTRIES) 187 return (EINVAL); 188 189 vioapic = vm_ioapic(vm); 190 191 VIOAPIC_LOCK(vioapic); 192 switch (irqstate) { 193 case IRQSTATE_ASSERT: 194 vioapic_set_pinstate(vioapic, irq, true); 195 break; 196 case IRQSTATE_DEASSERT: 197 vioapic_set_pinstate(vioapic, irq, false); 198 break; 199 case IRQSTATE_PULSE: 200 vioapic_set_pinstate(vioapic, irq, true); 201 vioapic_set_pinstate(vioapic, irq, false); 202 break; 203 default: 204 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 205 } 206 VIOAPIC_UNLOCK(vioapic); 207 208 return (0); 209 } 210 211 int 212 vioapic_assert_irq(struct vm *vm, int irq) 213 { 214 215 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 216 } 217 218 int 219 vioapic_deassert_irq(struct vm *vm, int irq) 220 { 221 222 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 223 } 224 225 int 226 vioapic_pulse_irq(struct vm *vm, int irq) 227 { 228 229 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 230 } 231 232 /* 233 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 234 * configuration. 235 */ 236 static void 237 vioapic_update_tmr(struct vcpu *vcpu, void *arg) 238 { 239 struct vioapic *vioapic; 240 struct vlapic *vlapic; 241 uint32_t low, high, dest; 242 int delmode, pin, vector; 243 bool level, phys; 244 245 vlapic = vm_lapic(vcpu); 246 vioapic = vm_ioapic(vcpu_vm(vcpu)); 247 248 VIOAPIC_LOCK(vioapic); 249 /* 250 * Reset all vectors to be edge-triggered. 251 */ 252 vlapic_reset_tmr(vlapic); 253 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 254 low = vioapic->rtbl[pin].reg; 255 high = vioapic->rtbl[pin].reg >> 32; 256 257 level = low & IOART_TRGRLVL ? true : false; 258 if (!level) 259 continue; 260 261 /* 262 * For a level-triggered 'pin' let the vlapic figure out if 263 * an assertion on this 'pin' would result in an interrupt 264 * being delivered to it. If yes, then it will modify the 265 * TMR bit associated with this vector to level-triggered. 266 */ 267 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 268 delmode = low & IOART_DELMOD; 269 vector = low & IOART_INTVEC; 270 dest = high >> APIC_ID_SHIFT; 271 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 272 } 273 VIOAPIC_UNLOCK(vioapic); 274 } 275 276 static uint32_t 277 vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr) 278 { 279 int regnum, pin, rshift; 280 281 regnum = addr & 0xff; 282 switch (regnum) { 283 case IOAPIC_ID: 284 return (vioapic->id); 285 break; 286 case IOAPIC_VER: 287 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 288 break; 289 case IOAPIC_ARB: 290 return (vioapic->id); 291 break; 292 default: 293 break; 294 } 295 296 /* redirection table entries */ 297 if (regnum >= IOAPIC_REDTBL && 298 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 299 pin = (regnum - IOAPIC_REDTBL) / 2; 300 if ((regnum - IOAPIC_REDTBL) % 2) 301 rshift = 32; 302 else 303 rshift = 0; 304 305 return (vioapic->rtbl[pin].reg >> rshift); 306 } 307 308 return (0); 309 } 310 311 static void 312 vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr, 313 uint32_t data) 314 { 315 uint64_t data64, mask64; 316 uint64_t last, changed; 317 int regnum, pin, lshift; 318 cpuset_t allvcpus; 319 320 regnum = addr & 0xff; 321 switch (regnum) { 322 case IOAPIC_ID: 323 vioapic->id = data & APIC_ID_MASK; 324 break; 325 case IOAPIC_VER: 326 case IOAPIC_ARB: 327 /* readonly */ 328 break; 329 default: 330 break; 331 } 332 333 /* redirection table entries */ 334 if (regnum >= IOAPIC_REDTBL && 335 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 336 pin = (regnum - IOAPIC_REDTBL) / 2; 337 if ((regnum - IOAPIC_REDTBL) % 2) 338 lshift = 32; 339 else 340 lshift = 0; 341 342 last = vioapic->rtbl[pin].reg; 343 344 data64 = (uint64_t)data << lshift; 345 mask64 = (uint64_t)0xffffffff << lshift; 346 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 347 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 348 349 /* 350 * Switching from level to edge triggering will clear the IRR 351 * bit. This is what FreeBSD will do in order to EOI an 352 * interrupt when the IO-APIC doesn't support targeted EOI (see 353 * _ioapic_eoi_source). 354 */ 355 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 356 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 357 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 358 359 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 360 pin, vioapic->rtbl[pin].reg); 361 362 /* 363 * If any fields in the redirection table entry (except mask 364 * or polarity) have changed then rendezvous all the vcpus 365 * to update their vlapic trigger-mode registers. 366 */ 367 changed = last ^ vioapic->rtbl[pin].reg; 368 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 369 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 370 "vlapic trigger-mode register", pin); 371 VIOAPIC_UNLOCK(vioapic); 372 allvcpus = vm_active_cpus(vioapic->vm); 373 (void)vm_smp_rendezvous(vcpu, allvcpus, 374 vioapic_update_tmr, NULL); 375 VIOAPIC_LOCK(vioapic); 376 } 377 378 /* 379 * Generate an interrupt if the following conditions are met: 380 * - pin trigger mode is level 381 * - pin level is asserted 382 */ 383 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 384 (vioapic->rtbl[pin].acnt > 0)) { 385 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 386 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 387 vioapic_send_intr(vioapic, pin); 388 } 389 } 390 } 391 392 static int 393 vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa, 394 uint64_t *data, int size, bool doread) 395 { 396 uint64_t offset; 397 398 offset = gpa - VIOAPIC_BASE; 399 400 /* 401 * The IOAPIC specification allows 32-bit wide accesses to the 402 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 403 */ 404 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 405 if (doread) 406 *data = 0; 407 return (0); 408 } 409 410 VIOAPIC_LOCK(vioapic); 411 if (offset == IOREGSEL) { 412 if (doread) 413 *data = vioapic->ioregsel; 414 else 415 vioapic->ioregsel = *data; 416 } else { 417 if (doread) { 418 *data = vioapic_read(vioapic, vcpu, 419 vioapic->ioregsel); 420 } else { 421 vioapic_write(vioapic, vcpu, vioapic->ioregsel, 422 *data); 423 } 424 } 425 VIOAPIC_UNLOCK(vioapic); 426 427 return (0); 428 } 429 430 int 431 vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, 432 int size, void *arg) 433 { 434 int error; 435 struct vioapic *vioapic; 436 437 vioapic = vm_ioapic(vcpu_vm(vcpu)); 438 error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true); 439 return (error); 440 } 441 442 int 443 vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, 444 int size, void *arg) 445 { 446 int error; 447 struct vioapic *vioapic; 448 449 vioapic = vm_ioapic(vcpu_vm(vcpu)); 450 error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false); 451 return (error); 452 } 453 454 void 455 vioapic_process_eoi(struct vm *vm, int vector) 456 { 457 struct vioapic *vioapic; 458 int pin; 459 460 KASSERT(vector >= 0 && vector < 256, 461 ("vioapic_process_eoi: invalid vector %d", vector)); 462 463 vioapic = vm_ioapic(vm); 464 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 465 466 /* 467 * XXX keep track of the pins associated with this vector instead 468 * of iterating on every single pin each time. 469 */ 470 VIOAPIC_LOCK(vioapic); 471 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 472 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 473 continue; 474 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 475 continue; 476 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 477 if (vioapic->rtbl[pin].acnt > 0) { 478 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 479 "acnt %d", pin, vioapic->rtbl[pin].acnt); 480 vioapic_send_intr(vioapic, pin); 481 } 482 } 483 VIOAPIC_UNLOCK(vioapic); 484 } 485 486 struct vioapic * 487 vioapic_init(struct vm *vm) 488 { 489 int i; 490 struct vioapic *vioapic; 491 492 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 493 494 vioapic->vm = vm; 495 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 496 497 /* Initialize all redirection entries to mask all interrupts */ 498 for (i = 0; i < REDIR_ENTRIES; i++) 499 vioapic->rtbl[i].reg = 0x0001000000010000UL; 500 501 return (vioapic); 502 } 503 504 void 505 vioapic_cleanup(struct vioapic *vioapic) 506 { 507 508 mtx_destroy(&vioapic->mtx); 509 free(vioapic, M_VIOAPIC); 510 } 511 512 int 513 vioapic_pincount(struct vm *vm) 514 { 515 516 return (REDIR_ENTRIES); 517 } 518 519 #ifdef BHYVE_SNAPSHOT 520 int 521 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta) 522 { 523 int ret; 524 int i; 525 526 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done); 527 528 for (i = 0; i < nitems(vioapic->rtbl); i++) { 529 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done); 530 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done); 531 } 532 533 done: 534 return (ret); 535 } 536 #endif 537