1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_bhyve_snapshot.h" 34 35 #include <sys/param.h> 36 #include <sys/queue.h> 37 #include <sys/lock.h> 38 #include <sys/mutex.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/malloc.h> 42 43 #include <x86/apicreg.h> 44 #include <machine/vmm.h> 45 #include <machine/vmm_snapshot.h> 46 47 #include "vmm_ktr.h" 48 #include "vmm_lapic.h" 49 #include "vlapic.h" 50 #include "vioapic.h" 51 52 #define IOREGSEL 0x00 53 #define IOWIN 0x10 54 55 #define REDIR_ENTRIES 32 56 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 57 58 struct vioapic { 59 struct vm *vm; 60 struct mtx mtx; 61 uint32_t id; 62 uint32_t ioregsel; 63 struct { 64 uint64_t reg; 65 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 66 } rtbl[REDIR_ENTRIES]; 67 }; 68 69 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 70 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 71 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 72 73 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 74 75 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 76 VM_CTR1((vioapic)->vm, fmt, a1) 77 78 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 79 VM_CTR2((vioapic)->vm, fmt, a1, a2) 80 81 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 82 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 83 84 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 85 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 86 87 #ifdef KTR 88 static const char * 89 pinstate_str(bool asserted) 90 { 91 92 if (asserted) 93 return ("asserted"); 94 else 95 return ("deasserted"); 96 } 97 #endif 98 99 static void 100 vioapic_send_intr(struct vioapic *vioapic, int pin) 101 { 102 int vector, delmode; 103 uint32_t low, high, dest; 104 bool level, phys; 105 106 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 107 ("vioapic_set_pinstate: invalid pin number %d", pin)); 108 109 KASSERT(VIOAPIC_LOCKED(vioapic), 110 ("vioapic_set_pinstate: vioapic is not locked")); 111 112 low = vioapic->rtbl[pin].reg; 113 high = vioapic->rtbl[pin].reg >> 32; 114 115 if ((low & IOART_INTMASK) == IOART_INTMSET) { 116 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 117 return; 118 } 119 120 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 121 delmode = low & IOART_DELMOD; 122 level = low & IOART_TRGRLVL ? true : false; 123 if (level) { 124 if ((low & IOART_REM_IRR) != 0) { 125 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending", 126 pin); 127 return; 128 } 129 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 130 } 131 132 vector = low & IOART_INTVEC; 133 dest = high >> APIC_ID_SHIFT; 134 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 135 } 136 137 static void 138 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 139 { 140 int oldcnt, newcnt; 141 bool needintr; 142 143 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 144 ("vioapic_set_pinstate: invalid pin number %d", pin)); 145 146 KASSERT(VIOAPIC_LOCKED(vioapic), 147 ("vioapic_set_pinstate: vioapic is not locked")); 148 149 oldcnt = vioapic->rtbl[pin].acnt; 150 if (newstate) 151 vioapic->rtbl[pin].acnt++; 152 else 153 vioapic->rtbl[pin].acnt--; 154 newcnt = vioapic->rtbl[pin].acnt; 155 156 if (newcnt < 0) { 157 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 158 pin, newcnt); 159 } 160 161 needintr = false; 162 if (oldcnt == 0 && newcnt == 1) { 163 needintr = true; 164 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 165 } else if (oldcnt == 1 && newcnt == 0) { 166 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 167 } else { 168 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 169 pin, pinstate_str(newstate), newcnt); 170 } 171 172 if (needintr) 173 vioapic_send_intr(vioapic, pin); 174 } 175 176 enum irqstate { 177 IRQSTATE_ASSERT, 178 IRQSTATE_DEASSERT, 179 IRQSTATE_PULSE 180 }; 181 182 static int 183 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 184 { 185 struct vioapic *vioapic; 186 187 if (irq < 0 || irq >= REDIR_ENTRIES) 188 return (EINVAL); 189 190 vioapic = vm_ioapic(vm); 191 192 VIOAPIC_LOCK(vioapic); 193 switch (irqstate) { 194 case IRQSTATE_ASSERT: 195 vioapic_set_pinstate(vioapic, irq, true); 196 break; 197 case IRQSTATE_DEASSERT: 198 vioapic_set_pinstate(vioapic, irq, false); 199 break; 200 case IRQSTATE_PULSE: 201 vioapic_set_pinstate(vioapic, irq, true); 202 vioapic_set_pinstate(vioapic, irq, false); 203 break; 204 default: 205 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 206 } 207 VIOAPIC_UNLOCK(vioapic); 208 209 return (0); 210 } 211 212 int 213 vioapic_assert_irq(struct vm *vm, int irq) 214 { 215 216 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 217 } 218 219 int 220 vioapic_deassert_irq(struct vm *vm, int irq) 221 { 222 223 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 224 } 225 226 int 227 vioapic_pulse_irq(struct vm *vm, int irq) 228 { 229 230 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 231 } 232 233 /* 234 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 235 * configuration. 236 */ 237 static void 238 vioapic_update_tmr(struct vcpu *vcpu, void *arg) 239 { 240 struct vioapic *vioapic; 241 struct vlapic *vlapic; 242 uint32_t low, high, dest; 243 int delmode, pin, vector; 244 bool level, phys; 245 246 vlapic = vm_lapic(vcpu); 247 vioapic = vm_ioapic(vcpu_vm(vcpu)); 248 249 VIOAPIC_LOCK(vioapic); 250 /* 251 * Reset all vectors to be edge-triggered. 252 */ 253 vlapic_reset_tmr(vlapic); 254 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 255 low = vioapic->rtbl[pin].reg; 256 high = vioapic->rtbl[pin].reg >> 32; 257 258 level = low & IOART_TRGRLVL ? true : false; 259 if (!level) 260 continue; 261 262 /* 263 * For a level-triggered 'pin' let the vlapic figure out if 264 * an assertion on this 'pin' would result in an interrupt 265 * being delivered to it. If yes, then it will modify the 266 * TMR bit associated with this vector to level-triggered. 267 */ 268 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 269 delmode = low & IOART_DELMOD; 270 vector = low & IOART_INTVEC; 271 dest = high >> APIC_ID_SHIFT; 272 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 273 } 274 VIOAPIC_UNLOCK(vioapic); 275 } 276 277 static uint32_t 278 vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr) 279 { 280 int regnum, pin, rshift; 281 282 regnum = addr & 0xff; 283 switch (regnum) { 284 case IOAPIC_ID: 285 return (vioapic->id); 286 break; 287 case IOAPIC_VER: 288 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 289 break; 290 case IOAPIC_ARB: 291 return (vioapic->id); 292 break; 293 default: 294 break; 295 } 296 297 /* redirection table entries */ 298 if (regnum >= IOAPIC_REDTBL && 299 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 300 pin = (regnum - IOAPIC_REDTBL) / 2; 301 if ((regnum - IOAPIC_REDTBL) % 2) 302 rshift = 32; 303 else 304 rshift = 0; 305 306 return (vioapic->rtbl[pin].reg >> rshift); 307 } 308 309 return (0); 310 } 311 312 static void 313 vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr, 314 uint32_t data) 315 { 316 uint64_t data64, mask64; 317 uint64_t last, changed; 318 int regnum, pin, lshift; 319 cpuset_t allvcpus; 320 321 regnum = addr & 0xff; 322 switch (regnum) { 323 case IOAPIC_ID: 324 vioapic->id = data & APIC_ID_MASK; 325 break; 326 case IOAPIC_VER: 327 case IOAPIC_ARB: 328 /* readonly */ 329 break; 330 default: 331 break; 332 } 333 334 /* redirection table entries */ 335 if (regnum >= IOAPIC_REDTBL && 336 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 337 pin = (regnum - IOAPIC_REDTBL) / 2; 338 if ((regnum - IOAPIC_REDTBL) % 2) 339 lshift = 32; 340 else 341 lshift = 0; 342 343 last = vioapic->rtbl[pin].reg; 344 345 data64 = (uint64_t)data << lshift; 346 mask64 = (uint64_t)0xffffffff << lshift; 347 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 348 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 349 350 /* 351 * Switching from level to edge triggering will clear the IRR 352 * bit. This is what FreeBSD will do in order to EOI an 353 * interrupt when the IO-APIC doesn't support targeted EOI (see 354 * _ioapic_eoi_source). 355 */ 356 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 357 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 358 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 359 360 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 361 pin, vioapic->rtbl[pin].reg); 362 363 /* 364 * If any fields in the redirection table entry (except mask 365 * or polarity) have changed then rendezvous all the vcpus 366 * to update their vlapic trigger-mode registers. 367 */ 368 changed = last ^ vioapic->rtbl[pin].reg; 369 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 370 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 371 "vlapic trigger-mode register", pin); 372 VIOAPIC_UNLOCK(vioapic); 373 allvcpus = vm_active_cpus(vioapic->vm); 374 (void)vm_smp_rendezvous(vcpu, allvcpus, 375 vioapic_update_tmr, NULL); 376 VIOAPIC_LOCK(vioapic); 377 } 378 379 /* 380 * Generate an interrupt if the following conditions are met: 381 * - pin trigger mode is level 382 * - pin level is asserted 383 */ 384 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 385 (vioapic->rtbl[pin].acnt > 0)) { 386 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 387 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 388 vioapic_send_intr(vioapic, pin); 389 } 390 } 391 } 392 393 static int 394 vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa, 395 uint64_t *data, int size, bool doread) 396 { 397 uint64_t offset; 398 399 offset = gpa - VIOAPIC_BASE; 400 401 /* 402 * The IOAPIC specification allows 32-bit wide accesses to the 403 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 404 */ 405 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 406 if (doread) 407 *data = 0; 408 return (0); 409 } 410 411 VIOAPIC_LOCK(vioapic); 412 if (offset == IOREGSEL) { 413 if (doread) 414 *data = vioapic->ioregsel; 415 else 416 vioapic->ioregsel = *data; 417 } else { 418 if (doread) { 419 *data = vioapic_read(vioapic, vcpu, 420 vioapic->ioregsel); 421 } else { 422 vioapic_write(vioapic, vcpu, vioapic->ioregsel, 423 *data); 424 } 425 } 426 VIOAPIC_UNLOCK(vioapic); 427 428 return (0); 429 } 430 431 int 432 vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, 433 int size, void *arg) 434 { 435 int error; 436 struct vioapic *vioapic; 437 438 vioapic = vm_ioapic(vcpu_vm(vcpu)); 439 error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true); 440 return (error); 441 } 442 443 int 444 vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, 445 int size, void *arg) 446 { 447 int error; 448 struct vioapic *vioapic; 449 450 vioapic = vm_ioapic(vcpu_vm(vcpu)); 451 error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false); 452 return (error); 453 } 454 455 void 456 vioapic_process_eoi(struct vm *vm, int vector) 457 { 458 struct vioapic *vioapic; 459 int pin; 460 461 KASSERT(vector >= 0 && vector < 256, 462 ("vioapic_process_eoi: invalid vector %d", vector)); 463 464 vioapic = vm_ioapic(vm); 465 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 466 467 /* 468 * XXX keep track of the pins associated with this vector instead 469 * of iterating on every single pin each time. 470 */ 471 VIOAPIC_LOCK(vioapic); 472 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 473 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 474 continue; 475 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 476 continue; 477 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 478 if (vioapic->rtbl[pin].acnt > 0) { 479 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 480 "acnt %d", pin, vioapic->rtbl[pin].acnt); 481 vioapic_send_intr(vioapic, pin); 482 } 483 } 484 VIOAPIC_UNLOCK(vioapic); 485 } 486 487 struct vioapic * 488 vioapic_init(struct vm *vm) 489 { 490 int i; 491 struct vioapic *vioapic; 492 493 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 494 495 vioapic->vm = vm; 496 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 497 498 /* Initialize all redirection entries to mask all interrupts */ 499 for (i = 0; i < REDIR_ENTRIES; i++) 500 vioapic->rtbl[i].reg = 0x0001000000010000UL; 501 502 return (vioapic); 503 } 504 505 void 506 vioapic_cleanup(struct vioapic *vioapic) 507 { 508 509 mtx_destroy(&vioapic->mtx); 510 free(vioapic, M_VIOAPIC); 511 } 512 513 int 514 vioapic_pincount(struct vm *vm) 515 { 516 517 return (REDIR_ENTRIES); 518 } 519 520 #ifdef BHYVE_SNAPSHOT 521 int 522 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta) 523 { 524 int ret; 525 int i; 526 527 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done); 528 529 for (i = 0; i < nitems(vioapic->rtbl); i++) { 530 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done); 531 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done); 532 } 533 534 done: 535 return (ret); 536 } 537 #endif 538