1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "opt_bhyve_snapshot.h" 32 33 #include <sys/param.h> 34 #include <sys/queue.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 41 #include <x86/apicreg.h> 42 #include <machine/vmm.h> 43 #include <machine/vmm_snapshot.h> 44 45 #include "vmm_ktr.h" 46 #include "vmm_lapic.h" 47 #include "vlapic.h" 48 #include "vioapic.h" 49 50 #define IOREGSEL 0x00 51 #define IOWIN 0x10 52 53 #define REDIR_ENTRIES 32 54 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 55 56 struct vioapic { 57 struct vm *vm; 58 struct mtx mtx; 59 uint32_t id; 60 uint32_t ioregsel; 61 struct { 62 uint64_t reg; 63 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 64 } rtbl[REDIR_ENTRIES]; 65 }; 66 67 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 68 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 69 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 70 71 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 72 73 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 74 VM_CTR1((vioapic)->vm, fmt, a1) 75 76 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 77 VM_CTR2((vioapic)->vm, fmt, a1, a2) 78 79 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 80 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 81 82 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 83 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 84 85 #ifdef KTR 86 static const char * 87 pinstate_str(bool asserted) 88 { 89 90 if (asserted) 91 return ("asserted"); 92 else 93 return ("deasserted"); 94 } 95 #endif 96 97 static void 98 vioapic_send_intr(struct vioapic *vioapic, int pin) 99 { 100 int vector, delmode; 101 uint32_t low, high, dest; 102 bool level, phys; 103 104 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 105 ("vioapic_set_pinstate: invalid pin number %d", pin)); 106 107 KASSERT(VIOAPIC_LOCKED(vioapic), 108 ("vioapic_set_pinstate: vioapic is not locked")); 109 110 low = vioapic->rtbl[pin].reg; 111 high = vioapic->rtbl[pin].reg >> 32; 112 113 if ((low & IOART_INTMASK) == IOART_INTMSET) { 114 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 115 return; 116 } 117 118 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 119 delmode = low & IOART_DELMOD; 120 level = low & IOART_TRGRLVL ? true : false; 121 if (level) { 122 if ((low & IOART_REM_IRR) != 0) { 123 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending", 124 pin); 125 return; 126 } 127 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 128 } 129 130 vector = low & IOART_INTVEC; 131 dest = high >> APIC_ID_SHIFT; 132 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 133 } 134 135 static void 136 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 137 { 138 int oldcnt, newcnt; 139 bool needintr; 140 141 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 142 ("vioapic_set_pinstate: invalid pin number %d", pin)); 143 144 KASSERT(VIOAPIC_LOCKED(vioapic), 145 ("vioapic_set_pinstate: vioapic is not locked")); 146 147 oldcnt = vioapic->rtbl[pin].acnt; 148 if (newstate) 149 vioapic->rtbl[pin].acnt++; 150 else 151 vioapic->rtbl[pin].acnt--; 152 newcnt = vioapic->rtbl[pin].acnt; 153 154 if (newcnt < 0) { 155 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 156 pin, newcnt); 157 } 158 159 needintr = false; 160 if (oldcnt == 0 && newcnt == 1) { 161 needintr = true; 162 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 163 } else if (oldcnt == 1 && newcnt == 0) { 164 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 165 } else { 166 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 167 pin, pinstate_str(newstate), newcnt); 168 } 169 170 if (needintr) 171 vioapic_send_intr(vioapic, pin); 172 } 173 174 enum irqstate { 175 IRQSTATE_ASSERT, 176 IRQSTATE_DEASSERT, 177 IRQSTATE_PULSE 178 }; 179 180 static int 181 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 182 { 183 struct vioapic *vioapic; 184 185 if (irq < 0 || irq >= REDIR_ENTRIES) 186 return (EINVAL); 187 188 vioapic = vm_ioapic(vm); 189 190 VIOAPIC_LOCK(vioapic); 191 switch (irqstate) { 192 case IRQSTATE_ASSERT: 193 vioapic_set_pinstate(vioapic, irq, true); 194 break; 195 case IRQSTATE_DEASSERT: 196 vioapic_set_pinstate(vioapic, irq, false); 197 break; 198 case IRQSTATE_PULSE: 199 vioapic_set_pinstate(vioapic, irq, true); 200 vioapic_set_pinstate(vioapic, irq, false); 201 break; 202 default: 203 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 204 } 205 VIOAPIC_UNLOCK(vioapic); 206 207 return (0); 208 } 209 210 int 211 vioapic_assert_irq(struct vm *vm, int irq) 212 { 213 214 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 215 } 216 217 int 218 vioapic_deassert_irq(struct vm *vm, int irq) 219 { 220 221 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 222 } 223 224 int 225 vioapic_pulse_irq(struct vm *vm, int irq) 226 { 227 228 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 229 } 230 231 /* 232 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 233 * configuration. 234 */ 235 static void 236 vioapic_update_tmr(struct vcpu *vcpu, void *arg) 237 { 238 struct vioapic *vioapic; 239 struct vlapic *vlapic; 240 uint32_t low, high, dest; 241 int delmode, pin, vector; 242 bool level, phys; 243 244 vlapic = vm_lapic(vcpu); 245 vioapic = vm_ioapic(vcpu_vm(vcpu)); 246 247 VIOAPIC_LOCK(vioapic); 248 /* 249 * Reset all vectors to be edge-triggered. 250 */ 251 vlapic_reset_tmr(vlapic); 252 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 253 low = vioapic->rtbl[pin].reg; 254 high = vioapic->rtbl[pin].reg >> 32; 255 256 level = low & IOART_TRGRLVL ? true : false; 257 if (!level) 258 continue; 259 260 /* 261 * For a level-triggered 'pin' let the vlapic figure out if 262 * an assertion on this 'pin' would result in an interrupt 263 * being delivered to it. If yes, then it will modify the 264 * TMR bit associated with this vector to level-triggered. 265 */ 266 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 267 delmode = low & IOART_DELMOD; 268 vector = low & IOART_INTVEC; 269 dest = high >> APIC_ID_SHIFT; 270 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 271 } 272 VIOAPIC_UNLOCK(vioapic); 273 } 274 275 static uint32_t 276 vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr) 277 { 278 int regnum, pin, rshift; 279 280 regnum = addr & 0xff; 281 switch (regnum) { 282 case IOAPIC_ID: 283 return (vioapic->id); 284 break; 285 case IOAPIC_VER: 286 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 287 break; 288 case IOAPIC_ARB: 289 return (vioapic->id); 290 break; 291 default: 292 break; 293 } 294 295 /* redirection table entries */ 296 if (regnum >= IOAPIC_REDTBL && 297 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 298 pin = (regnum - IOAPIC_REDTBL) / 2; 299 if ((regnum - IOAPIC_REDTBL) % 2) 300 rshift = 32; 301 else 302 rshift = 0; 303 304 return (vioapic->rtbl[pin].reg >> rshift); 305 } 306 307 return (0); 308 } 309 310 static void 311 vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr, 312 uint32_t data) 313 { 314 uint64_t data64, mask64; 315 uint64_t last, changed; 316 int regnum, pin, lshift; 317 cpuset_t allvcpus; 318 319 regnum = addr & 0xff; 320 switch (regnum) { 321 case IOAPIC_ID: 322 vioapic->id = data & APIC_ID_MASK; 323 break; 324 case IOAPIC_VER: 325 case IOAPIC_ARB: 326 /* readonly */ 327 break; 328 default: 329 break; 330 } 331 332 /* redirection table entries */ 333 if (regnum >= IOAPIC_REDTBL && 334 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 335 pin = (regnum - IOAPIC_REDTBL) / 2; 336 if ((regnum - IOAPIC_REDTBL) % 2) 337 lshift = 32; 338 else 339 lshift = 0; 340 341 last = vioapic->rtbl[pin].reg; 342 343 data64 = (uint64_t)data << lshift; 344 mask64 = (uint64_t)0xffffffff << lshift; 345 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 346 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 347 348 /* 349 * Switching from level to edge triggering will clear the IRR 350 * bit. This is what FreeBSD will do in order to EOI an 351 * interrupt when the IO-APIC doesn't support targeted EOI (see 352 * _ioapic_eoi_source). 353 */ 354 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 355 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 356 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 357 358 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 359 pin, vioapic->rtbl[pin].reg); 360 361 /* 362 * If any fields in the redirection table entry (except mask 363 * or polarity) have changed then rendezvous all the vcpus 364 * to update their vlapic trigger-mode registers. 365 */ 366 changed = last ^ vioapic->rtbl[pin].reg; 367 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 368 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 369 "vlapic trigger-mode register", pin); 370 VIOAPIC_UNLOCK(vioapic); 371 allvcpus = vm_active_cpus(vioapic->vm); 372 (void)vm_smp_rendezvous(vcpu, allvcpus, 373 vioapic_update_tmr, NULL); 374 VIOAPIC_LOCK(vioapic); 375 } 376 377 /* 378 * Generate an interrupt if the following conditions are met: 379 * - pin trigger mode is level 380 * - pin level is asserted 381 */ 382 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 383 (vioapic->rtbl[pin].acnt > 0)) { 384 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 385 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 386 vioapic_send_intr(vioapic, pin); 387 } 388 } 389 } 390 391 static int 392 vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa, 393 uint64_t *data, int size, bool doread) 394 { 395 uint64_t offset; 396 397 offset = gpa - VIOAPIC_BASE; 398 399 /* 400 * The IOAPIC specification allows 32-bit wide accesses to the 401 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 402 */ 403 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 404 if (doread) 405 *data = 0; 406 return (0); 407 } 408 409 VIOAPIC_LOCK(vioapic); 410 if (offset == IOREGSEL) { 411 if (doread) 412 *data = vioapic->ioregsel; 413 else 414 vioapic->ioregsel = *data; 415 } else { 416 if (doread) { 417 *data = vioapic_read(vioapic, vcpu, 418 vioapic->ioregsel); 419 } else { 420 vioapic_write(vioapic, vcpu, vioapic->ioregsel, 421 *data); 422 } 423 } 424 VIOAPIC_UNLOCK(vioapic); 425 426 return (0); 427 } 428 429 int 430 vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, 431 int size, void *arg) 432 { 433 int error; 434 struct vioapic *vioapic; 435 436 vioapic = vm_ioapic(vcpu_vm(vcpu)); 437 error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true); 438 return (error); 439 } 440 441 int 442 vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, 443 int size, void *arg) 444 { 445 int error; 446 struct vioapic *vioapic; 447 448 vioapic = vm_ioapic(vcpu_vm(vcpu)); 449 error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false); 450 return (error); 451 } 452 453 void 454 vioapic_process_eoi(struct vm *vm, int vector) 455 { 456 struct vioapic *vioapic; 457 int pin; 458 459 KASSERT(vector >= 0 && vector < 256, 460 ("vioapic_process_eoi: invalid vector %d", vector)); 461 462 vioapic = vm_ioapic(vm); 463 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 464 465 /* 466 * XXX keep track of the pins associated with this vector instead 467 * of iterating on every single pin each time. 468 */ 469 VIOAPIC_LOCK(vioapic); 470 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 471 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 472 continue; 473 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 474 continue; 475 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 476 if (vioapic->rtbl[pin].acnt > 0) { 477 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 478 "acnt %d", pin, vioapic->rtbl[pin].acnt); 479 vioapic_send_intr(vioapic, pin); 480 } 481 } 482 VIOAPIC_UNLOCK(vioapic); 483 } 484 485 struct vioapic * 486 vioapic_init(struct vm *vm) 487 { 488 int i; 489 struct vioapic *vioapic; 490 491 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 492 493 vioapic->vm = vm; 494 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 495 496 /* Initialize all redirection entries to mask all interrupts */ 497 for (i = 0; i < REDIR_ENTRIES; i++) 498 vioapic->rtbl[i].reg = 0x0001000000010000UL; 499 500 return (vioapic); 501 } 502 503 void 504 vioapic_cleanup(struct vioapic *vioapic) 505 { 506 507 mtx_destroy(&vioapic->mtx); 508 free(vioapic, M_VIOAPIC); 509 } 510 511 int 512 vioapic_pincount(struct vm *vm) 513 { 514 515 return (REDIR_ENTRIES); 516 } 517 518 #ifdef BHYVE_SNAPSHOT 519 int 520 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta) 521 { 522 int ret; 523 int i; 524 525 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done); 526 527 for (i = 0; i < nitems(vioapic->rtbl); i++) { 528 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done); 529 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done); 530 } 531 532 done: 533 return (ret); 534 } 535 #endif 536