1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_bhyve_snapshot.h" 36 37 #include <sys/param.h> 38 #include <sys/queue.h> 39 #include <sys/lock.h> 40 #include <sys/mutex.h> 41 #include <sys/systm.h> 42 #include <sys/kernel.h> 43 #include <sys/malloc.h> 44 45 #include <x86/apicreg.h> 46 #include <machine/vmm.h> 47 #include <machine/vmm_snapshot.h> 48 49 #include "vmm_ktr.h" 50 #include "vmm_lapic.h" 51 #include "vlapic.h" 52 #include "vioapic.h" 53 54 #define IOREGSEL 0x00 55 #define IOWIN 0x10 56 57 #define REDIR_ENTRIES 32 58 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 59 60 struct vioapic { 61 struct vm *vm; 62 struct mtx mtx; 63 uint32_t id; 64 uint32_t ioregsel; 65 struct { 66 uint64_t reg; 67 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 68 } rtbl[REDIR_ENTRIES]; 69 }; 70 71 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 72 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 73 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 74 75 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 76 77 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 78 VM_CTR1((vioapic)->vm, fmt, a1) 79 80 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 81 VM_CTR2((vioapic)->vm, fmt, a1, a2) 82 83 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 84 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 85 86 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 87 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 88 89 #ifdef KTR 90 static const char * 91 pinstate_str(bool asserted) 92 { 93 94 if (asserted) 95 return ("asserted"); 96 else 97 return ("deasserted"); 98 } 99 #endif 100 101 static void 102 vioapic_send_intr(struct vioapic *vioapic, int pin) 103 { 104 int vector, delmode; 105 uint32_t low, high, dest; 106 bool level, phys; 107 108 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 109 ("vioapic_set_pinstate: invalid pin number %d", pin)); 110 111 KASSERT(VIOAPIC_LOCKED(vioapic), 112 ("vioapic_set_pinstate: vioapic is not locked")); 113 114 low = vioapic->rtbl[pin].reg; 115 high = vioapic->rtbl[pin].reg >> 32; 116 117 if ((low & IOART_INTMASK) == IOART_INTMSET) { 118 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 119 return; 120 } 121 122 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 123 delmode = low & IOART_DELMOD; 124 level = low & IOART_TRGRLVL ? true : false; 125 if (level) { 126 if ((low & IOART_REM_IRR) != 0) { 127 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending", 128 pin); 129 return; 130 } 131 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 132 } 133 134 vector = low & IOART_INTVEC; 135 dest = high >> APIC_ID_SHIFT; 136 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 137 } 138 139 static void 140 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 141 { 142 int oldcnt, newcnt; 143 bool needintr; 144 145 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 146 ("vioapic_set_pinstate: invalid pin number %d", pin)); 147 148 KASSERT(VIOAPIC_LOCKED(vioapic), 149 ("vioapic_set_pinstate: vioapic is not locked")); 150 151 oldcnt = vioapic->rtbl[pin].acnt; 152 if (newstate) 153 vioapic->rtbl[pin].acnt++; 154 else 155 vioapic->rtbl[pin].acnt--; 156 newcnt = vioapic->rtbl[pin].acnt; 157 158 if (newcnt < 0) { 159 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 160 pin, newcnt); 161 } 162 163 needintr = false; 164 if (oldcnt == 0 && newcnt == 1) { 165 needintr = true; 166 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 167 } else if (oldcnt == 1 && newcnt == 0) { 168 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 169 } else { 170 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 171 pin, pinstate_str(newstate), newcnt); 172 } 173 174 if (needintr) 175 vioapic_send_intr(vioapic, pin); 176 } 177 178 enum irqstate { 179 IRQSTATE_ASSERT, 180 IRQSTATE_DEASSERT, 181 IRQSTATE_PULSE 182 }; 183 184 static int 185 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 186 { 187 struct vioapic *vioapic; 188 189 if (irq < 0 || irq >= REDIR_ENTRIES) 190 return (EINVAL); 191 192 vioapic = vm_ioapic(vm); 193 194 VIOAPIC_LOCK(vioapic); 195 switch (irqstate) { 196 case IRQSTATE_ASSERT: 197 vioapic_set_pinstate(vioapic, irq, true); 198 break; 199 case IRQSTATE_DEASSERT: 200 vioapic_set_pinstate(vioapic, irq, false); 201 break; 202 case IRQSTATE_PULSE: 203 vioapic_set_pinstate(vioapic, irq, true); 204 vioapic_set_pinstate(vioapic, irq, false); 205 break; 206 default: 207 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 208 } 209 VIOAPIC_UNLOCK(vioapic); 210 211 return (0); 212 } 213 214 int 215 vioapic_assert_irq(struct vm *vm, int irq) 216 { 217 218 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 219 } 220 221 int 222 vioapic_deassert_irq(struct vm *vm, int irq) 223 { 224 225 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 226 } 227 228 int 229 vioapic_pulse_irq(struct vm *vm, int irq) 230 { 231 232 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 233 } 234 235 /* 236 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 237 * configuration. 238 */ 239 static void 240 vioapic_update_tmr(struct vcpu *vcpu, void *arg) 241 { 242 struct vioapic *vioapic; 243 struct vlapic *vlapic; 244 uint32_t low, high, dest; 245 int delmode, pin, vector; 246 bool level, phys; 247 248 vlapic = vm_lapic(vcpu); 249 vioapic = vm_ioapic(vcpu_vm(vcpu)); 250 251 VIOAPIC_LOCK(vioapic); 252 /* 253 * Reset all vectors to be edge-triggered. 254 */ 255 vlapic_reset_tmr(vlapic); 256 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 257 low = vioapic->rtbl[pin].reg; 258 high = vioapic->rtbl[pin].reg >> 32; 259 260 level = low & IOART_TRGRLVL ? true : false; 261 if (!level) 262 continue; 263 264 /* 265 * For a level-triggered 'pin' let the vlapic figure out if 266 * an assertion on this 'pin' would result in an interrupt 267 * being delivered to it. If yes, then it will modify the 268 * TMR bit associated with this vector to level-triggered. 269 */ 270 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 271 delmode = low & IOART_DELMOD; 272 vector = low & IOART_INTVEC; 273 dest = high >> APIC_ID_SHIFT; 274 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 275 } 276 VIOAPIC_UNLOCK(vioapic); 277 } 278 279 static uint32_t 280 vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr) 281 { 282 int regnum, pin, rshift; 283 284 regnum = addr & 0xff; 285 switch (regnum) { 286 case IOAPIC_ID: 287 return (vioapic->id); 288 break; 289 case IOAPIC_VER: 290 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 291 break; 292 case IOAPIC_ARB: 293 return (vioapic->id); 294 break; 295 default: 296 break; 297 } 298 299 /* redirection table entries */ 300 if (regnum >= IOAPIC_REDTBL && 301 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 302 pin = (regnum - IOAPIC_REDTBL) / 2; 303 if ((regnum - IOAPIC_REDTBL) % 2) 304 rshift = 32; 305 else 306 rshift = 0; 307 308 return (vioapic->rtbl[pin].reg >> rshift); 309 } 310 311 return (0); 312 } 313 314 static void 315 vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr, 316 uint32_t data) 317 { 318 uint64_t data64, mask64; 319 uint64_t last, changed; 320 int regnum, pin, lshift; 321 cpuset_t allvcpus; 322 323 regnum = addr & 0xff; 324 switch (regnum) { 325 case IOAPIC_ID: 326 vioapic->id = data & APIC_ID_MASK; 327 break; 328 case IOAPIC_VER: 329 case IOAPIC_ARB: 330 /* readonly */ 331 break; 332 default: 333 break; 334 } 335 336 /* redirection table entries */ 337 if (regnum >= IOAPIC_REDTBL && 338 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 339 pin = (regnum - IOAPIC_REDTBL) / 2; 340 if ((regnum - IOAPIC_REDTBL) % 2) 341 lshift = 32; 342 else 343 lshift = 0; 344 345 last = vioapic->rtbl[pin].reg; 346 347 data64 = (uint64_t)data << lshift; 348 mask64 = (uint64_t)0xffffffff << lshift; 349 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 350 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 351 352 /* 353 * Switching from level to edge triggering will clear the IRR 354 * bit. This is what FreeBSD will do in order to EOI an 355 * interrupt when the IO-APIC doesn't support targeted EOI (see 356 * _ioapic_eoi_source). 357 */ 358 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 359 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 360 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 361 362 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 363 pin, vioapic->rtbl[pin].reg); 364 365 /* 366 * If any fields in the redirection table entry (except mask 367 * or polarity) have changed then rendezvous all the vcpus 368 * to update their vlapic trigger-mode registers. 369 */ 370 changed = last ^ vioapic->rtbl[pin].reg; 371 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 372 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 373 "vlapic trigger-mode register", pin); 374 VIOAPIC_UNLOCK(vioapic); 375 allvcpus = vm_active_cpus(vioapic->vm); 376 (void)vm_smp_rendezvous(vcpu, allvcpus, 377 vioapic_update_tmr, NULL); 378 VIOAPIC_LOCK(vioapic); 379 } 380 381 /* 382 * Generate an interrupt if the following conditions are met: 383 * - pin trigger mode is level 384 * - pin level is asserted 385 */ 386 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 387 (vioapic->rtbl[pin].acnt > 0)) { 388 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 389 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 390 vioapic_send_intr(vioapic, pin); 391 } 392 } 393 } 394 395 static int 396 vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa, 397 uint64_t *data, int size, bool doread) 398 { 399 uint64_t offset; 400 401 offset = gpa - VIOAPIC_BASE; 402 403 /* 404 * The IOAPIC specification allows 32-bit wide accesses to the 405 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 406 */ 407 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 408 if (doread) 409 *data = 0; 410 return (0); 411 } 412 413 VIOAPIC_LOCK(vioapic); 414 if (offset == IOREGSEL) { 415 if (doread) 416 *data = vioapic->ioregsel; 417 else 418 vioapic->ioregsel = *data; 419 } else { 420 if (doread) { 421 *data = vioapic_read(vioapic, vcpu, 422 vioapic->ioregsel); 423 } else { 424 vioapic_write(vioapic, vcpu, vioapic->ioregsel, 425 *data); 426 } 427 } 428 VIOAPIC_UNLOCK(vioapic); 429 430 return (0); 431 } 432 433 int 434 vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, 435 int size, void *arg) 436 { 437 int error; 438 struct vioapic *vioapic; 439 440 vioapic = vm_ioapic(vcpu_vm(vcpu)); 441 error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true); 442 return (error); 443 } 444 445 int 446 vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, 447 int size, void *arg) 448 { 449 int error; 450 struct vioapic *vioapic; 451 452 vioapic = vm_ioapic(vcpu_vm(vcpu)); 453 error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false); 454 return (error); 455 } 456 457 void 458 vioapic_process_eoi(struct vm *vm, int vector) 459 { 460 struct vioapic *vioapic; 461 int pin; 462 463 KASSERT(vector >= 0 && vector < 256, 464 ("vioapic_process_eoi: invalid vector %d", vector)); 465 466 vioapic = vm_ioapic(vm); 467 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 468 469 /* 470 * XXX keep track of the pins associated with this vector instead 471 * of iterating on every single pin each time. 472 */ 473 VIOAPIC_LOCK(vioapic); 474 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 475 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 476 continue; 477 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 478 continue; 479 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 480 if (vioapic->rtbl[pin].acnt > 0) { 481 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 482 "acnt %d", pin, vioapic->rtbl[pin].acnt); 483 vioapic_send_intr(vioapic, pin); 484 } 485 } 486 VIOAPIC_UNLOCK(vioapic); 487 } 488 489 struct vioapic * 490 vioapic_init(struct vm *vm) 491 { 492 int i; 493 struct vioapic *vioapic; 494 495 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 496 497 vioapic->vm = vm; 498 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 499 500 /* Initialize all redirection entries to mask all interrupts */ 501 for (i = 0; i < REDIR_ENTRIES; i++) 502 vioapic->rtbl[i].reg = 0x0001000000010000UL; 503 504 return (vioapic); 505 } 506 507 void 508 vioapic_cleanup(struct vioapic *vioapic) 509 { 510 511 mtx_destroy(&vioapic->mtx); 512 free(vioapic, M_VIOAPIC); 513 } 514 515 int 516 vioapic_pincount(struct vm *vm) 517 { 518 519 return (REDIR_ENTRIES); 520 } 521 522 #ifdef BHYVE_SNAPSHOT 523 int 524 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta) 525 { 526 int ret; 527 int i; 528 529 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done); 530 531 for (i = 0; i < nitems(vioapic->rtbl); i++) { 532 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done); 533 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done); 534 } 535 536 done: 537 return (ret); 538 } 539 #endif 540