1 /*- 2 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 3 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/queue.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 41 #include <x86/apicreg.h> 42 #include <machine/vmm.h> 43 44 #include "vmm_ktr.h" 45 #include "vmm_lapic.h" 46 #include "vlapic.h" 47 #include "vioapic.h" 48 49 #define IOREGSEL 0x00 50 #define IOWIN 0x10 51 52 #define REDIR_ENTRIES 24 53 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 54 55 struct vioapic { 56 struct vm *vm; 57 struct mtx mtx; 58 uint32_t id; 59 uint32_t ioregsel; 60 struct { 61 uint64_t reg; 62 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 63 } rtbl[REDIR_ENTRIES]; 64 }; 65 66 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 67 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 68 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 69 70 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 71 72 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 73 VM_CTR1((vioapic)->vm, fmt, a1) 74 75 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 76 VM_CTR2((vioapic)->vm, fmt, a1, a2) 77 78 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 79 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 80 81 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 82 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 83 84 #ifdef KTR 85 static const char * 86 pinstate_str(bool asserted) 87 { 88 89 if (asserted) 90 return ("asserted"); 91 else 92 return ("deasserted"); 93 } 94 #endif 95 96 static void 97 vioapic_send_intr(struct vioapic *vioapic, int pin) 98 { 99 int vector, delmode; 100 uint32_t low, high, dest; 101 bool level, phys; 102 103 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 104 ("vioapic_set_pinstate: invalid pin number %d", pin)); 105 106 KASSERT(VIOAPIC_LOCKED(vioapic), 107 ("vioapic_set_pinstate: vioapic is not locked")); 108 109 low = vioapic->rtbl[pin].reg; 110 high = vioapic->rtbl[pin].reg >> 32; 111 112 if ((low & IOART_INTMASK) == IOART_INTMSET) { 113 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 114 return; 115 } 116 117 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 118 delmode = low & IOART_DELMOD; 119 level = low & IOART_TRGRLVL ? true : false; 120 if (level) 121 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 122 123 vector = low & IOART_INTVEC; 124 dest = high >> APIC_ID_SHIFT; 125 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 126 } 127 128 static void 129 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 130 { 131 int oldcnt, newcnt; 132 bool needintr; 133 134 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 135 ("vioapic_set_pinstate: invalid pin number %d", pin)); 136 137 KASSERT(VIOAPIC_LOCKED(vioapic), 138 ("vioapic_set_pinstate: vioapic is not locked")); 139 140 oldcnt = vioapic->rtbl[pin].acnt; 141 if (newstate) 142 vioapic->rtbl[pin].acnt++; 143 else 144 vioapic->rtbl[pin].acnt--; 145 newcnt = vioapic->rtbl[pin].acnt; 146 147 if (newcnt < 0) { 148 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 149 pin, newcnt); 150 } 151 152 needintr = false; 153 if (oldcnt == 0 && newcnt == 1) { 154 needintr = true; 155 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 156 } else if (oldcnt == 1 && newcnt == 0) { 157 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 158 } else { 159 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 160 pin, pinstate_str(newstate), newcnt); 161 } 162 163 if (needintr) 164 vioapic_send_intr(vioapic, pin); 165 } 166 167 enum irqstate { 168 IRQSTATE_ASSERT, 169 IRQSTATE_DEASSERT, 170 IRQSTATE_PULSE 171 }; 172 173 static int 174 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 175 { 176 struct vioapic *vioapic; 177 178 if (irq < 0 || irq >= REDIR_ENTRIES) 179 return (EINVAL); 180 181 vioapic = vm_ioapic(vm); 182 183 VIOAPIC_LOCK(vioapic); 184 switch (irqstate) { 185 case IRQSTATE_ASSERT: 186 vioapic_set_pinstate(vioapic, irq, true); 187 break; 188 case IRQSTATE_DEASSERT: 189 vioapic_set_pinstate(vioapic, irq, false); 190 break; 191 case IRQSTATE_PULSE: 192 vioapic_set_pinstate(vioapic, irq, true); 193 vioapic_set_pinstate(vioapic, irq, false); 194 break; 195 default: 196 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 197 } 198 VIOAPIC_UNLOCK(vioapic); 199 200 return (0); 201 } 202 203 int 204 vioapic_assert_irq(struct vm *vm, int irq) 205 { 206 207 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 208 } 209 210 int 211 vioapic_deassert_irq(struct vm *vm, int irq) 212 { 213 214 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 215 } 216 217 int 218 vioapic_pulse_irq(struct vm *vm, int irq) 219 { 220 221 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 222 } 223 224 /* 225 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 226 * configuration. 227 */ 228 static void 229 vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg) 230 { 231 struct vioapic *vioapic; 232 struct vlapic *vlapic; 233 uint32_t low, high, dest; 234 int delmode, pin, vector; 235 bool level, phys; 236 237 vlapic = vm_lapic(vm, vcpuid); 238 vioapic = vm_ioapic(vm); 239 240 VIOAPIC_LOCK(vioapic); 241 /* 242 * Reset all vectors to be edge-triggered. 243 */ 244 vlapic_reset_tmr(vlapic); 245 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 246 low = vioapic->rtbl[pin].reg; 247 high = vioapic->rtbl[pin].reg >> 32; 248 249 level = low & IOART_TRGRLVL ? true : false; 250 if (!level) 251 continue; 252 253 /* 254 * For a level-triggered 'pin' let the vlapic figure out if 255 * an assertion on this 'pin' would result in an interrupt 256 * being delivered to it. If yes, then it will modify the 257 * TMR bit associated with this vector to level-triggered. 258 */ 259 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 260 delmode = low & IOART_DELMOD; 261 vector = low & IOART_INTVEC; 262 dest = high >> APIC_ID_SHIFT; 263 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 264 } 265 VIOAPIC_UNLOCK(vioapic); 266 } 267 268 static uint32_t 269 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) 270 { 271 int regnum, pin, rshift; 272 273 regnum = addr & 0xff; 274 switch (regnum) { 275 case IOAPIC_ID: 276 return (vioapic->id); 277 break; 278 case IOAPIC_VER: 279 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 280 break; 281 case IOAPIC_ARB: 282 return (vioapic->id); 283 break; 284 default: 285 break; 286 } 287 288 /* redirection table entries */ 289 if (regnum >= IOAPIC_REDTBL && 290 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 291 pin = (regnum - IOAPIC_REDTBL) / 2; 292 if ((regnum - IOAPIC_REDTBL) % 2) 293 rshift = 32; 294 else 295 rshift = 0; 296 297 return (vioapic->rtbl[pin].reg >> rshift); 298 } 299 300 return (0); 301 } 302 303 static void 304 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) 305 { 306 uint64_t data64, mask64; 307 uint64_t last, changed; 308 int regnum, pin, lshift; 309 cpuset_t allvcpus; 310 311 regnum = addr & 0xff; 312 switch (regnum) { 313 case IOAPIC_ID: 314 vioapic->id = data & APIC_ID_MASK; 315 break; 316 case IOAPIC_VER: 317 case IOAPIC_ARB: 318 /* readonly */ 319 break; 320 default: 321 break; 322 } 323 324 /* redirection table entries */ 325 if (regnum >= IOAPIC_REDTBL && 326 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 327 pin = (regnum - IOAPIC_REDTBL) / 2; 328 if ((regnum - IOAPIC_REDTBL) % 2) 329 lshift = 32; 330 else 331 lshift = 0; 332 333 last = vioapic->rtbl[pin].reg; 334 335 data64 = (uint64_t)data << lshift; 336 mask64 = (uint64_t)0xffffffff << lshift; 337 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 338 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 339 340 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 341 pin, vioapic->rtbl[pin].reg); 342 343 /* 344 * If any fields in the redirection table entry (except mask 345 * or polarity) have changed then rendezvous all the vcpus 346 * to update their vlapic trigger-mode registers. 347 */ 348 changed = last ^ vioapic->rtbl[pin].reg; 349 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 350 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 351 "vlapic trigger-mode register", pin); 352 VIOAPIC_UNLOCK(vioapic); 353 allvcpus = vm_active_cpus(vioapic->vm); 354 vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus, 355 vioapic_update_tmr, NULL); 356 VIOAPIC_LOCK(vioapic); 357 } 358 359 /* 360 * Generate an interrupt if the following conditions are met: 361 * - pin is not masked 362 * - previous interrupt has been EOIed 363 * - pin level is asserted 364 */ 365 if ((vioapic->rtbl[pin].reg & IOART_INTMASK) == IOART_INTMCLR && 366 (vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0 && 367 (vioapic->rtbl[pin].acnt > 0)) { 368 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 369 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 370 vioapic_send_intr(vioapic, pin); 371 } 372 } 373 } 374 375 static int 376 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa, 377 uint64_t *data, int size, bool doread) 378 { 379 uint64_t offset; 380 381 offset = gpa - VIOAPIC_BASE; 382 383 /* 384 * The IOAPIC specification allows 32-bit wide accesses to the 385 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 386 */ 387 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 388 if (doread) 389 *data = 0; 390 return (0); 391 } 392 393 VIOAPIC_LOCK(vioapic); 394 if (offset == IOREGSEL) { 395 if (doread) 396 *data = vioapic->ioregsel; 397 else 398 vioapic->ioregsel = *data; 399 } else { 400 if (doread) { 401 *data = vioapic_read(vioapic, vcpuid, 402 vioapic->ioregsel); 403 } else { 404 vioapic_write(vioapic, vcpuid, vioapic->ioregsel, 405 *data); 406 } 407 } 408 VIOAPIC_UNLOCK(vioapic); 409 410 return (0); 411 } 412 413 int 414 vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval, 415 int size, void *arg) 416 { 417 int error; 418 struct vioapic *vioapic; 419 420 vioapic = vm_ioapic(vm); 421 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true); 422 return (error); 423 } 424 425 int 426 vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval, 427 int size, void *arg) 428 { 429 int error; 430 struct vioapic *vioapic; 431 432 vioapic = vm_ioapic(vm); 433 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false); 434 return (error); 435 } 436 437 void 438 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector) 439 { 440 struct vioapic *vioapic; 441 int pin; 442 443 KASSERT(vector >= 0 && vector < 256, 444 ("vioapic_process_eoi: invalid vector %d", vector)); 445 446 vioapic = vm_ioapic(vm); 447 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 448 449 /* 450 * XXX keep track of the pins associated with this vector instead 451 * of iterating on every single pin each time. 452 */ 453 VIOAPIC_LOCK(vioapic); 454 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 455 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 456 continue; 457 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 458 continue; 459 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 460 if (vioapic->rtbl[pin].acnt > 0) { 461 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 462 "acnt %d", pin, vioapic->rtbl[pin].acnt); 463 vioapic_send_intr(vioapic, pin); 464 } 465 } 466 VIOAPIC_UNLOCK(vioapic); 467 } 468 469 struct vioapic * 470 vioapic_init(struct vm *vm) 471 { 472 int i; 473 struct vioapic *vioapic; 474 475 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 476 477 vioapic->vm = vm; 478 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 479 480 /* Initialize all redirection entries to mask all interrupts */ 481 for (i = 0; i < REDIR_ENTRIES; i++) 482 vioapic->rtbl[i].reg = 0x0001000000010000UL; 483 484 return (vioapic); 485 } 486 487 void 488 vioapic_cleanup(struct vioapic *vioapic) 489 { 490 491 free(vioapic, M_VIOAPIC); 492 } 493 494 int 495 vioapic_pincount(struct vm *vm) 496 { 497 498 return (REDIR_ENTRIES); 499 } 500