1 /*- 2 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 3 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/queue.h> 35 #include <sys/cpuset.h> 36 #include <sys/lock.h> 37 #include <sys/mutex.h> 38 #include <sys/systm.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 42 #include <x86/apicreg.h> 43 #include <machine/vmm.h> 44 45 #include "vmm_ktr.h" 46 #include "vmm_lapic.h" 47 #include "vlapic.h" 48 #include "vioapic.h" 49 50 #define IOREGSEL 0x00 51 #define IOWIN 0x10 52 53 #define REDIR_ENTRIES 24 54 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 55 56 struct vioapic { 57 struct vm *vm; 58 struct mtx mtx; 59 uint32_t id; 60 uint32_t ioregsel; 61 struct { 62 uint64_t reg; 63 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 64 } rtbl[REDIR_ENTRIES]; 65 }; 66 67 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 68 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 69 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 70 71 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 72 73 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 74 VM_CTR1((vioapic)->vm, fmt, a1) 75 76 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 77 VM_CTR2((vioapic)->vm, fmt, a1, a2) 78 79 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 80 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 81 82 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 83 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 84 85 #ifdef KTR 86 static const char * 87 pinstate_str(bool asserted) 88 { 89 90 if (asserted) 91 return ("asserted"); 92 else 93 return ("deasserted"); 94 } 95 #endif 96 97 static void 98 vioapic_send_intr(struct vioapic *vioapic, int pin) 99 { 100 int vector, delmode; 101 uint32_t low, high, dest; 102 bool level, phys; 103 104 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 105 ("vioapic_set_pinstate: invalid pin number %d", pin)); 106 107 KASSERT(VIOAPIC_LOCKED(vioapic), 108 ("vioapic_set_pinstate: vioapic is not locked")); 109 110 low = vioapic->rtbl[pin].reg; 111 high = vioapic->rtbl[pin].reg >> 32; 112 113 if ((low & IOART_INTMASK) == IOART_INTMSET) { 114 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 115 return; 116 } 117 118 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 119 delmode = low & IOART_DELMOD; 120 level = low & IOART_TRGRLVL ? true : false; 121 if (level) 122 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 123 124 vector = low & IOART_INTVEC; 125 dest = high >> APIC_ID_SHIFT; 126 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 127 } 128 129 static void 130 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 131 { 132 int oldcnt, newcnt; 133 bool needintr; 134 135 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 136 ("vioapic_set_pinstate: invalid pin number %d", pin)); 137 138 KASSERT(VIOAPIC_LOCKED(vioapic), 139 ("vioapic_set_pinstate: vioapic is not locked")); 140 141 oldcnt = vioapic->rtbl[pin].acnt; 142 if (newstate) 143 vioapic->rtbl[pin].acnt++; 144 else 145 vioapic->rtbl[pin].acnt--; 146 newcnt = vioapic->rtbl[pin].acnt; 147 148 if (newcnt < 0) { 149 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 150 pin, newcnt); 151 } 152 153 needintr = false; 154 if (oldcnt == 0 && newcnt == 1) { 155 needintr = true; 156 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 157 } else if (oldcnt == 1 && newcnt == 0) { 158 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 159 } else { 160 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 161 pin, pinstate_str(newstate), newcnt); 162 } 163 164 if (needintr) 165 vioapic_send_intr(vioapic, pin); 166 } 167 168 enum irqstate { 169 IRQSTATE_ASSERT, 170 IRQSTATE_DEASSERT, 171 IRQSTATE_PULSE 172 }; 173 174 static int 175 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 176 { 177 struct vioapic *vioapic; 178 179 if (irq < 0 || irq >= REDIR_ENTRIES) 180 return (EINVAL); 181 182 vioapic = vm_ioapic(vm); 183 184 VIOAPIC_LOCK(vioapic); 185 switch (irqstate) { 186 case IRQSTATE_ASSERT: 187 vioapic_set_pinstate(vioapic, irq, true); 188 break; 189 case IRQSTATE_DEASSERT: 190 vioapic_set_pinstate(vioapic, irq, false); 191 break; 192 case IRQSTATE_PULSE: 193 vioapic_set_pinstate(vioapic, irq, true); 194 vioapic_set_pinstate(vioapic, irq, false); 195 break; 196 default: 197 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 198 } 199 VIOAPIC_UNLOCK(vioapic); 200 201 return (0); 202 } 203 204 int 205 vioapic_assert_irq(struct vm *vm, int irq) 206 { 207 208 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 209 } 210 211 int 212 vioapic_deassert_irq(struct vm *vm, int irq) 213 { 214 215 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 216 } 217 218 int 219 vioapic_pulse_irq(struct vm *vm, int irq) 220 { 221 222 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 223 } 224 225 /* 226 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 227 * configuration. 228 */ 229 static void 230 vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg) 231 { 232 struct vioapic *vioapic; 233 struct vlapic *vlapic; 234 uint32_t low, high, dest; 235 int delmode, pin, vector; 236 bool level, phys; 237 238 vlapic = vm_lapic(vm, vcpuid); 239 vioapic = vm_ioapic(vm); 240 241 VIOAPIC_LOCK(vioapic); 242 /* 243 * Reset all vectors to be edge-triggered. 244 */ 245 vlapic_reset_tmr(vlapic); 246 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 247 low = vioapic->rtbl[pin].reg; 248 high = vioapic->rtbl[pin].reg >> 32; 249 250 level = low & IOART_TRGRLVL ? true : false; 251 if (!level) 252 continue; 253 254 /* 255 * For a level-triggered 'pin' let the vlapic figure out if 256 * an assertion on this 'pin' would result in an interrupt 257 * being delivered to it. If yes, then it will modify the 258 * TMR bit associated with this vector to level-triggered. 259 */ 260 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 261 delmode = low & IOART_DELMOD; 262 vector = low & IOART_INTVEC; 263 dest = high >> APIC_ID_SHIFT; 264 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 265 } 266 VIOAPIC_UNLOCK(vioapic); 267 } 268 269 static uint32_t 270 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) 271 { 272 int regnum, pin, rshift; 273 274 regnum = addr & 0xff; 275 switch (regnum) { 276 case IOAPIC_ID: 277 return (vioapic->id); 278 break; 279 case IOAPIC_VER: 280 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 281 break; 282 case IOAPIC_ARB: 283 return (vioapic->id); 284 break; 285 default: 286 break; 287 } 288 289 /* redirection table entries */ 290 if (regnum >= IOAPIC_REDTBL && 291 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 292 pin = (regnum - IOAPIC_REDTBL) / 2; 293 if ((regnum - IOAPIC_REDTBL) % 2) 294 rshift = 32; 295 else 296 rshift = 0; 297 298 return (vioapic->rtbl[pin].reg >> rshift); 299 } 300 301 return (0); 302 } 303 304 static void 305 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) 306 { 307 uint64_t data64, mask64; 308 uint64_t last, changed; 309 int regnum, pin, lshift; 310 cpuset_t allvcpus; 311 312 regnum = addr & 0xff; 313 switch (regnum) { 314 case IOAPIC_ID: 315 vioapic->id = data & APIC_ID_MASK; 316 break; 317 case IOAPIC_VER: 318 case IOAPIC_ARB: 319 /* readonly */ 320 break; 321 default: 322 break; 323 } 324 325 /* redirection table entries */ 326 if (regnum >= IOAPIC_REDTBL && 327 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 328 pin = (regnum - IOAPIC_REDTBL) / 2; 329 if ((regnum - IOAPIC_REDTBL) % 2) 330 lshift = 32; 331 else 332 lshift = 0; 333 334 last = vioapic->rtbl[pin].reg; 335 336 data64 = (uint64_t)data << lshift; 337 mask64 = (uint64_t)0xffffffff << lshift; 338 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 339 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 340 341 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 342 pin, vioapic->rtbl[pin].reg); 343 344 /* 345 * If any fields in the redirection table entry (except mask 346 * or polarity) have changed then rendezvous all the vcpus 347 * to update their vlapic trigger-mode registers. 348 */ 349 changed = last ^ vioapic->rtbl[pin].reg; 350 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 351 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 352 "vlapic trigger-mode register", pin); 353 VIOAPIC_UNLOCK(vioapic); 354 allvcpus = vm_active_cpus(vioapic->vm); 355 vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus, 356 vioapic_update_tmr, NULL); 357 VIOAPIC_LOCK(vioapic); 358 } 359 360 /* 361 * Generate an interrupt if the following conditions are met: 362 * - pin is not masked 363 * - previous interrupt has been EOIed 364 * - pin level is asserted 365 */ 366 if ((vioapic->rtbl[pin].reg & IOART_INTMASK) == IOART_INTMCLR && 367 (vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0 && 368 (vioapic->rtbl[pin].acnt > 0)) { 369 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 370 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 371 vioapic_send_intr(vioapic, pin); 372 } 373 } 374 } 375 376 static int 377 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa, 378 uint64_t *data, int size, bool doread) 379 { 380 uint64_t offset; 381 382 offset = gpa - VIOAPIC_BASE; 383 384 /* 385 * The IOAPIC specification allows 32-bit wide accesses to the 386 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 387 */ 388 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 389 if (doread) 390 *data = 0; 391 return (0); 392 } 393 394 VIOAPIC_LOCK(vioapic); 395 if (offset == IOREGSEL) { 396 if (doread) 397 *data = vioapic->ioregsel; 398 else 399 vioapic->ioregsel = *data; 400 } else { 401 if (doread) { 402 *data = vioapic_read(vioapic, vcpuid, 403 vioapic->ioregsel); 404 } else { 405 vioapic_write(vioapic, vcpuid, vioapic->ioregsel, 406 *data); 407 } 408 } 409 VIOAPIC_UNLOCK(vioapic); 410 411 return (0); 412 } 413 414 int 415 vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval, 416 int size, void *arg) 417 { 418 int error; 419 struct vioapic *vioapic; 420 421 vioapic = vm_ioapic(vm); 422 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true); 423 return (error); 424 } 425 426 int 427 vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval, 428 int size, void *arg) 429 { 430 int error; 431 struct vioapic *vioapic; 432 433 vioapic = vm_ioapic(vm); 434 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false); 435 return (error); 436 } 437 438 void 439 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector) 440 { 441 struct vioapic *vioapic; 442 int pin; 443 444 KASSERT(vector >= 0 && vector < 256, 445 ("vioapic_process_eoi: invalid vector %d", vector)); 446 447 vioapic = vm_ioapic(vm); 448 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 449 450 /* 451 * XXX keep track of the pins associated with this vector instead 452 * of iterating on every single pin each time. 453 */ 454 VIOAPIC_LOCK(vioapic); 455 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 456 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 457 continue; 458 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 459 continue; 460 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 461 if (vioapic->rtbl[pin].acnt > 0) { 462 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 463 "acnt %d", pin, vioapic->rtbl[pin].acnt); 464 vioapic_send_intr(vioapic, pin); 465 } 466 } 467 VIOAPIC_UNLOCK(vioapic); 468 } 469 470 struct vioapic * 471 vioapic_init(struct vm *vm) 472 { 473 int i; 474 struct vioapic *vioapic; 475 476 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 477 478 vioapic->vm = vm; 479 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 480 481 /* Initialize all redirection entries to mask all interrupts */ 482 for (i = 0; i < REDIR_ENTRIES; i++) 483 vioapic->rtbl[i].reg = 0x0001000000010000UL; 484 485 return (vioapic); 486 } 487 488 void 489 vioapic_cleanup(struct vioapic *vioapic) 490 { 491 492 free(vioapic, M_VIOAPIC); 493 } 494 495 int 496 vioapic_pincount(struct vm *vm) 497 { 498 499 return (REDIR_ENTRIES); 500 } 501