1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 /* 32 * This file and its contents are supplied under the terms of the 33 * Common Development and Distribution License ("CDDL"), version 1.0. 34 * You may only use this file in accordance with the terms of version 35 * 1.0 of the CDDL. 36 * 37 * A full copy of the text of the CDDL should have accompanied this 38 * source. A copy of the CDDL is also available via the Internet at 39 * http://www.illumos.org/license/CDDL. 40 * 41 * Copyright 2014 Pluribus Networks Inc. 42 * Copyright 2017 Joyent, Inc. 43 * Copyright 2021 Oxide Computer Company 44 */ 45 46 #include <sys/cdefs.h> 47 __FBSDID("$FreeBSD$"); 48 49 #include <sys/param.h> 50 #include <sys/queue.h> 51 #include <sys/mutex.h> 52 #include <sys/systm.h> 53 #include <sys/kernel.h> 54 #include <sys/malloc.h> 55 #include <sys/cpuset.h> 56 57 #include <x86/apicreg.h> 58 #include <machine/vmm.h> 59 60 #include "vmm_ktr.h" 61 #include "vmm_lapic.h" 62 #include "vlapic.h" 63 #include "vioapic.h" 64 65 #define IOREGSEL 0x00 66 #define IOWIN 0x10 67 68 #define REDIR_ENTRIES 32 69 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 70 71 struct ioapic_stats { 72 uint64_t is_interrupts; 73 uint64_t is_saturate_low; 74 uint64_t is_saturate_high; 75 }; 76 77 struct vioapic { 78 struct vm *vm; 79 kmutex_t lock; 80 uint32_t id; 81 uint32_t ioregsel; 82 struct { 83 uint64_t reg; 84 /* 85 * The sum of pin asserts (+1) and deasserts (-1) are tracked in 86 * 'acnt'. It is clamped to prevent overflow or underflow 87 * should emulation consumers feed it an invalid set of 88 * transitions. 89 */ 90 uint_t acnt; 91 } rtbl[REDIR_ENTRIES]; 92 struct ioapic_stats stats; 93 }; 94 95 #define VIOAPIC_LOCK(vioapic) mutex_enter(&((vioapic)->lock)) 96 #define VIOAPIC_UNLOCK(vioapic) mutex_exit(&((vioapic)->lock)) 97 #define VIOAPIC_LOCKED(vioapic) MUTEX_HELD(&((vioapic)->lock)) 98 99 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 100 101 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 102 VM_CTR1((vioapic)->vm, fmt, a1) 103 104 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 105 VM_CTR2((vioapic)->vm, fmt, a1, a2) 106 107 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 108 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 109 110 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 111 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 112 113 static void 114 vioapic_send_intr(struct vioapic *vioapic, int pin) 115 { 116 int vector, delmode; 117 uint32_t low, high, dest; 118 bool level, phys; 119 120 VERIFY(pin >= 0 && pin < REDIR_ENTRIES); 121 ASSERT(VIOAPIC_LOCKED(vioapic)); 122 123 low = vioapic->rtbl[pin].reg; 124 high = vioapic->rtbl[pin].reg >> 32; 125 126 if ((low & IOART_INTMASK) == IOART_INTMSET) { 127 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 128 return; 129 } 130 131 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 132 delmode = low & IOART_DELMOD; 133 level = low & IOART_TRGRLVL ? true : false; 134 if (level) { 135 if ((low & IOART_REM_IRR) != 0) { 136 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending", 137 pin); 138 return; 139 } 140 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 141 } 142 143 vector = low & IOART_INTVEC; 144 dest = high >> APIC_ID_SHIFT; 145 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 146 vioapic->stats.is_interrupts++; 147 } 148 149 static int 150 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 151 { 152 uint_t oldcnt, newcnt; 153 bool needintr = false; 154 int err = 0; 155 156 VERIFY(pin >= 0 && pin < REDIR_ENTRIES); 157 ASSERT(VIOAPIC_LOCKED(vioapic)); 158 159 oldcnt = newcnt = vioapic->rtbl[pin].acnt; 160 if (newstate) { 161 if (newcnt != UINT_MAX) { 162 newcnt++; 163 } else { 164 err = E2BIG; 165 DTRACE_PROBE2(vioapic__sat_high, 166 struct vioapic *, vioapic, int, pin); 167 vioapic->stats.is_saturate_high++; 168 } 169 } else { 170 if (newcnt != 0) { 171 newcnt--; 172 } else { 173 err = ERANGE; 174 DTRACE_PROBE2(vioapic__sat_low, 175 struct vioapic *, vioapic, int, pin); 176 vioapic->stats.is_saturate_low++; 177 } 178 } 179 vioapic->rtbl[pin].acnt = newcnt; 180 181 if (oldcnt == 0 && newcnt == 1) { 182 needintr = true; 183 DTRACE_PROBE2(vioapic__assert, struct vioapic *, vioapic, 184 int, pin); 185 } else if (oldcnt == 1 && newcnt == 0) { 186 DTRACE_PROBE2(vioapic__deassert, struct vioapic *, vioapic, 187 int, pin); 188 } 189 190 if (needintr) { 191 vioapic_send_intr(vioapic, pin); 192 } 193 return (err); 194 } 195 196 enum irqstate { 197 IRQSTATE_ASSERT, 198 IRQSTATE_DEASSERT, 199 IRQSTATE_PULSE 200 }; 201 202 static int 203 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 204 { 205 struct vioapic *vioapic; 206 int err = 0; 207 208 if (irq < 0 || irq >= REDIR_ENTRIES) 209 return (EINVAL); 210 211 vioapic = vm_ioapic(vm); 212 213 VIOAPIC_LOCK(vioapic); 214 switch (irqstate) { 215 case IRQSTATE_ASSERT: 216 err = vioapic_set_pinstate(vioapic, irq, true); 217 break; 218 case IRQSTATE_DEASSERT: 219 err = vioapic_set_pinstate(vioapic, irq, false); 220 break; 221 case IRQSTATE_PULSE: 222 err = vioapic_set_pinstate(vioapic, irq, true); 223 if (err == 0) { 224 err = vioapic_set_pinstate(vioapic, irq, false); 225 } 226 break; 227 default: 228 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 229 } 230 VIOAPIC_UNLOCK(vioapic); 231 232 return (err); 233 } 234 235 int 236 vioapic_assert_irq(struct vm *vm, int irq) 237 { 238 239 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 240 } 241 242 int 243 vioapic_deassert_irq(struct vm *vm, int irq) 244 { 245 246 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 247 } 248 249 int 250 vioapic_pulse_irq(struct vm *vm, int irq) 251 { 252 253 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 254 } 255 256 static uint32_t 257 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) 258 { 259 int regnum, pin, rshift; 260 261 regnum = addr & 0xff; 262 switch (regnum) { 263 case IOAPIC_ID: 264 return (vioapic->id); 265 break; 266 case IOAPIC_VER: 267 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 268 break; 269 case IOAPIC_ARB: 270 return (vioapic->id); 271 break; 272 default: 273 break; 274 } 275 276 /* redirection table entries */ 277 if (regnum >= IOAPIC_REDTBL && 278 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 279 pin = (regnum - IOAPIC_REDTBL) / 2; 280 if ((regnum - IOAPIC_REDTBL) % 2) 281 rshift = 32; 282 else 283 rshift = 0; 284 285 return (vioapic->rtbl[pin].reg >> rshift); 286 } 287 288 return (0); 289 } 290 291 static void 292 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) 293 { 294 uint64_t data64, mask64; 295 int regnum, pin, lshift; 296 297 regnum = addr & 0xff; 298 switch (regnum) { 299 case IOAPIC_ID: 300 vioapic->id = data & APIC_ID_MASK; 301 break; 302 case IOAPIC_VER: 303 case IOAPIC_ARB: 304 /* readonly */ 305 break; 306 default: 307 break; 308 } 309 310 /* redirection table entries */ 311 if (regnum >= IOAPIC_REDTBL && 312 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 313 pin = (regnum - IOAPIC_REDTBL) / 2; 314 if ((regnum - IOAPIC_REDTBL) % 2) 315 lshift = 32; 316 else 317 lshift = 0; 318 319 data64 = (uint64_t)data << lshift; 320 mask64 = (uint64_t)0xffffffff << lshift; 321 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 322 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 323 324 /* 325 * Switching from level to edge triggering will clear the IRR 326 * bit. This is what FreeBSD will do in order to EOI an 327 * interrupt when the IO-APIC doesn't support targeted EOI (see 328 * _ioapic_eoi_source). 329 */ 330 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 331 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 332 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 333 334 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 335 pin, vioapic->rtbl[pin].reg); 336 337 /* 338 * Generate an interrupt if the following conditions are met: 339 * - pin trigger mode is level 340 * - pin level is asserted 341 */ 342 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 343 (vioapic->rtbl[pin].acnt > 0)) { 344 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 345 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 346 vioapic_send_intr(vioapic, pin); 347 } 348 } 349 } 350 351 static int 352 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa, 353 uint64_t *data, int size, bool doread) 354 { 355 uint64_t offset; 356 357 offset = gpa - VIOAPIC_BASE; 358 359 /* 360 * The IOAPIC specification allows 32-bit wide accesses to the 361 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 362 */ 363 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 364 if (doread) 365 *data = 0; 366 return (0); 367 } 368 369 VIOAPIC_LOCK(vioapic); 370 if (offset == IOREGSEL) { 371 if (doread) 372 *data = vioapic->ioregsel; 373 else 374 vioapic->ioregsel = *data; 375 } else { 376 if (doread) { 377 *data = vioapic_read(vioapic, vcpuid, 378 vioapic->ioregsel); 379 } else { 380 vioapic_write(vioapic, vcpuid, vioapic->ioregsel, 381 *data); 382 } 383 } 384 VIOAPIC_UNLOCK(vioapic); 385 386 return (0); 387 } 388 389 int 390 vioapic_mmio_read(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t *rval, 391 int size) 392 { 393 int error; 394 struct vioapic *vioapic; 395 396 vioapic = vm_ioapic(vm); 397 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true); 398 return (error); 399 } 400 401 int 402 vioapic_mmio_write(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t wval, 403 int size) 404 { 405 int error; 406 struct vioapic *vioapic; 407 408 vioapic = vm_ioapic(vm); 409 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false); 410 return (error); 411 } 412 413 void 414 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector) 415 { 416 struct vioapic *vioapic; 417 int pin; 418 419 KASSERT(vector >= 0 && vector < 256, 420 ("vioapic_process_eoi: invalid vector %d", vector)); 421 422 vioapic = vm_ioapic(vm); 423 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 424 425 /* 426 * XXX keep track of the pins associated with this vector instead 427 * of iterating on every single pin each time. 428 */ 429 VIOAPIC_LOCK(vioapic); 430 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 431 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 432 continue; 433 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 434 continue; 435 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 436 if (vioapic->rtbl[pin].acnt > 0) { 437 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 438 "acnt %d", pin, vioapic->rtbl[pin].acnt); 439 vioapic_send_intr(vioapic, pin); 440 } 441 } 442 VIOAPIC_UNLOCK(vioapic); 443 } 444 445 struct vioapic * 446 vioapic_init(struct vm *vm) 447 { 448 int i; 449 struct vioapic *vioapic; 450 451 vioapic = malloc(sizeof (struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 452 453 vioapic->vm = vm; 454 mutex_init(&vioapic->lock, NULL, MUTEX_ADAPTIVE, NULL); 455 456 /* Initialize all redirection entries to mask all interrupts */ 457 for (i = 0; i < REDIR_ENTRIES; i++) 458 vioapic->rtbl[i].reg = 0x0001000000010000UL; 459 460 return (vioapic); 461 } 462 463 void 464 vioapic_cleanup(struct vioapic *vioapic) 465 { 466 mutex_destroy(&vioapic->lock); 467 free(vioapic, M_VIOAPIC); 468 } 469 470 int 471 vioapic_pincount(struct vm *vm) 472 { 473 474 return (REDIR_ENTRIES); 475 } 476