1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 /* 32 * This file and its contents are supplied under the terms of the 33 * Common Development and Distribution License ("CDDL"), version 1.0. 34 * You may only use this file in accordance with the terms of version 35 * 1.0 of the CDDL. 36 * 37 * A full copy of the text of the CDDL should have accompanied this 38 * source. A copy of the CDDL is also available via the Internet at 39 * http://www.illumos.org/license/CDDL. 40 * 41 * Copyright 2014 Pluribus Networks Inc. 42 * Copyright 2017 Joyent, Inc. 43 * Copyright 2021 Oxide Computer Company 44 */ 45 46 #include <sys/cdefs.h> 47 __FBSDID("$FreeBSD$"); 48 49 #include <sys/param.h> 50 #include <sys/queue.h> 51 #include <sys/mutex.h> 52 #include <sys/systm.h> 53 #include <sys/kernel.h> 54 #include <sys/kmem.h> 55 #include <sys/cpuset.h> 56 57 #include <x86/apicreg.h> 58 #include <machine/vmm.h> 59 60 #include "vmm_lapic.h" 61 #include "vlapic.h" 62 #include "vioapic.h" 63 64 #define IOREGSEL 0x00 65 #define IOWIN 0x10 66 67 #define REDIR_ENTRIES 32 68 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 69 70 struct ioapic_stats { 71 uint64_t is_interrupts; 72 uint64_t is_saturate_low; 73 uint64_t is_saturate_high; 74 }; 75 76 struct vioapic { 77 struct vm *vm; 78 kmutex_t lock; 79 uint32_t id; 80 uint32_t ioregsel; 81 struct { 82 uint64_t reg; 83 /* 84 * The sum of pin asserts (+1) and deasserts (-1) are tracked in 85 * 'acnt'. It is clamped to prevent overflow or underflow 86 * should emulation consumers feed it an invalid set of 87 * transitions. 88 */ 89 uint_t acnt; 90 } rtbl[REDIR_ENTRIES]; 91 struct ioapic_stats stats; 92 }; 93 94 #define VIOAPIC_LOCK(vioapic) mutex_enter(&((vioapic)->lock)) 95 #define VIOAPIC_UNLOCK(vioapic) mutex_exit(&((vioapic)->lock)) 96 #define VIOAPIC_LOCKED(vioapic) MUTEX_HELD(&((vioapic)->lock)) 97 98 99 static void 100 vioapic_send_intr(struct vioapic *vioapic, int pin) 101 { 102 int vector, delmode; 103 uint32_t low, high, dest; 104 bool level, phys; 105 106 VERIFY(pin >= 0 && pin < REDIR_ENTRIES); 107 ASSERT(VIOAPIC_LOCKED(vioapic)); 108 109 low = vioapic->rtbl[pin].reg; 110 high = vioapic->rtbl[pin].reg >> 32; 111 112 if ((low & IOART_INTMASK) == IOART_INTMSET) { 113 /* Pin is masked */ 114 return; 115 } 116 117 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 118 delmode = low & IOART_DELMOD; 119 level = low & IOART_TRGRLVL ? true : false; 120 if (level) { 121 if ((low & IOART_REM_IRR) != 0) { 122 /* IRR already pending */ 123 return; 124 } 125 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 126 } 127 128 vector = low & IOART_INTVEC; 129 dest = high >> APIC_ID_SHIFT; 130 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 131 vioapic->stats.is_interrupts++; 132 } 133 134 static int 135 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 136 { 137 uint_t oldcnt, newcnt; 138 bool needintr = false; 139 int err = 0; 140 141 VERIFY(pin >= 0 && pin < REDIR_ENTRIES); 142 ASSERT(VIOAPIC_LOCKED(vioapic)); 143 144 oldcnt = newcnt = vioapic->rtbl[pin].acnt; 145 if (newstate) { 146 if (newcnt != UINT_MAX) { 147 newcnt++; 148 } else { 149 err = E2BIG; 150 DTRACE_PROBE2(vioapic__sat_high, 151 struct vioapic *, vioapic, int, pin); 152 vioapic->stats.is_saturate_high++; 153 } 154 } else { 155 if (newcnt != 0) { 156 newcnt--; 157 } else { 158 err = ERANGE; 159 DTRACE_PROBE2(vioapic__sat_low, 160 struct vioapic *, vioapic, int, pin); 161 vioapic->stats.is_saturate_low++; 162 } 163 } 164 vioapic->rtbl[pin].acnt = newcnt; 165 166 if (oldcnt == 0 && newcnt == 1) { 167 needintr = true; 168 DTRACE_PROBE2(vioapic__assert, struct vioapic *, vioapic, 169 int, pin); 170 } else if (oldcnt == 1 && newcnt == 0) { 171 DTRACE_PROBE2(vioapic__deassert, struct vioapic *, vioapic, 172 int, pin); 173 } 174 175 if (needintr) { 176 vioapic_send_intr(vioapic, pin); 177 } 178 return (err); 179 } 180 181 enum irqstate { 182 IRQSTATE_ASSERT, 183 IRQSTATE_DEASSERT, 184 IRQSTATE_PULSE 185 }; 186 187 static int 188 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 189 { 190 struct vioapic *vioapic; 191 int err = 0; 192 193 if (irq < 0 || irq >= REDIR_ENTRIES) 194 return (EINVAL); 195 196 vioapic = vm_ioapic(vm); 197 198 VIOAPIC_LOCK(vioapic); 199 switch (irqstate) { 200 case IRQSTATE_ASSERT: 201 err = vioapic_set_pinstate(vioapic, irq, true); 202 break; 203 case IRQSTATE_DEASSERT: 204 err = vioapic_set_pinstate(vioapic, irq, false); 205 break; 206 case IRQSTATE_PULSE: 207 err = vioapic_set_pinstate(vioapic, irq, true); 208 if (err == 0) { 209 err = vioapic_set_pinstate(vioapic, irq, false); 210 } 211 break; 212 default: 213 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 214 } 215 VIOAPIC_UNLOCK(vioapic); 216 217 return (err); 218 } 219 220 int 221 vioapic_assert_irq(struct vm *vm, int irq) 222 { 223 224 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 225 } 226 227 int 228 vioapic_deassert_irq(struct vm *vm, int irq) 229 { 230 231 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 232 } 233 234 int 235 vioapic_pulse_irq(struct vm *vm, int irq) 236 { 237 238 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 239 } 240 241 static uint32_t 242 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) 243 { 244 int regnum, pin, rshift; 245 246 regnum = addr & 0xff; 247 switch (regnum) { 248 case IOAPIC_ID: 249 return (vioapic->id); 250 break; 251 case IOAPIC_VER: 252 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 253 break; 254 case IOAPIC_ARB: 255 return (vioapic->id); 256 break; 257 default: 258 break; 259 } 260 261 /* redirection table entries */ 262 if (regnum >= IOAPIC_REDTBL && 263 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 264 pin = (regnum - IOAPIC_REDTBL) / 2; 265 if ((regnum - IOAPIC_REDTBL) % 2) 266 rshift = 32; 267 else 268 rshift = 0; 269 270 return (vioapic->rtbl[pin].reg >> rshift); 271 } 272 273 return (0); 274 } 275 276 static void 277 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) 278 { 279 uint64_t data64, mask64; 280 int regnum, pin, lshift; 281 282 regnum = addr & 0xff; 283 switch (regnum) { 284 case IOAPIC_ID: 285 vioapic->id = data & APIC_ID_MASK; 286 break; 287 case IOAPIC_VER: 288 case IOAPIC_ARB: 289 /* readonly */ 290 break; 291 default: 292 break; 293 } 294 295 /* redirection table entries */ 296 if (regnum >= IOAPIC_REDTBL && 297 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 298 pin = (regnum - IOAPIC_REDTBL) / 2; 299 if ((regnum - IOAPIC_REDTBL) % 2) 300 lshift = 32; 301 else 302 lshift = 0; 303 304 data64 = (uint64_t)data << lshift; 305 mask64 = (uint64_t)0xffffffff << lshift; 306 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 307 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 308 309 /* 310 * Switching from level to edge triggering will clear the IRR 311 * bit. This is what FreeBSD will do in order to EOI an 312 * interrupt when the IO-APIC doesn't support targeted EOI (see 313 * _ioapic_eoi_source). 314 */ 315 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 316 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 317 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 318 319 /* 320 * Generate an interrupt if the following conditions are met: 321 * - pin trigger mode is level 322 * - pin level is asserted 323 */ 324 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 325 (vioapic->rtbl[pin].acnt > 0)) { 326 vioapic_send_intr(vioapic, pin); 327 } 328 } 329 } 330 331 static int 332 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa, 333 uint64_t *data, int size, bool doread) 334 { 335 uint64_t offset; 336 337 offset = gpa - VIOAPIC_BASE; 338 339 /* 340 * The IOAPIC specification allows 32-bit wide accesses to the 341 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 342 */ 343 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 344 if (doread) 345 *data = 0; 346 return (0); 347 } 348 349 VIOAPIC_LOCK(vioapic); 350 if (offset == IOREGSEL) { 351 if (doread) 352 *data = vioapic->ioregsel; 353 else 354 vioapic->ioregsel = *data; 355 } else { 356 if (doread) { 357 *data = vioapic_read(vioapic, vcpuid, 358 vioapic->ioregsel); 359 } else { 360 vioapic_write(vioapic, vcpuid, vioapic->ioregsel, 361 *data); 362 } 363 } 364 VIOAPIC_UNLOCK(vioapic); 365 366 return (0); 367 } 368 369 int 370 vioapic_mmio_read(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t *rval, 371 int size) 372 { 373 int error; 374 struct vioapic *vioapic; 375 376 vioapic = vm_ioapic(vm); 377 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true); 378 return (error); 379 } 380 381 int 382 vioapic_mmio_write(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t wval, 383 int size) 384 { 385 int error; 386 struct vioapic *vioapic; 387 388 vioapic = vm_ioapic(vm); 389 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false); 390 return (error); 391 } 392 393 void 394 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector) 395 { 396 struct vioapic *vioapic; 397 int pin; 398 399 KASSERT(vector >= 0 && vector < 256, 400 ("vioapic_process_eoi: invalid vector %d", vector)); 401 402 vioapic = vm_ioapic(vm); 403 404 /* 405 * XXX keep track of the pins associated with this vector instead 406 * of iterating on every single pin each time. 407 */ 408 VIOAPIC_LOCK(vioapic); 409 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 410 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 411 continue; 412 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 413 continue; 414 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 415 if (vioapic->rtbl[pin].acnt > 0) { 416 /* Pin asserted at EOI */ 417 vioapic_send_intr(vioapic, pin); 418 } 419 } 420 VIOAPIC_UNLOCK(vioapic); 421 } 422 423 struct vioapic * 424 vioapic_init(struct vm *vm) 425 { 426 int i; 427 struct vioapic *vioapic; 428 429 vioapic = kmem_zalloc(sizeof (struct vioapic), KM_SLEEP); 430 431 vioapic->vm = vm; 432 mutex_init(&vioapic->lock, NULL, MUTEX_ADAPTIVE, NULL); 433 434 /* Initialize all redirection entries to mask all interrupts */ 435 for (i = 0; i < REDIR_ENTRIES; i++) 436 vioapic->rtbl[i].reg = 0x0001000000010000UL; 437 438 return (vioapic); 439 } 440 441 void 442 vioapic_cleanup(struct vioapic *vioapic) 443 { 444 mutex_destroy(&vioapic->lock); 445 kmem_free(vioapic, sizeof (*vioapic)); 446 } 447 448 int 449 vioapic_pincount(struct vm *vm) 450 { 451 452 return (REDIR_ENTRIES); 453 } 454