1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * Copyright (c) 2019 Joyent, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_bhyve_snapshot.h" 36 37 #include <sys/param.h> 38 #include <sys/lock.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/mutex.h> 42 #include <sys/systm.h> 43 #include <sys/smp.h> 44 45 #include <x86/specialreg.h> 46 #include <x86/apicreg.h> 47 48 #include <machine/clock.h> 49 #include <machine/smp.h> 50 51 #include <machine/vmm.h> 52 #include <machine/vmm_snapshot.h> 53 54 #include "vmm_lapic.h" 55 #include "vmm_ktr.h" 56 #include "vmm_stat.h" 57 58 #include "vlapic.h" 59 #include "vlapic_priv.h" 60 #include "vioapic.h" 61 62 #define PRIO(x) ((x) >> 4) 63 64 #define VLAPIC_VERSION (16) 65 66 #define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0) 67 68 /* 69 * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the 70 * vlapic_callout_handler() and vcpu accesses to: 71 * - timer_freq_bt, timer_period_bt, timer_fire_bt 72 * - timer LVT register 73 */ 74 #define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx)) 75 #define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx)) 76 #define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx)) 77 78 /* 79 * APIC timer frequency: 80 * - arbitrary but chosen to be in the ballpark of contemporary hardware. 81 * - power-of-two to avoid loss of precision when converted to a bintime. 82 */ 83 #define VLAPIC_BUS_FREQ (128 * 1024 * 1024) 84 85 static void vlapic_set_error(struct vlapic *, uint32_t, bool); 86 static void vlapic_callout_handler(void *arg); 87 88 static __inline uint32_t 89 vlapic_get_id(struct vlapic *vlapic) 90 { 91 92 if (x2apic(vlapic)) 93 return (vlapic->vcpuid); 94 else 95 return (vlapic->vcpuid << 24); 96 } 97 98 static uint32_t 99 x2apic_ldr(struct vlapic *vlapic) 100 { 101 int apicid; 102 uint32_t ldr; 103 104 apicid = vlapic_get_id(vlapic); 105 ldr = 1 << (apicid & 0xf); 106 ldr |= (apicid & 0xffff0) << 12; 107 return (ldr); 108 } 109 110 void 111 vlapic_dfr_write_handler(struct vlapic *vlapic) 112 { 113 struct LAPIC *lapic; 114 115 lapic = vlapic->apic_page; 116 if (x2apic(vlapic)) { 117 VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x", 118 lapic->dfr); 119 lapic->dfr = 0; 120 return; 121 } 122 123 lapic->dfr &= APIC_DFR_MODEL_MASK; 124 lapic->dfr |= APIC_DFR_RESERVED; 125 126 if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT) 127 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model"); 128 else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER) 129 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model"); 130 else 131 VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr); 132 } 133 134 void 135 vlapic_ldr_write_handler(struct vlapic *vlapic) 136 { 137 struct LAPIC *lapic; 138 139 lapic = vlapic->apic_page; 140 141 /* LDR is read-only in x2apic mode */ 142 if (x2apic(vlapic)) { 143 VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x", 144 lapic->ldr); 145 lapic->ldr = x2apic_ldr(vlapic); 146 } else { 147 lapic->ldr &= ~APIC_LDR_RESERVED; 148 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr); 149 } 150 } 151 152 void 153 vlapic_id_write_handler(struct vlapic *vlapic) 154 { 155 struct LAPIC *lapic; 156 157 /* 158 * We don't allow the ID register to be modified so reset it back to 159 * its default value. 160 */ 161 lapic = vlapic->apic_page; 162 lapic->id = vlapic_get_id(vlapic); 163 } 164 165 static int 166 vlapic_timer_divisor(uint32_t dcr) 167 { 168 switch (dcr & 0xB) { 169 case APIC_TDCR_1: 170 return (1); 171 case APIC_TDCR_2: 172 return (2); 173 case APIC_TDCR_4: 174 return (4); 175 case APIC_TDCR_8: 176 return (8); 177 case APIC_TDCR_16: 178 return (16); 179 case APIC_TDCR_32: 180 return (32); 181 case APIC_TDCR_64: 182 return (64); 183 case APIC_TDCR_128: 184 return (128); 185 default: 186 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr); 187 } 188 } 189 190 #if 0 191 static inline void 192 vlapic_dump_lvt(uint32_t offset, uint32_t *lvt) 193 { 194 printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset, 195 *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS, 196 *lvt & APIC_LVTT_M); 197 } 198 #endif 199 200 static uint32_t 201 vlapic_get_ccr(struct vlapic *vlapic) 202 { 203 struct bintime bt_now, bt_rem; 204 struct LAPIC *lapic; 205 uint32_t ccr; 206 207 ccr = 0; 208 lapic = vlapic->apic_page; 209 210 VLAPIC_TIMER_LOCK(vlapic); 211 if (callout_active(&vlapic->callout)) { 212 /* 213 * If the timer is scheduled to expire in the future then 214 * compute the value of 'ccr' based on the remaining time. 215 */ 216 binuptime(&bt_now); 217 if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) { 218 bt_rem = vlapic->timer_fire_bt; 219 bintime_sub(&bt_rem, &bt_now); 220 ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt); 221 ccr += bt_rem.frac / vlapic->timer_freq_bt.frac; 222 } 223 } 224 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, " 225 "icr_timer is %#x", ccr, lapic->icr_timer)); 226 VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x", 227 ccr, lapic->icr_timer); 228 VLAPIC_TIMER_UNLOCK(vlapic); 229 return (ccr); 230 } 231 232 void 233 vlapic_dcr_write_handler(struct vlapic *vlapic) 234 { 235 struct LAPIC *lapic; 236 int divisor; 237 238 lapic = vlapic->apic_page; 239 VLAPIC_TIMER_LOCK(vlapic); 240 241 divisor = vlapic_timer_divisor(lapic->dcr_timer); 242 VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d", 243 lapic->dcr_timer, divisor); 244 245 /* 246 * Update the timer frequency and the timer period. 247 * 248 * XXX changes to the frequency divider will not take effect until 249 * the timer is reloaded. 250 */ 251 FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt); 252 vlapic->timer_period_bt = vlapic->timer_freq_bt; 253 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer); 254 255 VLAPIC_TIMER_UNLOCK(vlapic); 256 } 257 258 void 259 vlapic_esr_write_handler(struct vlapic *vlapic) 260 { 261 struct LAPIC *lapic; 262 263 lapic = vlapic->apic_page; 264 lapic->esr = vlapic->esr_pending; 265 vlapic->esr_pending = 0; 266 } 267 268 int 269 vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level) 270 { 271 struct LAPIC *lapic; 272 uint32_t *irrptr, *tmrptr, mask; 273 int idx; 274 275 KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector)); 276 277 lapic = vlapic->apic_page; 278 if (!(lapic->svr & APIC_SVR_ENABLE)) { 279 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring " 280 "interrupt %d", vector); 281 return (0); 282 } 283 284 if (vector < 16) { 285 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR, 286 false); 287 VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d", 288 vector); 289 return (1); 290 } 291 292 if (vlapic->ops.set_intr_ready) 293 return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level)); 294 295 idx = (vector / 32) * 4; 296 mask = 1 << (vector % 32); 297 298 irrptr = &lapic->irr0; 299 atomic_set_int(&irrptr[idx], mask); 300 301 /* 302 * Verify that the trigger-mode of the interrupt matches with 303 * the vlapic TMR registers. 304 */ 305 tmrptr = &lapic->tmr0; 306 if ((tmrptr[idx] & mask) != (level ? mask : 0)) { 307 VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but " 308 "interrupt is %s-triggered", idx / 4, tmrptr[idx], 309 level ? "level" : "edge"); 310 } 311 312 VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready"); 313 return (1); 314 } 315 316 static __inline uint32_t * 317 vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset) 318 { 319 struct LAPIC *lapic = vlapic->apic_page; 320 int i; 321 322 switch (offset) { 323 case APIC_OFFSET_CMCI_LVT: 324 return (&lapic->lvt_cmci); 325 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 326 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2; 327 return ((&lapic->lvt_timer) + i); 328 default: 329 panic("vlapic_get_lvt: invalid LVT\n"); 330 } 331 } 332 333 static __inline int 334 lvt_off_to_idx(uint32_t offset) 335 { 336 int index; 337 338 switch (offset) { 339 case APIC_OFFSET_CMCI_LVT: 340 index = APIC_LVT_CMCI; 341 break; 342 case APIC_OFFSET_TIMER_LVT: 343 index = APIC_LVT_TIMER; 344 break; 345 case APIC_OFFSET_THERM_LVT: 346 index = APIC_LVT_THERMAL; 347 break; 348 case APIC_OFFSET_PERF_LVT: 349 index = APIC_LVT_PMC; 350 break; 351 case APIC_OFFSET_LINT0_LVT: 352 index = APIC_LVT_LINT0; 353 break; 354 case APIC_OFFSET_LINT1_LVT: 355 index = APIC_LVT_LINT1; 356 break; 357 case APIC_OFFSET_ERROR_LVT: 358 index = APIC_LVT_ERROR; 359 break; 360 default: 361 index = -1; 362 break; 363 } 364 KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: " 365 "invalid lvt index %d for offset %#x", index, offset)); 366 367 return (index); 368 } 369 370 static __inline uint32_t 371 vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset) 372 { 373 int idx; 374 uint32_t val; 375 376 idx = lvt_off_to_idx(offset); 377 val = atomic_load_acq_32(&vlapic->lvt_last[idx]); 378 return (val); 379 } 380 381 void 382 vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset) 383 { 384 uint32_t *lvtptr, mask, val; 385 struct LAPIC *lapic; 386 int idx; 387 388 lapic = vlapic->apic_page; 389 lvtptr = vlapic_get_lvtptr(vlapic, offset); 390 val = *lvtptr; 391 idx = lvt_off_to_idx(offset); 392 393 if (!(lapic->svr & APIC_SVR_ENABLE)) 394 val |= APIC_LVT_M; 395 mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR; 396 switch (offset) { 397 case APIC_OFFSET_TIMER_LVT: 398 mask |= APIC_LVTT_TM; 399 break; 400 case APIC_OFFSET_ERROR_LVT: 401 break; 402 case APIC_OFFSET_LINT0_LVT: 403 case APIC_OFFSET_LINT1_LVT: 404 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP; 405 /* FALLTHROUGH */ 406 default: 407 mask |= APIC_LVT_DM; 408 break; 409 } 410 val &= mask; 411 *lvtptr = val; 412 atomic_store_rel_32(&vlapic->lvt_last[idx], val); 413 } 414 415 static void 416 vlapic_mask_lvts(struct vlapic *vlapic) 417 { 418 struct LAPIC *lapic = vlapic->apic_page; 419 420 lapic->lvt_cmci |= APIC_LVT_M; 421 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT); 422 423 lapic->lvt_timer |= APIC_LVT_M; 424 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT); 425 426 lapic->lvt_thermal |= APIC_LVT_M; 427 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT); 428 429 lapic->lvt_pcint |= APIC_LVT_M; 430 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT); 431 432 lapic->lvt_lint0 |= APIC_LVT_M; 433 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT); 434 435 lapic->lvt_lint1 |= APIC_LVT_M; 436 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT); 437 438 lapic->lvt_error |= APIC_LVT_M; 439 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT); 440 } 441 442 static int 443 vlapic_fire_lvt(struct vlapic *vlapic, u_int lvt) 444 { 445 uint32_t mode, reg, vec; 446 447 reg = atomic_load_acq_32(&vlapic->lvt_last[lvt]); 448 449 if (reg & APIC_LVT_M) 450 return (0); 451 vec = reg & APIC_LVT_VECTOR; 452 mode = reg & APIC_LVT_DM; 453 454 switch (mode) { 455 case APIC_LVT_DM_FIXED: 456 if (vec < 16) { 457 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR, 458 lvt == APIC_LVT_ERROR); 459 return (0); 460 } 461 if (vlapic_set_intr_ready(vlapic, vec, false)) 462 vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true); 463 break; 464 case APIC_LVT_DM_NMI: 465 vm_inject_nmi(vlapic->vm, vlapic->vcpuid); 466 break; 467 case APIC_LVT_DM_EXTINT: 468 vm_inject_extint(vlapic->vm, vlapic->vcpuid); 469 break; 470 default: 471 // Other modes ignored 472 return (0); 473 } 474 return (1); 475 } 476 477 #if 1 478 static void 479 dump_isrvec_stk(struct vlapic *vlapic) 480 { 481 int i; 482 uint32_t *isrptr; 483 484 isrptr = &vlapic->apic_page->isr0; 485 for (i = 0; i < 8; i++) 486 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]); 487 488 for (i = 0; i <= vlapic->isrvec_stk_top; i++) 489 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]); 490 } 491 #endif 492 493 /* 494 * Algorithm adopted from section "Interrupt, Task and Processor Priority" 495 * in Intel Architecture Manual Vol 3a. 496 */ 497 static void 498 vlapic_update_ppr(struct vlapic *vlapic) 499 { 500 int isrvec, tpr, ppr; 501 502 /* 503 * Note that the value on the stack at index 0 is always 0. 504 * 505 * This is a placeholder for the value of ISRV when none of the 506 * bits is set in the ISRx registers. 507 */ 508 isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top]; 509 tpr = vlapic->apic_page->tpr; 510 511 #if 1 512 { 513 int i, lastprio, curprio, vector, idx; 514 uint32_t *isrptr; 515 516 if (vlapic->isrvec_stk_top == 0 && isrvec != 0) 517 panic("isrvec_stk is corrupted: %d", isrvec); 518 519 /* 520 * Make sure that the priority of the nested interrupts is 521 * always increasing. 522 */ 523 lastprio = -1; 524 for (i = 1; i <= vlapic->isrvec_stk_top; i++) { 525 curprio = PRIO(vlapic->isrvec_stk[i]); 526 if (curprio <= lastprio) { 527 dump_isrvec_stk(vlapic); 528 panic("isrvec_stk does not satisfy invariant"); 529 } 530 lastprio = curprio; 531 } 532 533 /* 534 * Make sure that each bit set in the ISRx registers has a 535 * corresponding entry on the isrvec stack. 536 */ 537 i = 1; 538 isrptr = &vlapic->apic_page->isr0; 539 for (vector = 0; vector < 256; vector++) { 540 idx = (vector / 32) * 4; 541 if (isrptr[idx] & (1 << (vector % 32))) { 542 if (i > vlapic->isrvec_stk_top || 543 vlapic->isrvec_stk[i] != vector) { 544 dump_isrvec_stk(vlapic); 545 panic("ISR and isrvec_stk out of sync"); 546 } 547 i++; 548 } 549 } 550 } 551 #endif 552 553 if (PRIO(tpr) >= PRIO(isrvec)) 554 ppr = tpr; 555 else 556 ppr = isrvec & 0xf0; 557 558 vlapic->apic_page->ppr = ppr; 559 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr); 560 } 561 562 void 563 vlapic_sync_tpr(struct vlapic *vlapic) 564 { 565 vlapic_update_ppr(vlapic); 566 } 567 568 static VMM_STAT(VLAPIC_GRATUITOUS_EOI, "EOI without any in-service interrupt"); 569 570 static void 571 vlapic_process_eoi(struct vlapic *vlapic) 572 { 573 struct LAPIC *lapic = vlapic->apic_page; 574 uint32_t *isrptr, *tmrptr; 575 int i, idx, bitpos, vector; 576 577 isrptr = &lapic->isr0; 578 tmrptr = &lapic->tmr0; 579 580 for (i = 7; i >= 0; i--) { 581 idx = i * 4; 582 bitpos = fls(isrptr[idx]); 583 if (bitpos-- != 0) { 584 if (vlapic->isrvec_stk_top <= 0) { 585 panic("invalid vlapic isrvec_stk_top %d", 586 vlapic->isrvec_stk_top); 587 } 588 isrptr[idx] &= ~(1 << bitpos); 589 vector = i * 32 + bitpos; 590 VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "EOI vector %d", 591 vector); 592 VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi"); 593 vlapic->isrvec_stk_top--; 594 vlapic_update_ppr(vlapic); 595 if ((tmrptr[idx] & (1 << bitpos)) != 0) { 596 vioapic_process_eoi(vlapic->vm, vlapic->vcpuid, 597 vector); 598 } 599 return; 600 } 601 } 602 VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "Gratuitous EOI"); 603 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_GRATUITOUS_EOI, 1); 604 } 605 606 static __inline int 607 vlapic_get_lvt_field(uint32_t lvt, uint32_t mask) 608 { 609 610 return (lvt & mask); 611 } 612 613 static __inline int 614 vlapic_periodic_timer(struct vlapic *vlapic) 615 { 616 uint32_t lvt; 617 618 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT); 619 620 return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC)); 621 } 622 623 static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic"); 624 625 static void 626 vlapic_set_error(struct vlapic *vlapic, uint32_t mask, bool lvt_error) 627 { 628 629 vlapic->esr_pending |= mask; 630 631 /* 632 * Avoid infinite recursion if the error LVT itself is configured with 633 * an illegal vector. 634 */ 635 if (lvt_error) 636 return; 637 638 if (vlapic_fire_lvt(vlapic, APIC_LVT_ERROR)) { 639 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1); 640 } 641 } 642 643 static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic"); 644 645 static void 646 vlapic_fire_timer(struct vlapic *vlapic) 647 { 648 649 KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked")); 650 651 if (vlapic_fire_lvt(vlapic, APIC_LVT_TIMER)) { 652 VLAPIC_CTR0(vlapic, "vlapic timer fired"); 653 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1); 654 } 655 } 656 657 static VMM_STAT(VLAPIC_INTR_CMC, 658 "corrected machine check interrupts generated by vlapic"); 659 660 void 661 vlapic_fire_cmci(struct vlapic *vlapic) 662 { 663 664 if (vlapic_fire_lvt(vlapic, APIC_LVT_CMCI)) { 665 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1); 666 } 667 } 668 669 static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1, 670 "lvts triggered"); 671 672 int 673 vlapic_trigger_lvt(struct vlapic *vlapic, int vector) 674 { 675 676 if (vlapic_enabled(vlapic) == false) { 677 /* 678 * When the local APIC is global/hardware disabled, 679 * LINT[1:0] pins are configured as INTR and NMI pins, 680 * respectively. 681 */ 682 switch (vector) { 683 case APIC_LVT_LINT0: 684 vm_inject_extint(vlapic->vm, vlapic->vcpuid); 685 break; 686 case APIC_LVT_LINT1: 687 vm_inject_nmi(vlapic->vm, vlapic->vcpuid); 688 break; 689 default: 690 break; 691 } 692 return (0); 693 } 694 695 switch (vector) { 696 case APIC_LVT_LINT0: 697 case APIC_LVT_LINT1: 698 case APIC_LVT_TIMER: 699 case APIC_LVT_ERROR: 700 case APIC_LVT_PMC: 701 case APIC_LVT_THERMAL: 702 case APIC_LVT_CMCI: 703 if (vlapic_fire_lvt(vlapic, vector)) { 704 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, 705 LVTS_TRIGGERRED, vector, 1); 706 } 707 break; 708 default: 709 return (EINVAL); 710 } 711 return (0); 712 } 713 714 static void 715 vlapic_callout_reset(struct vlapic *vlapic, sbintime_t t) 716 { 717 callout_reset_sbt_curcpu(&vlapic->callout, t, 0, 718 vlapic_callout_handler, vlapic, 0); 719 } 720 721 static void 722 vlapic_callout_handler(void *arg) 723 { 724 struct vlapic *vlapic; 725 struct bintime bt, btnow; 726 sbintime_t rem_sbt; 727 728 vlapic = arg; 729 730 VLAPIC_TIMER_LOCK(vlapic); 731 if (callout_pending(&vlapic->callout)) /* callout was reset */ 732 goto done; 733 734 if (!callout_active(&vlapic->callout)) /* callout was stopped */ 735 goto done; 736 737 callout_deactivate(&vlapic->callout); 738 739 vlapic_fire_timer(vlapic); 740 741 if (vlapic_periodic_timer(vlapic)) { 742 binuptime(&btnow); 743 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=), 744 ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx", 745 btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec, 746 vlapic->timer_fire_bt.frac)); 747 748 /* 749 * Compute the delta between when the timer was supposed to 750 * fire and the present time. 751 */ 752 bt = btnow; 753 bintime_sub(&bt, &vlapic->timer_fire_bt); 754 755 rem_sbt = bttosbt(vlapic->timer_period_bt); 756 if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) { 757 /* 758 * Adjust the time until the next countdown downward 759 * to account for the lost time. 760 */ 761 rem_sbt -= bttosbt(bt); 762 } else { 763 /* 764 * If the delta is greater than the timer period then 765 * just reset our time base instead of trying to catch 766 * up. 767 */ 768 vlapic->timer_fire_bt = btnow; 769 VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu " 770 "usecs, period is %lu usecs - resetting time base", 771 bttosbt(bt) / SBT_1US, 772 bttosbt(vlapic->timer_period_bt) / SBT_1US); 773 } 774 775 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt); 776 vlapic_callout_reset(vlapic, rem_sbt); 777 } 778 done: 779 VLAPIC_TIMER_UNLOCK(vlapic); 780 } 781 782 void 783 vlapic_icrtmr_write_handler(struct vlapic *vlapic) 784 { 785 struct LAPIC *lapic; 786 sbintime_t sbt; 787 uint32_t icr_timer; 788 789 VLAPIC_TIMER_LOCK(vlapic); 790 791 lapic = vlapic->apic_page; 792 icr_timer = lapic->icr_timer; 793 794 vlapic->timer_period_bt = vlapic->timer_freq_bt; 795 bintime_mul(&vlapic->timer_period_bt, icr_timer); 796 797 if (icr_timer != 0) { 798 binuptime(&vlapic->timer_fire_bt); 799 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt); 800 801 sbt = bttosbt(vlapic->timer_period_bt); 802 vlapic_callout_reset(vlapic, sbt); 803 } else 804 callout_stop(&vlapic->callout); 805 806 VLAPIC_TIMER_UNLOCK(vlapic); 807 } 808 809 /* 810 * This function populates 'dmask' with the set of vcpus that match the 811 * addressing specified by the (dest, phys, lowprio) tuple. 812 * 813 * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit) 814 * or xAPIC (8-bit) destination field. 815 */ 816 static void 817 vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys, 818 bool lowprio, bool x2apic_dest) 819 { 820 struct vlapic *vlapic; 821 uint32_t dfr, ldr, ldest, cluster; 822 uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id; 823 cpuset_t amask; 824 int vcpuid; 825 826 if ((x2apic_dest && dest == 0xffffffff) || 827 (!x2apic_dest && dest == 0xff)) { 828 /* 829 * Broadcast in both logical and physical modes. 830 */ 831 *dmask = vm_active_cpus(vm); 832 return; 833 } 834 835 if (phys) { 836 /* 837 * Physical mode: destination is APIC ID. 838 */ 839 CPU_ZERO(dmask); 840 vcpuid = vm_apicid2vcpuid(vm, dest); 841 amask = vm_active_cpus(vm); 842 if (vcpuid < vm_get_maxcpus(vm) && CPU_ISSET(vcpuid, &amask)) 843 CPU_SET(vcpuid, dmask); 844 } else { 845 /* 846 * In the "Flat Model" the MDA is interpreted as an 8-bit wide 847 * bitmask. This model is only available in the xAPIC mode. 848 */ 849 mda_flat_ldest = dest & 0xff; 850 851 /* 852 * In the "Cluster Model" the MDA is used to identify a 853 * specific cluster and a set of APICs in that cluster. 854 */ 855 if (x2apic_dest) { 856 mda_cluster_id = dest >> 16; 857 mda_cluster_ldest = dest & 0xffff; 858 } else { 859 mda_cluster_id = (dest >> 4) & 0xf; 860 mda_cluster_ldest = dest & 0xf; 861 } 862 863 /* 864 * Logical mode: match each APIC that has a bit set 865 * in its LDR that matches a bit in the ldest. 866 */ 867 CPU_ZERO(dmask); 868 amask = vm_active_cpus(vm); 869 CPU_FOREACH_ISSET(vcpuid, &amask) { 870 vlapic = vm_lapic(vm, vcpuid); 871 dfr = vlapic->apic_page->dfr; 872 ldr = vlapic->apic_page->ldr; 873 874 if ((dfr & APIC_DFR_MODEL_MASK) == 875 APIC_DFR_MODEL_FLAT) { 876 ldest = ldr >> 24; 877 mda_ldest = mda_flat_ldest; 878 } else if ((dfr & APIC_DFR_MODEL_MASK) == 879 APIC_DFR_MODEL_CLUSTER) { 880 if (x2apic(vlapic)) { 881 cluster = ldr >> 16; 882 ldest = ldr & 0xffff; 883 } else { 884 cluster = ldr >> 28; 885 ldest = (ldr >> 24) & 0xf; 886 } 887 if (cluster != mda_cluster_id) 888 continue; 889 mda_ldest = mda_cluster_ldest; 890 } else { 891 /* 892 * Guest has configured a bad logical 893 * model for this vcpu - skip it. 894 */ 895 VLAPIC_CTR1(vlapic, "vlapic has bad logical " 896 "model %x - cannot deliver interrupt", dfr); 897 continue; 898 } 899 900 if ((mda_ldest & ldest) != 0) { 901 CPU_SET(vcpuid, dmask); 902 if (lowprio) 903 break; 904 } 905 } 906 } 907 } 908 909 static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu"); 910 911 static void 912 vlapic_set_tpr(struct vlapic *vlapic, uint8_t val) 913 { 914 struct LAPIC *lapic = vlapic->apic_page; 915 916 if (lapic->tpr != val) { 917 VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vlapic TPR changed " 918 "from %#x to %#x", lapic->tpr, val); 919 lapic->tpr = val; 920 vlapic_update_ppr(vlapic); 921 } 922 } 923 924 static uint8_t 925 vlapic_get_tpr(struct vlapic *vlapic) 926 { 927 struct LAPIC *lapic = vlapic->apic_page; 928 929 return (lapic->tpr); 930 } 931 932 void 933 vlapic_set_cr8(struct vlapic *vlapic, uint64_t val) 934 { 935 uint8_t tpr; 936 937 if (val & ~0xf) { 938 vm_inject_gp(vlapic->vm, vlapic->vcpuid); 939 return; 940 } 941 942 tpr = val << 4; 943 vlapic_set_tpr(vlapic, tpr); 944 } 945 946 uint64_t 947 vlapic_get_cr8(struct vlapic *vlapic) 948 { 949 uint8_t tpr; 950 951 tpr = vlapic_get_tpr(vlapic); 952 return (tpr >> 4); 953 } 954 955 int 956 vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu) 957 { 958 int i; 959 bool phys; 960 cpuset_t dmask; 961 uint64_t icrval; 962 uint32_t dest, vec, mode; 963 struct vlapic *vlapic2; 964 struct vm_exit *vmexit; 965 struct LAPIC *lapic; 966 uint16_t maxcpus; 967 968 lapic = vlapic->apic_page; 969 lapic->icr_lo &= ~APIC_DELSTAT_PEND; 970 icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo; 971 972 if (x2apic(vlapic)) 973 dest = icrval >> 32; 974 else 975 dest = icrval >> (32 + 24); 976 vec = icrval & APIC_VECTOR_MASK; 977 mode = icrval & APIC_DELMODE_MASK; 978 979 if (mode == APIC_DELMODE_FIXED && vec < 16) { 980 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR, false); 981 VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec); 982 return (0); 983 } 984 985 VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec); 986 987 if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) { 988 switch (icrval & APIC_DEST_MASK) { 989 case APIC_DEST_DESTFLD: 990 phys = ((icrval & APIC_DESTMODE_LOG) == 0); 991 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false, 992 x2apic(vlapic)); 993 break; 994 case APIC_DEST_SELF: 995 CPU_SETOF(vlapic->vcpuid, &dmask); 996 break; 997 case APIC_DEST_ALLISELF: 998 dmask = vm_active_cpus(vlapic->vm); 999 break; 1000 case APIC_DEST_ALLESELF: 1001 dmask = vm_active_cpus(vlapic->vm); 1002 CPU_CLR(vlapic->vcpuid, &dmask); 1003 break; 1004 default: 1005 CPU_ZERO(&dmask); /* satisfy gcc */ 1006 break; 1007 } 1008 1009 CPU_FOREACH_ISSET(i, &dmask) { 1010 if (mode == APIC_DELMODE_FIXED) { 1011 lapic_intr_edge(vlapic->vm, i, vec); 1012 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, 1013 IPIS_SENT, i, 1); 1014 VLAPIC_CTR2(vlapic, "vlapic sending ipi %d " 1015 "to vcpuid %d", vec, i); 1016 } else { 1017 vm_inject_nmi(vlapic->vm, i); 1018 VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi " 1019 "to vcpuid %d", i); 1020 } 1021 } 1022 1023 return (0); /* handled completely in the kernel */ 1024 } 1025 1026 maxcpus = vm_get_maxcpus(vlapic->vm); 1027 if (mode == APIC_DELMODE_INIT) { 1028 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT) 1029 return (0); 1030 1031 if (vlapic->vcpuid == 0 && dest != 0 && dest < maxcpus) { 1032 vlapic2 = vm_lapic(vlapic->vm, dest); 1033 1034 /* move from INIT to waiting-for-SIPI state */ 1035 if (vlapic2->boot_state == BS_INIT) { 1036 vlapic2->boot_state = BS_SIPI; 1037 } 1038 1039 return (0); 1040 } 1041 } 1042 1043 if (mode == APIC_DELMODE_STARTUP) { 1044 if (vlapic->vcpuid == 0 && dest != 0 && dest < maxcpus) { 1045 vlapic2 = vm_lapic(vlapic->vm, dest); 1046 1047 /* 1048 * Ignore SIPIs in any state other than wait-for-SIPI 1049 */ 1050 if (vlapic2->boot_state != BS_SIPI) 1051 return (0); 1052 1053 vlapic2->boot_state = BS_RUNNING; 1054 1055 *retu = true; 1056 vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid); 1057 vmexit->exitcode = VM_EXITCODE_SPINUP_AP; 1058 vmexit->u.spinup_ap.vcpu = dest; 1059 vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT; 1060 1061 return (0); 1062 } 1063 } 1064 1065 /* 1066 * This will cause a return to userland. 1067 */ 1068 return (1); 1069 } 1070 1071 void 1072 vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val) 1073 { 1074 int vec; 1075 1076 KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode")); 1077 1078 vec = val & 0xff; 1079 lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec); 1080 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, IPIS_SENT, 1081 vlapic->vcpuid, 1); 1082 VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec); 1083 } 1084 1085 int 1086 vlapic_pending_intr(struct vlapic *vlapic, int *vecptr) 1087 { 1088 struct LAPIC *lapic = vlapic->apic_page; 1089 int idx, i, bitpos, vector; 1090 uint32_t *irrptr, val; 1091 1092 vlapic_update_ppr(vlapic); 1093 1094 if (vlapic->ops.pending_intr) 1095 return ((*vlapic->ops.pending_intr)(vlapic, vecptr)); 1096 1097 irrptr = &lapic->irr0; 1098 1099 for (i = 7; i >= 0; i--) { 1100 idx = i * 4; 1101 val = atomic_load_acq_int(&irrptr[idx]); 1102 bitpos = fls(val); 1103 if (bitpos != 0) { 1104 vector = i * 32 + (bitpos - 1); 1105 if (PRIO(vector) > PRIO(lapic->ppr)) { 1106 VLAPIC_CTR1(vlapic, "pending intr %d", vector); 1107 if (vecptr != NULL) 1108 *vecptr = vector; 1109 return (1); 1110 } else 1111 break; 1112 } 1113 } 1114 return (0); 1115 } 1116 1117 void 1118 vlapic_intr_accepted(struct vlapic *vlapic, int vector) 1119 { 1120 struct LAPIC *lapic = vlapic->apic_page; 1121 uint32_t *irrptr, *isrptr; 1122 int idx, stk_top; 1123 1124 if (vlapic->ops.intr_accepted) 1125 return ((*vlapic->ops.intr_accepted)(vlapic, vector)); 1126 1127 /* 1128 * clear the ready bit for vector being accepted in irr 1129 * and set the vector as in service in isr. 1130 */ 1131 idx = (vector / 32) * 4; 1132 1133 irrptr = &lapic->irr0; 1134 atomic_clear_int(&irrptr[idx], 1 << (vector % 32)); 1135 VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted"); 1136 1137 isrptr = &lapic->isr0; 1138 isrptr[idx] |= 1 << (vector % 32); 1139 VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted"); 1140 1141 /* 1142 * Update the PPR 1143 */ 1144 vlapic->isrvec_stk_top++; 1145 1146 stk_top = vlapic->isrvec_stk_top; 1147 if (stk_top >= ISRVEC_STK_SIZE) 1148 panic("isrvec_stk_top overflow %d", stk_top); 1149 1150 vlapic->isrvec_stk[stk_top] = vector; 1151 } 1152 1153 void 1154 vlapic_svr_write_handler(struct vlapic *vlapic) 1155 { 1156 struct LAPIC *lapic; 1157 uint32_t old, new, changed; 1158 1159 lapic = vlapic->apic_page; 1160 1161 new = lapic->svr; 1162 old = vlapic->svr_last; 1163 vlapic->svr_last = new; 1164 1165 changed = old ^ new; 1166 if ((changed & APIC_SVR_ENABLE) != 0) { 1167 if ((new & APIC_SVR_ENABLE) == 0) { 1168 /* 1169 * The apic is now disabled so stop the apic timer 1170 * and mask all the LVT entries. 1171 */ 1172 VLAPIC_CTR0(vlapic, "vlapic is software-disabled"); 1173 VLAPIC_TIMER_LOCK(vlapic); 1174 callout_stop(&vlapic->callout); 1175 VLAPIC_TIMER_UNLOCK(vlapic); 1176 vlapic_mask_lvts(vlapic); 1177 } else { 1178 /* 1179 * The apic is now enabled so restart the apic timer 1180 * if it is configured in periodic mode. 1181 */ 1182 VLAPIC_CTR0(vlapic, "vlapic is software-enabled"); 1183 if (vlapic_periodic_timer(vlapic)) 1184 vlapic_icrtmr_write_handler(vlapic); 1185 } 1186 } 1187 } 1188 1189 int 1190 vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset, 1191 uint64_t *data, bool *retu) 1192 { 1193 struct LAPIC *lapic = vlapic->apic_page; 1194 uint32_t *reg; 1195 int i; 1196 1197 /* Ignore MMIO accesses in x2APIC mode */ 1198 if (x2apic(vlapic) && mmio_access) { 1199 VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode", 1200 offset); 1201 *data = 0; 1202 goto done; 1203 } 1204 1205 if (!x2apic(vlapic) && !mmio_access) { 1206 /* 1207 * XXX Generate GP fault for MSR accesses in xAPIC mode 1208 */ 1209 VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in " 1210 "xAPIC mode", offset); 1211 *data = 0; 1212 goto done; 1213 } 1214 1215 if (offset > sizeof(*lapic)) { 1216 *data = 0; 1217 goto done; 1218 } 1219 1220 offset &= ~3; 1221 switch(offset) 1222 { 1223 case APIC_OFFSET_ID: 1224 *data = lapic->id; 1225 break; 1226 case APIC_OFFSET_VER: 1227 *data = lapic->version; 1228 break; 1229 case APIC_OFFSET_TPR: 1230 *data = vlapic_get_tpr(vlapic); 1231 break; 1232 case APIC_OFFSET_APR: 1233 *data = lapic->apr; 1234 break; 1235 case APIC_OFFSET_PPR: 1236 *data = lapic->ppr; 1237 break; 1238 case APIC_OFFSET_EOI: 1239 *data = lapic->eoi; 1240 break; 1241 case APIC_OFFSET_LDR: 1242 *data = lapic->ldr; 1243 break; 1244 case APIC_OFFSET_DFR: 1245 *data = lapic->dfr; 1246 break; 1247 case APIC_OFFSET_SVR: 1248 *data = lapic->svr; 1249 break; 1250 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 1251 i = (offset - APIC_OFFSET_ISR0) >> 2; 1252 reg = &lapic->isr0; 1253 *data = *(reg + i); 1254 break; 1255 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 1256 i = (offset - APIC_OFFSET_TMR0) >> 2; 1257 reg = &lapic->tmr0; 1258 *data = *(reg + i); 1259 break; 1260 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 1261 i = (offset - APIC_OFFSET_IRR0) >> 2; 1262 reg = &lapic->irr0; 1263 *data = atomic_load_acq_int(reg + i); 1264 break; 1265 case APIC_OFFSET_ESR: 1266 *data = lapic->esr; 1267 break; 1268 case APIC_OFFSET_ICR_LOW: 1269 *data = lapic->icr_lo; 1270 if (x2apic(vlapic)) 1271 *data |= (uint64_t)lapic->icr_hi << 32; 1272 break; 1273 case APIC_OFFSET_ICR_HI: 1274 *data = lapic->icr_hi; 1275 break; 1276 case APIC_OFFSET_CMCI_LVT: 1277 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 1278 *data = vlapic_get_lvt(vlapic, offset); 1279 #ifdef INVARIANTS 1280 reg = vlapic_get_lvtptr(vlapic, offset); 1281 KASSERT(*data == *reg, ("inconsistent lvt value at " 1282 "offset %#lx: %#lx/%#x", offset, *data, *reg)); 1283 #endif 1284 break; 1285 case APIC_OFFSET_TIMER_ICR: 1286 *data = lapic->icr_timer; 1287 break; 1288 case APIC_OFFSET_TIMER_CCR: 1289 *data = vlapic_get_ccr(vlapic); 1290 break; 1291 case APIC_OFFSET_TIMER_DCR: 1292 *data = lapic->dcr_timer; 1293 break; 1294 case APIC_OFFSET_SELF_IPI: 1295 /* 1296 * XXX generate a GP fault if vlapic is in x2apic mode 1297 */ 1298 *data = 0; 1299 break; 1300 case APIC_OFFSET_RRR: 1301 default: 1302 *data = 0; 1303 break; 1304 } 1305 done: 1306 VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data); 1307 return 0; 1308 } 1309 1310 int 1311 vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset, 1312 uint64_t data, bool *retu) 1313 { 1314 struct LAPIC *lapic = vlapic->apic_page; 1315 uint32_t *regptr; 1316 int retval; 1317 1318 KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE, 1319 ("vlapic_write: invalid offset %#lx", offset)); 1320 1321 VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx", 1322 offset, data); 1323 1324 if (offset > sizeof(*lapic)) 1325 return (0); 1326 1327 /* Ignore MMIO accesses in x2APIC mode */ 1328 if (x2apic(vlapic) && mmio_access) { 1329 VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx " 1330 "in x2APIC mode", data, offset); 1331 return (0); 1332 } 1333 1334 /* 1335 * XXX Generate GP fault for MSR accesses in xAPIC mode 1336 */ 1337 if (!x2apic(vlapic) && !mmio_access) { 1338 VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx " 1339 "in xAPIC mode", data, offset); 1340 return (0); 1341 } 1342 1343 retval = 0; 1344 switch(offset) 1345 { 1346 case APIC_OFFSET_ID: 1347 lapic->id = data; 1348 vlapic_id_write_handler(vlapic); 1349 break; 1350 case APIC_OFFSET_TPR: 1351 vlapic_set_tpr(vlapic, data & 0xff); 1352 break; 1353 case APIC_OFFSET_EOI: 1354 vlapic_process_eoi(vlapic); 1355 break; 1356 case APIC_OFFSET_LDR: 1357 lapic->ldr = data; 1358 vlapic_ldr_write_handler(vlapic); 1359 break; 1360 case APIC_OFFSET_DFR: 1361 lapic->dfr = data; 1362 vlapic_dfr_write_handler(vlapic); 1363 break; 1364 case APIC_OFFSET_SVR: 1365 lapic->svr = data; 1366 vlapic_svr_write_handler(vlapic); 1367 break; 1368 case APIC_OFFSET_ICR_LOW: 1369 lapic->icr_lo = data; 1370 if (x2apic(vlapic)) 1371 lapic->icr_hi = data >> 32; 1372 retval = vlapic_icrlo_write_handler(vlapic, retu); 1373 break; 1374 case APIC_OFFSET_ICR_HI: 1375 lapic->icr_hi = data; 1376 break; 1377 case APIC_OFFSET_CMCI_LVT: 1378 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 1379 regptr = vlapic_get_lvtptr(vlapic, offset); 1380 *regptr = data; 1381 vlapic_lvt_write_handler(vlapic, offset); 1382 break; 1383 case APIC_OFFSET_TIMER_ICR: 1384 lapic->icr_timer = data; 1385 vlapic_icrtmr_write_handler(vlapic); 1386 break; 1387 1388 case APIC_OFFSET_TIMER_DCR: 1389 lapic->dcr_timer = data; 1390 vlapic_dcr_write_handler(vlapic); 1391 break; 1392 1393 case APIC_OFFSET_ESR: 1394 vlapic_esr_write_handler(vlapic); 1395 break; 1396 1397 case APIC_OFFSET_SELF_IPI: 1398 if (x2apic(vlapic)) 1399 vlapic_self_ipi_handler(vlapic, data); 1400 break; 1401 1402 case APIC_OFFSET_VER: 1403 case APIC_OFFSET_APR: 1404 case APIC_OFFSET_PPR: 1405 case APIC_OFFSET_RRR: 1406 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 1407 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 1408 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 1409 case APIC_OFFSET_TIMER_CCR: 1410 default: 1411 // Read only. 1412 break; 1413 } 1414 1415 return (retval); 1416 } 1417 1418 static void 1419 vlapic_reset(struct vlapic *vlapic) 1420 { 1421 struct LAPIC *lapic; 1422 1423 lapic = vlapic->apic_page; 1424 bzero(lapic, sizeof(struct LAPIC)); 1425 1426 lapic->id = vlapic_get_id(vlapic); 1427 lapic->version = VLAPIC_VERSION; 1428 lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT); 1429 lapic->dfr = 0xffffffff; 1430 lapic->svr = APIC_SVR_VECTOR; 1431 vlapic_mask_lvts(vlapic); 1432 vlapic_reset_tmr(vlapic); 1433 1434 lapic->dcr_timer = 0; 1435 vlapic_dcr_write_handler(vlapic); 1436 1437 if (vlapic->vcpuid == 0) 1438 vlapic->boot_state = BS_RUNNING; /* BSP */ 1439 else 1440 vlapic->boot_state = BS_INIT; /* AP */ 1441 1442 vlapic->svr_last = lapic->svr; 1443 } 1444 1445 void 1446 vlapic_init(struct vlapic *vlapic) 1447 { 1448 KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized")); 1449 KASSERT(vlapic->vcpuid >= 0 && 1450 vlapic->vcpuid < vm_get_maxcpus(vlapic->vm), 1451 ("vlapic_init: vcpuid is not initialized")); 1452 KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not " 1453 "initialized")); 1454 1455 /* 1456 * If the vlapic is configured in x2apic mode then it will be 1457 * accessed in the critical section via the MSR emulation code. 1458 * 1459 * Therefore the timer mutex must be a spinlock because blockable 1460 * mutexes cannot be acquired in a critical section. 1461 */ 1462 mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN); 1463 callout_init(&vlapic->callout, 1); 1464 1465 vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED; 1466 1467 if (vlapic->vcpuid == 0) 1468 vlapic->msr_apicbase |= APICBASE_BSP; 1469 1470 vlapic_reset(vlapic); 1471 } 1472 1473 void 1474 vlapic_cleanup(struct vlapic *vlapic) 1475 { 1476 1477 callout_drain(&vlapic->callout); 1478 } 1479 1480 uint64_t 1481 vlapic_get_apicbase(struct vlapic *vlapic) 1482 { 1483 1484 return (vlapic->msr_apicbase); 1485 } 1486 1487 int 1488 vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new) 1489 { 1490 1491 if (vlapic->msr_apicbase != new) { 1492 VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx " 1493 "not supported", vlapic->msr_apicbase, new); 1494 return (-1); 1495 } 1496 1497 return (0); 1498 } 1499 1500 void 1501 vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 1502 { 1503 struct vlapic *vlapic; 1504 struct LAPIC *lapic; 1505 1506 vlapic = vm_lapic(vm, vcpuid); 1507 1508 if (state == X2APIC_DISABLED) 1509 vlapic->msr_apicbase &= ~APICBASE_X2APIC; 1510 else 1511 vlapic->msr_apicbase |= APICBASE_X2APIC; 1512 1513 /* 1514 * Reset the local APIC registers whose values are mode-dependent. 1515 * 1516 * XXX this works because the APIC mode can be changed only at vcpu 1517 * initialization time. 1518 */ 1519 lapic = vlapic->apic_page; 1520 lapic->id = vlapic_get_id(vlapic); 1521 if (x2apic(vlapic)) { 1522 lapic->ldr = x2apic_ldr(vlapic); 1523 lapic->dfr = 0; 1524 } else { 1525 lapic->ldr = 0; 1526 lapic->dfr = 0xffffffff; 1527 } 1528 1529 if (state == X2APIC_ENABLED) { 1530 if (vlapic->ops.enable_x2apic_mode) 1531 (*vlapic->ops.enable_x2apic_mode)(vlapic); 1532 } 1533 } 1534 1535 void 1536 vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys, 1537 int delmode, int vec) 1538 { 1539 bool lowprio; 1540 int vcpuid; 1541 cpuset_t dmask; 1542 1543 if (delmode != IOART_DELFIXED && 1544 delmode != IOART_DELLOPRI && 1545 delmode != IOART_DELEXINT) { 1546 VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode); 1547 return; 1548 } 1549 lowprio = (delmode == IOART_DELLOPRI); 1550 1551 /* 1552 * We don't provide any virtual interrupt redirection hardware so 1553 * all interrupts originating from the ioapic or MSI specify the 1554 * 'dest' in the legacy xAPIC format. 1555 */ 1556 vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false); 1557 1558 CPU_FOREACH_ISSET(vcpuid, &dmask) { 1559 if (delmode == IOART_DELEXINT) { 1560 vm_inject_extint(vm, vcpuid); 1561 } else { 1562 lapic_set_intr(vm, vcpuid, vec, level); 1563 } 1564 } 1565 } 1566 1567 void 1568 vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum) 1569 { 1570 /* 1571 * Post an interrupt to the vcpu currently running on 'hostcpu'. 1572 * 1573 * This is done by leveraging features like Posted Interrupts (Intel) 1574 * Doorbell MSR (AMD AVIC) that avoid a VM exit. 1575 * 1576 * If neither of these features are available then fallback to 1577 * sending an IPI to 'hostcpu'. 1578 */ 1579 if (vlapic->ops.post_intr) 1580 (*vlapic->ops.post_intr)(vlapic, hostcpu); 1581 else 1582 ipi_cpu(hostcpu, ipinum); 1583 } 1584 1585 bool 1586 vlapic_enabled(struct vlapic *vlapic) 1587 { 1588 struct LAPIC *lapic = vlapic->apic_page; 1589 1590 if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 && 1591 (lapic->svr & APIC_SVR_ENABLE) != 0) 1592 return (true); 1593 else 1594 return (false); 1595 } 1596 1597 static void 1598 vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level) 1599 { 1600 struct LAPIC *lapic; 1601 uint32_t *tmrptr, mask; 1602 int idx; 1603 1604 lapic = vlapic->apic_page; 1605 tmrptr = &lapic->tmr0; 1606 idx = (vector / 32) * 4; 1607 mask = 1 << (vector % 32); 1608 if (level) 1609 tmrptr[idx] |= mask; 1610 else 1611 tmrptr[idx] &= ~mask; 1612 1613 if (vlapic->ops.set_tmr != NULL) 1614 (*vlapic->ops.set_tmr)(vlapic, vector, level); 1615 } 1616 1617 void 1618 vlapic_reset_tmr(struct vlapic *vlapic) 1619 { 1620 int vector; 1621 1622 VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered"); 1623 1624 for (vector = 0; vector <= 255; vector++) 1625 vlapic_set_tmr(vlapic, vector, false); 1626 } 1627 1628 void 1629 vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys, 1630 int delmode, int vector) 1631 { 1632 cpuset_t dmask; 1633 bool lowprio; 1634 1635 KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); 1636 1637 /* 1638 * A level trigger is valid only for fixed and lowprio delivery modes. 1639 */ 1640 if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) { 1641 VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for " 1642 "delivery-mode %d", delmode); 1643 return; 1644 } 1645 1646 lowprio = (delmode == APIC_DELMODE_LOWPRIO); 1647 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false); 1648 1649 if (!CPU_ISSET(vlapic->vcpuid, &dmask)) 1650 return; 1651 1652 VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector); 1653 vlapic_set_tmr(vlapic, vector, true); 1654 } 1655 1656 #ifdef BHYVE_SNAPSHOT 1657 static void 1658 vlapic_reset_callout(struct vlapic *vlapic, uint32_t ccr) 1659 { 1660 /* The implementation is similar to the one in the 1661 * `vlapic_icrtmr_write_handler` function 1662 */ 1663 sbintime_t sbt; 1664 struct bintime bt; 1665 1666 VLAPIC_TIMER_LOCK(vlapic); 1667 1668 bt = vlapic->timer_freq_bt; 1669 bintime_mul(&bt, ccr); 1670 1671 if (ccr != 0) { 1672 binuptime(&vlapic->timer_fire_bt); 1673 bintime_add(&vlapic->timer_fire_bt, &bt); 1674 1675 sbt = bttosbt(bt); 1676 vlapic_callout_reset(vlapic, sbt); 1677 } else { 1678 /* even if the CCR was 0, periodic timers should be reset */ 1679 if (vlapic_periodic_timer(vlapic)) { 1680 binuptime(&vlapic->timer_fire_bt); 1681 bintime_add(&vlapic->timer_fire_bt, 1682 &vlapic->timer_period_bt); 1683 sbt = bttosbt(vlapic->timer_period_bt); 1684 1685 callout_stop(&vlapic->callout); 1686 vlapic_callout_reset(vlapic, sbt); 1687 } 1688 } 1689 1690 VLAPIC_TIMER_UNLOCK(vlapic); 1691 } 1692 1693 int 1694 vlapic_snapshot(struct vm *vm, struct vm_snapshot_meta *meta) 1695 { 1696 int i, ret; 1697 struct vlapic *vlapic; 1698 struct LAPIC *lapic; 1699 uint32_t ccr; 1700 1701 KASSERT(vm != NULL, ("%s: arg was NULL", __func__)); 1702 1703 ret = 0; 1704 1705 for (i = 0; i < VM_MAXCPU; i++) { 1706 vlapic = vm_lapic(vm, i); 1707 1708 /* snapshot the page first; timer period depends on icr_timer */ 1709 lapic = vlapic->apic_page; 1710 SNAPSHOT_BUF_OR_LEAVE(lapic, PAGE_SIZE, meta, ret, done); 1711 1712 SNAPSHOT_VAR_OR_LEAVE(vlapic->esr_pending, meta, ret, done); 1713 1714 SNAPSHOT_VAR_OR_LEAVE(vlapic->timer_freq_bt.sec, 1715 meta, ret, done); 1716 SNAPSHOT_VAR_OR_LEAVE(vlapic->timer_freq_bt.frac, 1717 meta, ret, done); 1718 1719 /* 1720 * Timer period is equal to 'icr_timer' ticks at a frequency of 1721 * 'timer_freq_bt'. 1722 */ 1723 if (meta->op == VM_SNAPSHOT_RESTORE) { 1724 vlapic->timer_period_bt = vlapic->timer_freq_bt; 1725 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer); 1726 } 1727 1728 SNAPSHOT_BUF_OR_LEAVE(vlapic->isrvec_stk, 1729 sizeof(vlapic->isrvec_stk), 1730 meta, ret, done); 1731 SNAPSHOT_VAR_OR_LEAVE(vlapic->isrvec_stk_top, meta, ret, done); 1732 SNAPSHOT_VAR_OR_LEAVE(vlapic->boot_state, meta, ret, done); 1733 1734 SNAPSHOT_BUF_OR_LEAVE(vlapic->lvt_last, 1735 sizeof(vlapic->lvt_last), 1736 meta, ret, done); 1737 1738 if (meta->op == VM_SNAPSHOT_SAVE) 1739 ccr = vlapic_get_ccr(vlapic); 1740 1741 SNAPSHOT_VAR_OR_LEAVE(ccr, meta, ret, done); 1742 1743 if (meta->op == VM_SNAPSHOT_RESTORE) { 1744 /* Reset the value of the 'timer_fire_bt' and the vlapic 1745 * callout based on the value of the current count 1746 * register saved when the VM snapshot was created 1747 */ 1748 vlapic_reset_callout(vlapic, ccr); 1749 } 1750 } 1751 1752 done: 1753 return (ret); 1754 } 1755 #endif 1756