1366f6083SPeter Grehan /*- 2366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 3366f6083SPeter Grehan * All rights reserved. 4366f6083SPeter Grehan * 5366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 6366f6083SPeter Grehan * modification, are permitted provided that the following conditions 7366f6083SPeter Grehan * are met: 8366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 9366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 10366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 12366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 13366f6083SPeter Grehan * 14366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24366f6083SPeter Grehan * SUCH DAMAGE. 25366f6083SPeter Grehan * 26366f6083SPeter Grehan * $FreeBSD$ 27366f6083SPeter Grehan */ 28366f6083SPeter Grehan 29366f6083SPeter Grehan #include <sys/cdefs.h> 30366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 31366f6083SPeter Grehan 32366f6083SPeter Grehan #include <sys/param.h> 33fb03ca4eSNeel Natu #include <sys/lock.h> 34366f6083SPeter Grehan #include <sys/kernel.h> 35366f6083SPeter Grehan #include <sys/malloc.h> 36fb03ca4eSNeel Natu #include <sys/mutex.h> 37366f6083SPeter Grehan #include <sys/systm.h> 38a5615c90SPeter Grehan #include <sys/smp.h> 39366f6083SPeter Grehan 40366f6083SPeter Grehan #include <machine/clock.h> 412d3a73edSNeel Natu #include <x86/specialreg.h> 4234a6b2d6SJohn Baldwin #include <x86/apicreg.h> 43366f6083SPeter Grehan 44366f6083SPeter Grehan #include <machine/vmm.h> 45366f6083SPeter Grehan 4677d8fd9bSNeel Natu #include "vmm_stat.h" 47366f6083SPeter Grehan #include "vmm_lapic.h" 48366f6083SPeter Grehan #include "vmm_ktr.h" 49366f6083SPeter Grehan #include "vlapic.h" 50b5b28fc9SNeel Natu #include "vioapic.h" 51366f6083SPeter Grehan 52366f6083SPeter Grehan #define VLAPIC_CTR0(vlapic, format) \ 53513c8d33SNeel Natu VCPU_CTR0((vlapic)->vm, (vlapic)->vcpuid, format) 54366f6083SPeter Grehan 55366f6083SPeter Grehan #define VLAPIC_CTR1(vlapic, format, p1) \ 56513c8d33SNeel Natu VCPU_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1) 57366f6083SPeter Grehan 581c052192SNeel Natu #define VLAPIC_CTR2(vlapic, format, p1, p2) \ 591c052192SNeel Natu VCPU_CTR2((vlapic)->vm, (vlapic)->vcpuid, format, p1, p2) 601c052192SNeel Natu 61366f6083SPeter Grehan #define VLAPIC_CTR_IRR(vlapic, msg) \ 62366f6083SPeter Grehan do { \ 63366f6083SPeter Grehan uint32_t *irrptr = &(vlapic)->apic.irr0; \ 64366f6083SPeter Grehan irrptr[0] = irrptr[0]; /* silence compiler */ \ 65366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]); \ 66366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]); \ 67366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]); \ 68366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]); \ 69366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]); \ 70366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]); \ 71366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]); \ 72366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]); \ 73366f6083SPeter Grehan } while (0) 74366f6083SPeter Grehan 75366f6083SPeter Grehan #define VLAPIC_CTR_ISR(vlapic, msg) \ 76366f6083SPeter Grehan do { \ 77366f6083SPeter Grehan uint32_t *isrptr = &(vlapic)->apic.isr0; \ 78366f6083SPeter Grehan isrptr[0] = isrptr[0]; /* silence compiler */ \ 79366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]); \ 80366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]); \ 81366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]); \ 82366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]); \ 83366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]); \ 84366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]); \ 85366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]); \ 86366f6083SPeter Grehan VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]); \ 87366f6083SPeter Grehan } while (0) 88366f6083SPeter Grehan 89366f6083SPeter Grehan static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic"); 90366f6083SPeter Grehan 91366f6083SPeter Grehan #define PRIO(x) ((x) >> 4) 92366f6083SPeter Grehan 93366f6083SPeter Grehan #define VLAPIC_VERSION (16) 94366f6083SPeter Grehan #define VLAPIC_MAXLVT_ENTRIES (5) 95366f6083SPeter Grehan 96a2da7af6SNeel Natu #define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0) 972d3a73edSNeel Natu 98edf89256SNeel Natu enum boot_state { 99edf89256SNeel Natu BS_INIT, 100edf89256SNeel Natu BS_SIPI, 101edf89256SNeel Natu BS_RUNNING 102edf89256SNeel Natu }; 103edf89256SNeel Natu 104366f6083SPeter Grehan struct vlapic { 105366f6083SPeter Grehan struct vm *vm; 106366f6083SPeter Grehan int vcpuid; 107366f6083SPeter Grehan 108366f6083SPeter Grehan struct LAPIC apic; 109366f6083SPeter Grehan 110366f6083SPeter Grehan int esr_update; 111366f6083SPeter Grehan 112fb03ca4eSNeel Natu struct callout callout; /* vlapic timer */ 113fb03ca4eSNeel Natu struct bintime timer_fire_bt; /* callout expiry time */ 114fb03ca4eSNeel Natu struct bintime timer_freq_bt; /* timer frequency */ 115fb03ca4eSNeel Natu struct bintime timer_period_bt; /* timer period */ 116fb03ca4eSNeel Natu struct mtx timer_mtx; 117366f6083SPeter Grehan 118366f6083SPeter Grehan /* 119366f6083SPeter Grehan * The 'isrvec_stk' is a stack of vectors injected by the local apic. 120366f6083SPeter Grehan * A vector is popped from the stack when the processor does an EOI. 121366f6083SPeter Grehan * The vector on the top of the stack is used to compute the 122366f6083SPeter Grehan * Processor Priority in conjunction with the TPR. 123366f6083SPeter Grehan */ 124366f6083SPeter Grehan uint8_t isrvec_stk[ISRVEC_STK_SIZE]; 125366f6083SPeter Grehan int isrvec_stk_top; 1262d3a73edSNeel Natu 1272d3a73edSNeel Natu uint64_t msr_apicbase; 128edf89256SNeel Natu enum boot_state boot_state; 129366f6083SPeter Grehan }; 130366f6083SPeter Grehan 131fb03ca4eSNeel Natu /* 132fb03ca4eSNeel Natu * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the 133fb03ca4eSNeel Natu * vlapic_callout_handler() and vcpu accesses to the following registers: 134fb03ca4eSNeel Natu * - initial count register aka icr_timer 135fb03ca4eSNeel Natu * - current count register aka ccr_timer 136fb03ca4eSNeel Natu * - divide config register aka dcr_timer 137fb03ca4eSNeel Natu * - timer LVT register 138fb03ca4eSNeel Natu * 139fb03ca4eSNeel Natu * Note that the vlapic_callout_handler() does not write to any of these 140fb03ca4eSNeel Natu * registers so they can be safely read from the vcpu context without locking. 141fb03ca4eSNeel Natu */ 142*becd9849SNeel Natu #define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx)) 143*becd9849SNeel Natu #define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx)) 144fb03ca4eSNeel Natu #define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx)) 145fb03ca4eSNeel Natu 1462e25737aSNeel Natu #define VLAPIC_BUS_FREQ tsc_freq 1472e25737aSNeel Natu 1482e25737aSNeel Natu static int 1492e25737aSNeel Natu vlapic_timer_divisor(uint32_t dcr) 1502e25737aSNeel Natu { 1512e25737aSNeel Natu switch (dcr & 0xB) { 152117e8f37SPeter Grehan case APIC_TDCR_1: 153117e8f37SPeter Grehan return (1); 1542e25737aSNeel Natu case APIC_TDCR_2: 1552e25737aSNeel Natu return (2); 1562e25737aSNeel Natu case APIC_TDCR_4: 1572e25737aSNeel Natu return (4); 1582e25737aSNeel Natu case APIC_TDCR_8: 1592e25737aSNeel Natu return (8); 1602e25737aSNeel Natu case APIC_TDCR_16: 1612e25737aSNeel Natu return (16); 1622e25737aSNeel Natu case APIC_TDCR_32: 1632e25737aSNeel Natu return (32); 1642e25737aSNeel Natu case APIC_TDCR_64: 1652e25737aSNeel Natu return (64); 1662e25737aSNeel Natu case APIC_TDCR_128: 1672e25737aSNeel Natu return (128); 1682e25737aSNeel Natu default: 1692e25737aSNeel Natu panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr); 1702e25737aSNeel Natu } 1712e25737aSNeel Natu } 1722e25737aSNeel Natu 173366f6083SPeter Grehan static void 174366f6083SPeter Grehan vlapic_mask_lvts(uint32_t *lvts, int num_lvt) 175366f6083SPeter Grehan { 176366f6083SPeter Grehan int i; 177366f6083SPeter Grehan for (i = 0; i < num_lvt; i++) { 178366f6083SPeter Grehan *lvts |= APIC_LVT_M; 179366f6083SPeter Grehan lvts += 4; 180366f6083SPeter Grehan } 181366f6083SPeter Grehan } 182366f6083SPeter Grehan 183366f6083SPeter Grehan #if 0 184366f6083SPeter Grehan static inline void 185366f6083SPeter Grehan vlapic_dump_lvt(uint32_t offset, uint32_t *lvt) 186366f6083SPeter Grehan { 187366f6083SPeter Grehan printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset, 188366f6083SPeter Grehan *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS, 189366f6083SPeter Grehan *lvt & APIC_LVTT_M); 190366f6083SPeter Grehan } 191366f6083SPeter Grehan #endif 192366f6083SPeter Grehan 193fb03ca4eSNeel Natu static uint32_t 194366f6083SPeter Grehan vlapic_get_ccr(struct vlapic *vlapic) 195366f6083SPeter Grehan { 196fb03ca4eSNeel Natu struct bintime bt_now, bt_rem; 197fb03ca4eSNeel Natu struct LAPIC *lapic; 198fb03ca4eSNeel Natu uint32_t ccr; 199fb03ca4eSNeel Natu 200fb03ca4eSNeel Natu ccr = 0; 201fb03ca4eSNeel Natu lapic = &vlapic->apic; 202fb03ca4eSNeel Natu 203fb03ca4eSNeel Natu VLAPIC_TIMER_LOCK(vlapic); 204fb03ca4eSNeel Natu if (callout_active(&vlapic->callout)) { 205fb03ca4eSNeel Natu /* 206fb03ca4eSNeel Natu * If the timer is scheduled to expire in the future then 207fb03ca4eSNeel Natu * compute the value of 'ccr' based on the remaining time. 208fb03ca4eSNeel Natu */ 209fb03ca4eSNeel Natu binuptime(&bt_now); 210fb03ca4eSNeel Natu if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) { 211fb03ca4eSNeel Natu bt_rem = vlapic->timer_fire_bt; 212fb03ca4eSNeel Natu bintime_sub(&bt_rem, &bt_now); 213fb03ca4eSNeel Natu ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt); 214fb03ca4eSNeel Natu ccr += bt_rem.frac / vlapic->timer_freq_bt.frac; 215fb03ca4eSNeel Natu } 216fb03ca4eSNeel Natu } 217fb03ca4eSNeel Natu KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, " 218fb03ca4eSNeel Natu "icr_timer is %#x", ccr, lapic->icr_timer)); 219fb03ca4eSNeel Natu VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x", 220fb03ca4eSNeel Natu ccr, lapic->icr_timer); 221fb03ca4eSNeel Natu VLAPIC_TIMER_UNLOCK(vlapic); 222fb03ca4eSNeel Natu return (ccr); 223fb03ca4eSNeel Natu } 224fb03ca4eSNeel Natu 225fb03ca4eSNeel Natu static void 226fb03ca4eSNeel Natu vlapic_set_dcr(struct vlapic *vlapic, uint32_t dcr) 227fb03ca4eSNeel Natu { 228fb03ca4eSNeel Natu struct LAPIC *lapic; 229fb03ca4eSNeel Natu int divisor; 230fb03ca4eSNeel Natu 231fb03ca4eSNeel Natu lapic = &vlapic->apic; 232fb03ca4eSNeel Natu VLAPIC_TIMER_LOCK(vlapic); 233fb03ca4eSNeel Natu 234fb03ca4eSNeel Natu lapic->dcr_timer = dcr; 235fb03ca4eSNeel Natu divisor = vlapic_timer_divisor(dcr); 236fb03ca4eSNeel Natu VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d", dcr, divisor); 237fb03ca4eSNeel Natu 238fb03ca4eSNeel Natu /* 239fb03ca4eSNeel Natu * Update the timer frequency and the timer period. 240fb03ca4eSNeel Natu * 241fb03ca4eSNeel Natu * XXX changes to the frequency divider will not take effect until 242fb03ca4eSNeel Natu * the timer is reloaded. 243fb03ca4eSNeel Natu */ 244fb03ca4eSNeel Natu FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt); 245fb03ca4eSNeel Natu vlapic->timer_period_bt = vlapic->timer_freq_bt; 246fb03ca4eSNeel Natu bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer); 247fb03ca4eSNeel Natu 248fb03ca4eSNeel Natu VLAPIC_TIMER_UNLOCK(vlapic); 249366f6083SPeter Grehan } 250366f6083SPeter Grehan 251366f6083SPeter Grehan static void 252366f6083SPeter Grehan vlapic_update_errors(struct vlapic *vlapic) 253366f6083SPeter Grehan { 254366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 255366f6083SPeter Grehan lapic->esr = 0; // XXX 256366f6083SPeter Grehan } 257366f6083SPeter Grehan 258366f6083SPeter Grehan static void 259366f6083SPeter Grehan vlapic_init_ipi(struct vlapic *vlapic) 260366f6083SPeter Grehan { 261366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 262366f6083SPeter Grehan lapic->version = VLAPIC_VERSION; 263366f6083SPeter Grehan lapic->version |= (VLAPIC_MAXLVT_ENTRIES < MAXLVTSHIFT); 264366f6083SPeter Grehan lapic->dfr = 0xffffffff; 265366f6083SPeter Grehan lapic->svr = APIC_SVR_VECTOR; 266366f6083SPeter Grehan vlapic_mask_lvts(&lapic->lvt_timer, VLAPIC_MAXLVT_ENTRIES+1); 267366f6083SPeter Grehan } 268366f6083SPeter Grehan 269366f6083SPeter Grehan static int 27003cd0501SNeel Natu vlapic_reset(struct vlapic *vlapic) 271366f6083SPeter Grehan { 272366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 273366f6083SPeter Grehan 274366f6083SPeter Grehan memset(lapic, 0, sizeof(*lapic)); 275366f6083SPeter Grehan lapic->apr = vlapic->vcpuid; 276366f6083SPeter Grehan vlapic_init_ipi(vlapic); 277fb03ca4eSNeel Natu vlapic_set_dcr(vlapic, 0); 278366f6083SPeter Grehan 279edf89256SNeel Natu if (vlapic->vcpuid == 0) 280edf89256SNeel Natu vlapic->boot_state = BS_RUNNING; /* BSP */ 281edf89256SNeel Natu else 282edf89256SNeel Natu vlapic->boot_state = BS_INIT; /* AP */ 283edf89256SNeel Natu 284366f6083SPeter Grehan return 0; 285366f6083SPeter Grehan 286366f6083SPeter Grehan } 287366f6083SPeter Grehan 288366f6083SPeter Grehan void 289b5b28fc9SNeel Natu vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level) 290366f6083SPeter Grehan { 291366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 292b5b28fc9SNeel Natu uint32_t *irrptr, *tmrptr, mask; 293366f6083SPeter Grehan int idx; 294366f6083SPeter Grehan 295366f6083SPeter Grehan if (vector < 0 || vector >= 256) 296366f6083SPeter Grehan panic("vlapic_set_intr_ready: invalid vector %d\n", vector); 297366f6083SPeter Grehan 2981c052192SNeel Natu if (!(lapic->svr & APIC_SVR_ENABLE)) { 2991c052192SNeel Natu VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring " 3001c052192SNeel Natu "interrupt %d", vector); 3011c052192SNeel Natu return; 3021c052192SNeel Natu } 3031c052192SNeel Natu 304366f6083SPeter Grehan idx = (vector / 32) * 4; 305b5b28fc9SNeel Natu mask = 1 << (vector % 32); 306b5b28fc9SNeel Natu 307366f6083SPeter Grehan irrptr = &lapic->irr0; 308b5b28fc9SNeel Natu atomic_set_int(&irrptr[idx], mask); 309b5b28fc9SNeel Natu 310b5b28fc9SNeel Natu /* 311b5b28fc9SNeel Natu * Upon acceptance of an interrupt into the IRR the corresponding 312b5b28fc9SNeel Natu * TMR bit is cleared for edge-triggered interrupts and set for 313b5b28fc9SNeel Natu * level-triggered interrupts. 314b5b28fc9SNeel Natu */ 315b5b28fc9SNeel Natu tmrptr = &lapic->tmr0; 316b5b28fc9SNeel Natu if (level) 317b5b28fc9SNeel Natu atomic_set_int(&tmrptr[idx], mask); 318b5b28fc9SNeel Natu else 319b5b28fc9SNeel Natu atomic_clear_int(&tmrptr[idx], mask); 320b5b28fc9SNeel Natu 321366f6083SPeter Grehan VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready"); 322366f6083SPeter Grehan } 323366f6083SPeter Grehan 324366f6083SPeter Grehan static __inline uint32_t * 325fb03ca4eSNeel Natu vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset) 326366f6083SPeter Grehan { 327366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 328366f6083SPeter Grehan int i; 329366f6083SPeter Grehan 330366f6083SPeter Grehan if (offset < APIC_OFFSET_TIMER_LVT || offset > APIC_OFFSET_ERROR_LVT) { 331366f6083SPeter Grehan panic("vlapic_get_lvt: invalid LVT\n"); 332366f6083SPeter Grehan } 333366f6083SPeter Grehan i = (offset - APIC_OFFSET_TIMER_LVT) >> 2; 334366f6083SPeter Grehan return ((&lapic->lvt_timer) + i);; 335366f6083SPeter Grehan } 336366f6083SPeter Grehan 337fb03ca4eSNeel Natu static __inline uint32_t 338fb03ca4eSNeel Natu vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset) 339fb03ca4eSNeel Natu { 340fb03ca4eSNeel Natu 341fb03ca4eSNeel Natu return (*vlapic_get_lvtptr(vlapic, offset)); 342fb03ca4eSNeel Natu } 343fb03ca4eSNeel Natu 344fb03ca4eSNeel Natu static void 345fb03ca4eSNeel Natu vlapic_set_lvt(struct vlapic *vlapic, uint32_t offset, uint32_t val) 346fb03ca4eSNeel Natu { 347fb03ca4eSNeel Natu uint32_t *lvtptr; 348fb03ca4eSNeel Natu struct LAPIC *lapic; 349fb03ca4eSNeel Natu 350fb03ca4eSNeel Natu lapic = &vlapic->apic; 351fb03ca4eSNeel Natu lvtptr = vlapic_get_lvtptr(vlapic, offset); 352fb03ca4eSNeel Natu 353fb03ca4eSNeel Natu if (offset == APIC_OFFSET_TIMER_LVT) 354fb03ca4eSNeel Natu VLAPIC_TIMER_LOCK(vlapic); 355fb03ca4eSNeel Natu 356fb03ca4eSNeel Natu if (!(lapic->svr & APIC_SVR_ENABLE)) 357fb03ca4eSNeel Natu val |= APIC_LVT_M; 358fb03ca4eSNeel Natu *lvtptr = val; 359fb03ca4eSNeel Natu 360fb03ca4eSNeel Natu if (offset == APIC_OFFSET_TIMER_LVT) 361fb03ca4eSNeel Natu VLAPIC_TIMER_UNLOCK(vlapic); 362fb03ca4eSNeel Natu } 363fb03ca4eSNeel Natu 364366f6083SPeter Grehan #if 1 365366f6083SPeter Grehan static void 366366f6083SPeter Grehan dump_isrvec_stk(struct vlapic *vlapic) 367366f6083SPeter Grehan { 368366f6083SPeter Grehan int i; 369366f6083SPeter Grehan uint32_t *isrptr; 370366f6083SPeter Grehan 371366f6083SPeter Grehan isrptr = &vlapic->apic.isr0; 372366f6083SPeter Grehan for (i = 0; i < 8; i++) 373366f6083SPeter Grehan printf("ISR%d 0x%08x\n", i, isrptr[i * 4]); 374366f6083SPeter Grehan 375366f6083SPeter Grehan for (i = 0; i <= vlapic->isrvec_stk_top; i++) 376366f6083SPeter Grehan printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]); 377366f6083SPeter Grehan } 378366f6083SPeter Grehan #endif 379366f6083SPeter Grehan 380366f6083SPeter Grehan /* 381366f6083SPeter Grehan * Algorithm adopted from section "Interrupt, Task and Processor Priority" 382366f6083SPeter Grehan * in Intel Architecture Manual Vol 3a. 383366f6083SPeter Grehan */ 384366f6083SPeter Grehan static void 385366f6083SPeter Grehan vlapic_update_ppr(struct vlapic *vlapic) 386366f6083SPeter Grehan { 387366f6083SPeter Grehan int isrvec, tpr, ppr; 388366f6083SPeter Grehan 389366f6083SPeter Grehan /* 390366f6083SPeter Grehan * Note that the value on the stack at index 0 is always 0. 391366f6083SPeter Grehan * 392366f6083SPeter Grehan * This is a placeholder for the value of ISRV when none of the 393366f6083SPeter Grehan * bits is set in the ISRx registers. 394366f6083SPeter Grehan */ 395366f6083SPeter Grehan isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top]; 396366f6083SPeter Grehan tpr = vlapic->apic.tpr; 397366f6083SPeter Grehan 398366f6083SPeter Grehan #if 1 399366f6083SPeter Grehan { 400366f6083SPeter Grehan int i, lastprio, curprio, vector, idx; 401366f6083SPeter Grehan uint32_t *isrptr; 402366f6083SPeter Grehan 403366f6083SPeter Grehan if (vlapic->isrvec_stk_top == 0 && isrvec != 0) 404366f6083SPeter Grehan panic("isrvec_stk is corrupted: %d", isrvec); 405366f6083SPeter Grehan 406366f6083SPeter Grehan /* 407366f6083SPeter Grehan * Make sure that the priority of the nested interrupts is 408366f6083SPeter Grehan * always increasing. 409366f6083SPeter Grehan */ 410366f6083SPeter Grehan lastprio = -1; 411366f6083SPeter Grehan for (i = 1; i <= vlapic->isrvec_stk_top; i++) { 412366f6083SPeter Grehan curprio = PRIO(vlapic->isrvec_stk[i]); 413366f6083SPeter Grehan if (curprio <= lastprio) { 414366f6083SPeter Grehan dump_isrvec_stk(vlapic); 415366f6083SPeter Grehan panic("isrvec_stk does not satisfy invariant"); 416366f6083SPeter Grehan } 417366f6083SPeter Grehan lastprio = curprio; 418366f6083SPeter Grehan } 419366f6083SPeter Grehan 420366f6083SPeter Grehan /* 421366f6083SPeter Grehan * Make sure that each bit set in the ISRx registers has a 422366f6083SPeter Grehan * corresponding entry on the isrvec stack. 423366f6083SPeter Grehan */ 424366f6083SPeter Grehan i = 1; 425366f6083SPeter Grehan isrptr = &vlapic->apic.isr0; 426366f6083SPeter Grehan for (vector = 0; vector < 256; vector++) { 427366f6083SPeter Grehan idx = (vector / 32) * 4; 428366f6083SPeter Grehan if (isrptr[idx] & (1 << (vector % 32))) { 429366f6083SPeter Grehan if (i > vlapic->isrvec_stk_top || 430366f6083SPeter Grehan vlapic->isrvec_stk[i] != vector) { 431366f6083SPeter Grehan dump_isrvec_stk(vlapic); 432366f6083SPeter Grehan panic("ISR and isrvec_stk out of sync"); 433366f6083SPeter Grehan } 434366f6083SPeter Grehan i++; 435366f6083SPeter Grehan } 436366f6083SPeter Grehan } 437366f6083SPeter Grehan } 438366f6083SPeter Grehan #endif 439366f6083SPeter Grehan 440366f6083SPeter Grehan if (PRIO(tpr) >= PRIO(isrvec)) 441366f6083SPeter Grehan ppr = tpr; 442366f6083SPeter Grehan else 443366f6083SPeter Grehan ppr = isrvec & 0xf0; 444366f6083SPeter Grehan 445366f6083SPeter Grehan vlapic->apic.ppr = ppr; 446366f6083SPeter Grehan VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr); 447366f6083SPeter Grehan } 448366f6083SPeter Grehan 449366f6083SPeter Grehan static void 450366f6083SPeter Grehan vlapic_process_eoi(struct vlapic *vlapic) 451366f6083SPeter Grehan { 452366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 453b5b28fc9SNeel Natu uint32_t *isrptr, *tmrptr; 454b5b28fc9SNeel Natu int i, idx, bitpos, vector; 455366f6083SPeter Grehan 456366f6083SPeter Grehan isrptr = &lapic->isr0; 457b5b28fc9SNeel Natu tmrptr = &lapic->tmr0; 458366f6083SPeter Grehan 459366f6083SPeter Grehan /* 460366f6083SPeter Grehan * The x86 architecture reserves the the first 32 vectors for use 461366f6083SPeter Grehan * by the processor. 462366f6083SPeter Grehan */ 463366f6083SPeter Grehan for (i = 7; i > 0; i--) { 464366f6083SPeter Grehan idx = i * 4; 465366f6083SPeter Grehan bitpos = fls(isrptr[idx]); 466b5b28fc9SNeel Natu if (bitpos-- != 0) { 467366f6083SPeter Grehan if (vlapic->isrvec_stk_top <= 0) { 468366f6083SPeter Grehan panic("invalid vlapic isrvec_stk_top %d", 469366f6083SPeter Grehan vlapic->isrvec_stk_top); 470366f6083SPeter Grehan } 471b5b28fc9SNeel Natu isrptr[idx] &= ~(1 << bitpos); 472366f6083SPeter Grehan VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi"); 473366f6083SPeter Grehan vlapic->isrvec_stk_top--; 474366f6083SPeter Grehan vlapic_update_ppr(vlapic); 475b5b28fc9SNeel Natu if ((tmrptr[idx] & (1 << bitpos)) != 0) { 476b5b28fc9SNeel Natu vector = i * 32 + bitpos; 477b5b28fc9SNeel Natu vioapic_process_eoi(vlapic->vm, vlapic->vcpuid, 478b5b28fc9SNeel Natu vector); 479b5b28fc9SNeel Natu } 480366f6083SPeter Grehan return; 481366f6083SPeter Grehan } 482366f6083SPeter Grehan } 483366f6083SPeter Grehan } 484366f6083SPeter Grehan 485366f6083SPeter Grehan static __inline int 486fb03ca4eSNeel Natu vlapic_get_lvt_field(uint32_t lvt, uint32_t mask) 487366f6083SPeter Grehan { 488fb03ca4eSNeel Natu 489fb03ca4eSNeel Natu return (lvt & mask); 490366f6083SPeter Grehan } 491366f6083SPeter Grehan 492366f6083SPeter Grehan static __inline int 493366f6083SPeter Grehan vlapic_periodic_timer(struct vlapic *vlapic) 494366f6083SPeter Grehan { 495fb03ca4eSNeel Natu uint32_t lvt; 496366f6083SPeter Grehan 497366f6083SPeter Grehan lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT); 498366f6083SPeter Grehan 499366f6083SPeter Grehan return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC)); 500366f6083SPeter Grehan } 501366f6083SPeter Grehan 50277d8fd9bSNeel Natu static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic"); 50377d8fd9bSNeel Natu 504366f6083SPeter Grehan static void 505366f6083SPeter Grehan vlapic_fire_timer(struct vlapic *vlapic) 506366f6083SPeter Grehan { 507366f6083SPeter Grehan int vector; 508fb03ca4eSNeel Natu uint32_t lvt; 509fb03ca4eSNeel Natu 510fb03ca4eSNeel Natu KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked")); 511366f6083SPeter Grehan 512366f6083SPeter Grehan lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT); 513366f6083SPeter Grehan 514366f6083SPeter Grehan if (!vlapic_get_lvt_field(lvt, APIC_LVTT_M)) { 51577d8fd9bSNeel Natu vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1); 516366f6083SPeter Grehan vector = vlapic_get_lvt_field(lvt, APIC_LVTT_VECTOR); 517b5b28fc9SNeel Natu vlapic_set_intr_ready(vlapic, vector, false); 518fb03ca4eSNeel Natu vcpu_notify_event(vlapic->vm, vlapic->vcpuid); 519366f6083SPeter Grehan } 520366f6083SPeter Grehan } 521366f6083SPeter Grehan 522fb03ca4eSNeel Natu static void 523fb03ca4eSNeel Natu vlapic_callout_handler(void *arg) 524fb03ca4eSNeel Natu { 525fb03ca4eSNeel Natu struct vlapic *vlapic; 526fb03ca4eSNeel Natu struct bintime bt, btnow; 527fb03ca4eSNeel Natu sbintime_t rem_sbt; 528fb03ca4eSNeel Natu 529fb03ca4eSNeel Natu vlapic = arg; 530fb03ca4eSNeel Natu 531fb03ca4eSNeel Natu VLAPIC_TIMER_LOCK(vlapic); 532fb03ca4eSNeel Natu if (callout_pending(&vlapic->callout)) /* callout was reset */ 533fb03ca4eSNeel Natu goto done; 534fb03ca4eSNeel Natu 535fb03ca4eSNeel Natu if (!callout_active(&vlapic->callout)) /* callout was stopped */ 536fb03ca4eSNeel Natu goto done; 537fb03ca4eSNeel Natu 538fb03ca4eSNeel Natu callout_deactivate(&vlapic->callout); 539fb03ca4eSNeel Natu 540fb03ca4eSNeel Natu KASSERT(vlapic->apic.icr_timer != 0, ("vlapic timer is disabled")); 541fb03ca4eSNeel Natu 542fb03ca4eSNeel Natu vlapic_fire_timer(vlapic); 543fb03ca4eSNeel Natu 544fb03ca4eSNeel Natu if (vlapic_periodic_timer(vlapic)) { 545fb03ca4eSNeel Natu binuptime(&btnow); 546fb03ca4eSNeel Natu KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=), 547fb03ca4eSNeel Natu ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx", 548fb03ca4eSNeel Natu btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec, 549fb03ca4eSNeel Natu vlapic->timer_fire_bt.frac)); 550fb03ca4eSNeel Natu 551fb03ca4eSNeel Natu /* 552fb03ca4eSNeel Natu * Compute the delta between when the timer was supposed to 553fb03ca4eSNeel Natu * fire and the present time. 554fb03ca4eSNeel Natu */ 555fb03ca4eSNeel Natu bt = btnow; 556fb03ca4eSNeel Natu bintime_sub(&bt, &vlapic->timer_fire_bt); 557fb03ca4eSNeel Natu 558fb03ca4eSNeel Natu rem_sbt = bttosbt(vlapic->timer_period_bt); 559fb03ca4eSNeel Natu if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) { 560fb03ca4eSNeel Natu /* 561fb03ca4eSNeel Natu * Adjust the time until the next countdown downward 562fb03ca4eSNeel Natu * to account for the lost time. 563fb03ca4eSNeel Natu */ 564fb03ca4eSNeel Natu rem_sbt -= bttosbt(bt); 565fb03ca4eSNeel Natu } else { 566fb03ca4eSNeel Natu /* 567fb03ca4eSNeel Natu * If the delta is greater than the timer period then 568fb03ca4eSNeel Natu * just reset our time base instead of trying to catch 569fb03ca4eSNeel Natu * up. 570fb03ca4eSNeel Natu */ 571fb03ca4eSNeel Natu vlapic->timer_fire_bt = btnow; 572fb03ca4eSNeel Natu VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu " 573fb03ca4eSNeel Natu "usecs, period is %lu usecs - resetting time base", 574fb03ca4eSNeel Natu bttosbt(bt) / SBT_1US, 575fb03ca4eSNeel Natu bttosbt(vlapic->timer_period_bt) / SBT_1US); 576fb03ca4eSNeel Natu } 577fb03ca4eSNeel Natu 578fb03ca4eSNeel Natu bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt); 579fb03ca4eSNeel Natu callout_reset_sbt(&vlapic->callout, rem_sbt, 0, 580fb03ca4eSNeel Natu vlapic_callout_handler, vlapic, 0); 581fb03ca4eSNeel Natu } 582fb03ca4eSNeel Natu done: 583fb03ca4eSNeel Natu VLAPIC_TIMER_UNLOCK(vlapic); 584fb03ca4eSNeel Natu } 585fb03ca4eSNeel Natu 586fb03ca4eSNeel Natu static void 587fb03ca4eSNeel Natu vlapic_set_icr_timer(struct vlapic *vlapic, uint32_t icr_timer) 588fb03ca4eSNeel Natu { 589fb03ca4eSNeel Natu struct LAPIC *lapic; 590fb03ca4eSNeel Natu sbintime_t sbt; 591fb03ca4eSNeel Natu 592fb03ca4eSNeel Natu VLAPIC_TIMER_LOCK(vlapic); 593fb03ca4eSNeel Natu 594fb03ca4eSNeel Natu lapic = &vlapic->apic; 595fb03ca4eSNeel Natu lapic->icr_timer = icr_timer; 596fb03ca4eSNeel Natu 597fb03ca4eSNeel Natu vlapic->timer_period_bt = vlapic->timer_freq_bt; 598fb03ca4eSNeel Natu bintime_mul(&vlapic->timer_period_bt, icr_timer); 599fb03ca4eSNeel Natu 600fb03ca4eSNeel Natu if (icr_timer != 0) { 601fb03ca4eSNeel Natu binuptime(&vlapic->timer_fire_bt); 602fb03ca4eSNeel Natu bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt); 603fb03ca4eSNeel Natu 604fb03ca4eSNeel Natu sbt = bttosbt(vlapic->timer_period_bt); 605fb03ca4eSNeel Natu callout_reset_sbt(&vlapic->callout, sbt, 0, 606fb03ca4eSNeel Natu vlapic_callout_handler, vlapic, 0); 607fb03ca4eSNeel Natu } else 608fb03ca4eSNeel Natu callout_stop(&vlapic->callout); 609fb03ca4eSNeel Natu 610fb03ca4eSNeel Natu VLAPIC_TIMER_UNLOCK(vlapic); 611fb03ca4eSNeel Natu } 612fb03ca4eSNeel Natu 6130acb0d84SNeel Natu static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu"); 6140acb0d84SNeel Natu 615366f6083SPeter Grehan static int 616*becd9849SNeel Natu lapic_process_icr(struct vlapic *vlapic, uint64_t icrval, bool *retu) 617366f6083SPeter Grehan { 618366f6083SPeter Grehan int i; 619a5615c90SPeter Grehan cpuset_t dmask; 620366f6083SPeter Grehan uint32_t dest, vec, mode; 621edf89256SNeel Natu struct vlapic *vlapic2; 622edf89256SNeel Natu struct vm_exit *vmexit; 623366f6083SPeter Grehan 624a2da7af6SNeel Natu if (x2apic(vlapic)) 625366f6083SPeter Grehan dest = icrval >> 32; 626a2da7af6SNeel Natu else 627a2da7af6SNeel Natu dest = icrval >> (32 + 24); 628366f6083SPeter Grehan vec = icrval & APIC_VECTOR_MASK; 629366f6083SPeter Grehan mode = icrval & APIC_DELMODE_MASK; 630366f6083SPeter Grehan 631366f6083SPeter Grehan if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) { 632366f6083SPeter Grehan switch (icrval & APIC_DEST_MASK) { 633366f6083SPeter Grehan case APIC_DEST_DESTFLD: 634a5615c90SPeter Grehan CPU_SETOF(dest, &dmask); 635366f6083SPeter Grehan break; 636366f6083SPeter Grehan case APIC_DEST_SELF: 637a5615c90SPeter Grehan CPU_SETOF(vlapic->vcpuid, &dmask); 638366f6083SPeter Grehan break; 639366f6083SPeter Grehan case APIC_DEST_ALLISELF: 640366f6083SPeter Grehan dmask = vm_active_cpus(vlapic->vm); 641366f6083SPeter Grehan break; 642366f6083SPeter Grehan case APIC_DEST_ALLESELF: 643a5615c90SPeter Grehan dmask = vm_active_cpus(vlapic->vm); 644a5615c90SPeter Grehan CPU_CLR(vlapic->vcpuid, &dmask); 645366f6083SPeter Grehan break; 6461e2751ddSSergey Kandaurov default: 6471e2751ddSSergey Kandaurov CPU_ZERO(&dmask); /* satisfy gcc */ 6481e2751ddSSergey Kandaurov break; 649366f6083SPeter Grehan } 650366f6083SPeter Grehan 65182f2974aSSergey Kandaurov while ((i = CPU_FFS(&dmask)) != 0) { 652a5615c90SPeter Grehan i--; 653a5615c90SPeter Grehan CPU_CLR(i, &dmask); 6540acb0d84SNeel Natu if (mode == APIC_DELMODE_FIXED) { 655b5b28fc9SNeel Natu lapic_intr_edge(vlapic->vm, i, vec); 6560acb0d84SNeel Natu vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, 6570acb0d84SNeel Natu IPIS_SENT, i, 1); 6580acb0d84SNeel Natu } else 659366f6083SPeter Grehan vm_inject_nmi(vlapic->vm, i); 660366f6083SPeter Grehan } 661366f6083SPeter Grehan 662366f6083SPeter Grehan return (0); /* handled completely in the kernel */ 663366f6083SPeter Grehan } 664366f6083SPeter Grehan 665edf89256SNeel Natu if (mode == APIC_DELMODE_INIT) { 666edf89256SNeel Natu if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT) 667edf89256SNeel Natu return (0); 668edf89256SNeel Natu 669edf89256SNeel Natu if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) { 670edf89256SNeel Natu vlapic2 = vm_lapic(vlapic->vm, dest); 671edf89256SNeel Natu 672edf89256SNeel Natu /* move from INIT to waiting-for-SIPI state */ 673edf89256SNeel Natu if (vlapic2->boot_state == BS_INIT) { 674edf89256SNeel Natu vlapic2->boot_state = BS_SIPI; 675edf89256SNeel Natu } 676edf89256SNeel Natu 677edf89256SNeel Natu return (0); 678edf89256SNeel Natu } 679edf89256SNeel Natu } 680edf89256SNeel Natu 681edf89256SNeel Natu if (mode == APIC_DELMODE_STARTUP) { 682edf89256SNeel Natu if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) { 683edf89256SNeel Natu vlapic2 = vm_lapic(vlapic->vm, dest); 684edf89256SNeel Natu 685edf89256SNeel Natu /* 686edf89256SNeel Natu * Ignore SIPIs in any state other than wait-for-SIPI 687edf89256SNeel Natu */ 688edf89256SNeel Natu if (vlapic2->boot_state != BS_SIPI) 689edf89256SNeel Natu return (0); 690edf89256SNeel Natu 691366f6083SPeter Grehan /* 692366f6083SPeter Grehan * XXX this assumes that the startup IPI always succeeds 693366f6083SPeter Grehan */ 694edf89256SNeel Natu vlapic2->boot_state = BS_RUNNING; 695edf89256SNeel Natu vm_activate_cpu(vlapic2->vm, dest); 696edf89256SNeel Natu 697*becd9849SNeel Natu *retu = true; 698*becd9849SNeel Natu vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid); 699*becd9849SNeel Natu vmexit->exitcode = VM_EXITCODE_SPINUP_AP; 700*becd9849SNeel Natu vmexit->u.spinup_ap.vcpu = dest; 701*becd9849SNeel Natu vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT; 702*becd9849SNeel Natu 703edf89256SNeel Natu return (0); 704edf89256SNeel Natu } 705edf89256SNeel Natu } 706366f6083SPeter Grehan 707366f6083SPeter Grehan /* 708366f6083SPeter Grehan * This will cause a return to userland. 709366f6083SPeter Grehan */ 710366f6083SPeter Grehan return (1); 711366f6083SPeter Grehan } 712366f6083SPeter Grehan 713366f6083SPeter Grehan int 714366f6083SPeter Grehan vlapic_pending_intr(struct vlapic *vlapic) 715366f6083SPeter Grehan { 716366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 717366f6083SPeter Grehan int idx, i, bitpos, vector; 718366f6083SPeter Grehan uint32_t *irrptr, val; 719366f6083SPeter Grehan 720366f6083SPeter Grehan irrptr = &lapic->irr0; 721366f6083SPeter Grehan 722366f6083SPeter Grehan /* 723366f6083SPeter Grehan * The x86 architecture reserves the the first 32 vectors for use 724366f6083SPeter Grehan * by the processor. 725366f6083SPeter Grehan */ 726366f6083SPeter Grehan for (i = 7; i > 0; i--) { 727366f6083SPeter Grehan idx = i * 4; 728366f6083SPeter Grehan val = atomic_load_acq_int(&irrptr[idx]); 729366f6083SPeter Grehan bitpos = fls(val); 730366f6083SPeter Grehan if (bitpos != 0) { 731366f6083SPeter Grehan vector = i * 32 + (bitpos - 1); 732366f6083SPeter Grehan if (PRIO(vector) > PRIO(lapic->ppr)) { 733366f6083SPeter Grehan VLAPIC_CTR1(vlapic, "pending intr %d", vector); 734366f6083SPeter Grehan return (vector); 735366f6083SPeter Grehan } else 736366f6083SPeter Grehan break; 737366f6083SPeter Grehan } 738366f6083SPeter Grehan } 739366f6083SPeter Grehan return (-1); 740366f6083SPeter Grehan } 741366f6083SPeter Grehan 742366f6083SPeter Grehan void 743366f6083SPeter Grehan vlapic_intr_accepted(struct vlapic *vlapic, int vector) 744366f6083SPeter Grehan { 745366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 746366f6083SPeter Grehan uint32_t *irrptr, *isrptr; 747366f6083SPeter Grehan int idx, stk_top; 748366f6083SPeter Grehan 749366f6083SPeter Grehan /* 750366f6083SPeter Grehan * clear the ready bit for vector being accepted in irr 751366f6083SPeter Grehan * and set the vector as in service in isr. 752366f6083SPeter Grehan */ 753366f6083SPeter Grehan idx = (vector / 32) * 4; 754366f6083SPeter Grehan 755366f6083SPeter Grehan irrptr = &lapic->irr0; 756366f6083SPeter Grehan atomic_clear_int(&irrptr[idx], 1 << (vector % 32)); 757366f6083SPeter Grehan VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted"); 758366f6083SPeter Grehan 759366f6083SPeter Grehan isrptr = &lapic->isr0; 760366f6083SPeter Grehan isrptr[idx] |= 1 << (vector % 32); 761366f6083SPeter Grehan VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted"); 762366f6083SPeter Grehan 763366f6083SPeter Grehan /* 764366f6083SPeter Grehan * Update the PPR 765366f6083SPeter Grehan */ 766366f6083SPeter Grehan vlapic->isrvec_stk_top++; 767366f6083SPeter Grehan 768366f6083SPeter Grehan stk_top = vlapic->isrvec_stk_top; 769366f6083SPeter Grehan if (stk_top >= ISRVEC_STK_SIZE) 770366f6083SPeter Grehan panic("isrvec_stk_top overflow %d", stk_top); 771366f6083SPeter Grehan 772366f6083SPeter Grehan vlapic->isrvec_stk[stk_top] = vector; 773366f6083SPeter Grehan vlapic_update_ppr(vlapic); 774366f6083SPeter Grehan } 775366f6083SPeter Grehan 7761c052192SNeel Natu static void 7771c052192SNeel Natu lapic_set_svr(struct vlapic *vlapic, uint32_t new) 7781c052192SNeel Natu { 7791c052192SNeel Natu struct LAPIC *lapic; 7801c052192SNeel Natu uint32_t old, changed; 7811c052192SNeel Natu 7821c052192SNeel Natu lapic = &vlapic->apic; 7831c052192SNeel Natu old = lapic->svr; 7841c052192SNeel Natu changed = old ^ new; 7851c052192SNeel Natu if ((changed & APIC_SVR_ENABLE) != 0) { 7861c052192SNeel Natu if ((new & APIC_SVR_ENABLE) == 0) { 787fb03ca4eSNeel Natu /* 788fb03ca4eSNeel Natu * The apic is now disabled so stop the apic timer. 789fb03ca4eSNeel Natu */ 7901c052192SNeel Natu VLAPIC_CTR0(vlapic, "vlapic is software-disabled"); 791fb03ca4eSNeel Natu VLAPIC_TIMER_LOCK(vlapic); 792fb03ca4eSNeel Natu callout_stop(&vlapic->callout); 793fb03ca4eSNeel Natu VLAPIC_TIMER_UNLOCK(vlapic); 7941c052192SNeel Natu } else { 795fb03ca4eSNeel Natu /* 796fb03ca4eSNeel Natu * The apic is now enabled so restart the apic timer 797fb03ca4eSNeel Natu * if it is configured in periodic mode. 798fb03ca4eSNeel Natu */ 7991c052192SNeel Natu VLAPIC_CTR0(vlapic, "vlapic is software-enabled"); 800fb03ca4eSNeel Natu if (vlapic_periodic_timer(vlapic)) 801fb03ca4eSNeel Natu vlapic_set_icr_timer(vlapic, lapic->icr_timer); 8021c052192SNeel Natu } 8031c052192SNeel Natu } 8041c052192SNeel Natu lapic->svr = new; 8051c052192SNeel Natu } 8061c052192SNeel Natu 807366f6083SPeter Grehan int 808*becd9849SNeel Natu vlapic_read(struct vlapic *vlapic, uint64_t offset, uint64_t *data, bool *retu) 809366f6083SPeter Grehan { 810366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 811366f6083SPeter Grehan uint32_t *reg; 812366f6083SPeter Grehan int i; 813366f6083SPeter Grehan 814366f6083SPeter Grehan if (offset > sizeof(*lapic)) { 815366f6083SPeter Grehan *data = 0; 8161c052192SNeel Natu goto done; 817366f6083SPeter Grehan } 818366f6083SPeter Grehan 819366f6083SPeter Grehan offset &= ~3; 820366f6083SPeter Grehan switch(offset) 821366f6083SPeter Grehan { 822366f6083SPeter Grehan case APIC_OFFSET_ID: 8232d3a73edSNeel Natu if (x2apic(vlapic)) 8242d3a73edSNeel Natu *data = vlapic->vcpuid; 8252d3a73edSNeel Natu else 8262d3a73edSNeel Natu *data = vlapic->vcpuid << 24; 827366f6083SPeter Grehan break; 828366f6083SPeter Grehan case APIC_OFFSET_VER: 829366f6083SPeter Grehan *data = lapic->version; 830366f6083SPeter Grehan break; 831366f6083SPeter Grehan case APIC_OFFSET_TPR: 832366f6083SPeter Grehan *data = lapic->tpr; 833366f6083SPeter Grehan break; 834366f6083SPeter Grehan case APIC_OFFSET_APR: 835366f6083SPeter Grehan *data = lapic->apr; 836366f6083SPeter Grehan break; 837366f6083SPeter Grehan case APIC_OFFSET_PPR: 838366f6083SPeter Grehan *data = lapic->ppr; 839366f6083SPeter Grehan break; 840366f6083SPeter Grehan case APIC_OFFSET_EOI: 841366f6083SPeter Grehan *data = lapic->eoi; 842366f6083SPeter Grehan break; 843366f6083SPeter Grehan case APIC_OFFSET_LDR: 844366f6083SPeter Grehan *data = lapic->ldr; 845366f6083SPeter Grehan break; 846366f6083SPeter Grehan case APIC_OFFSET_DFR: 847366f6083SPeter Grehan *data = lapic->dfr; 848366f6083SPeter Grehan break; 849366f6083SPeter Grehan case APIC_OFFSET_SVR: 850366f6083SPeter Grehan *data = lapic->svr; 851366f6083SPeter Grehan break; 852366f6083SPeter Grehan case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 853366f6083SPeter Grehan i = (offset - APIC_OFFSET_ISR0) >> 2; 854366f6083SPeter Grehan reg = &lapic->isr0; 855366f6083SPeter Grehan *data = *(reg + i); 856366f6083SPeter Grehan break; 857366f6083SPeter Grehan case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 858366f6083SPeter Grehan i = (offset - APIC_OFFSET_TMR0) >> 2; 859366f6083SPeter Grehan reg = &lapic->tmr0; 860366f6083SPeter Grehan *data = *(reg + i); 861366f6083SPeter Grehan break; 862366f6083SPeter Grehan case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 863366f6083SPeter Grehan i = (offset - APIC_OFFSET_IRR0) >> 2; 864366f6083SPeter Grehan reg = &lapic->irr0; 865366f6083SPeter Grehan *data = atomic_load_acq_int(reg + i); 866366f6083SPeter Grehan break; 867366f6083SPeter Grehan case APIC_OFFSET_ESR: 868366f6083SPeter Grehan *data = lapic->esr; 869366f6083SPeter Grehan break; 870366f6083SPeter Grehan case APIC_OFFSET_ICR_LOW: 871366f6083SPeter Grehan *data = lapic->icr_lo; 872366f6083SPeter Grehan break; 873366f6083SPeter Grehan case APIC_OFFSET_ICR_HI: 874366f6083SPeter Grehan *data = lapic->icr_hi; 875366f6083SPeter Grehan break; 876366f6083SPeter Grehan case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 877fb03ca4eSNeel Natu *data = vlapic_get_lvt(vlapic, offset); 878366f6083SPeter Grehan break; 879366f6083SPeter Grehan case APIC_OFFSET_ICR: 880366f6083SPeter Grehan *data = lapic->icr_timer; 881366f6083SPeter Grehan break; 882366f6083SPeter Grehan case APIC_OFFSET_CCR: 883366f6083SPeter Grehan *data = vlapic_get_ccr(vlapic); 884366f6083SPeter Grehan break; 885366f6083SPeter Grehan case APIC_OFFSET_DCR: 886366f6083SPeter Grehan *data = lapic->dcr_timer; 887366f6083SPeter Grehan break; 888366f6083SPeter Grehan case APIC_OFFSET_RRR: 889366f6083SPeter Grehan default: 890366f6083SPeter Grehan *data = 0; 891366f6083SPeter Grehan break; 892366f6083SPeter Grehan } 8931c052192SNeel Natu done: 8941c052192SNeel Natu VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data); 895366f6083SPeter Grehan return 0; 896366f6083SPeter Grehan } 897366f6083SPeter Grehan 898366f6083SPeter Grehan int 899*becd9849SNeel Natu vlapic_write(struct vlapic *vlapic, uint64_t offset, uint64_t data, bool *retu) 900366f6083SPeter Grehan { 901366f6083SPeter Grehan struct LAPIC *lapic = &vlapic->apic; 902366f6083SPeter Grehan int retval; 903366f6083SPeter Grehan 9041c052192SNeel Natu VLAPIC_CTR2(vlapic, "vlapic write offset %#x, data %#lx", offset, data); 9051c052192SNeel Natu 906366f6083SPeter Grehan if (offset > sizeof(*lapic)) { 907366f6083SPeter Grehan return 0; 908366f6083SPeter Grehan } 909366f6083SPeter Grehan 910366f6083SPeter Grehan retval = 0; 911366f6083SPeter Grehan offset &= ~3; 912366f6083SPeter Grehan switch(offset) 913366f6083SPeter Grehan { 914366f6083SPeter Grehan case APIC_OFFSET_ID: 915366f6083SPeter Grehan break; 916366f6083SPeter Grehan case APIC_OFFSET_TPR: 917366f6083SPeter Grehan lapic->tpr = data & 0xff; 918366f6083SPeter Grehan vlapic_update_ppr(vlapic); 919366f6083SPeter Grehan break; 920366f6083SPeter Grehan case APIC_OFFSET_EOI: 921366f6083SPeter Grehan vlapic_process_eoi(vlapic); 922366f6083SPeter Grehan break; 923366f6083SPeter Grehan case APIC_OFFSET_LDR: 924366f6083SPeter Grehan break; 925366f6083SPeter Grehan case APIC_OFFSET_DFR: 926366f6083SPeter Grehan break; 927366f6083SPeter Grehan case APIC_OFFSET_SVR: 9281c052192SNeel Natu lapic_set_svr(vlapic, data); 929366f6083SPeter Grehan break; 930366f6083SPeter Grehan case APIC_OFFSET_ICR_LOW: 931a2da7af6SNeel Natu if (!x2apic(vlapic)) { 932a2da7af6SNeel Natu data &= 0xffffffff; 933a2da7af6SNeel Natu data |= (uint64_t)lapic->icr_hi << 32; 934a2da7af6SNeel Natu } 935*becd9849SNeel Natu retval = lapic_process_icr(vlapic, data, retu); 936366f6083SPeter Grehan break; 937a2da7af6SNeel Natu case APIC_OFFSET_ICR_HI: 938a2da7af6SNeel Natu if (!x2apic(vlapic)) { 939a2da7af6SNeel Natu retval = 0; 940a2da7af6SNeel Natu lapic->icr_hi = data; 941a2da7af6SNeel Natu } 942a2da7af6SNeel Natu break; 943366f6083SPeter Grehan case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 944fb03ca4eSNeel Natu vlapic_set_lvt(vlapic, offset, data); 945366f6083SPeter Grehan break; 946366f6083SPeter Grehan case APIC_OFFSET_ICR: 947fb03ca4eSNeel Natu vlapic_set_icr_timer(vlapic, data); 948366f6083SPeter Grehan break; 949366f6083SPeter Grehan 950366f6083SPeter Grehan case APIC_OFFSET_DCR: 951fb03ca4eSNeel Natu vlapic_set_dcr(vlapic, data); 952366f6083SPeter Grehan break; 953366f6083SPeter Grehan 954366f6083SPeter Grehan case APIC_OFFSET_ESR: 955366f6083SPeter Grehan vlapic_update_errors(vlapic); 956366f6083SPeter Grehan break; 957366f6083SPeter Grehan case APIC_OFFSET_VER: 958366f6083SPeter Grehan case APIC_OFFSET_APR: 959366f6083SPeter Grehan case APIC_OFFSET_PPR: 960366f6083SPeter Grehan case APIC_OFFSET_RRR: 961366f6083SPeter Grehan case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 962366f6083SPeter Grehan case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 963366f6083SPeter Grehan case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 964366f6083SPeter Grehan case APIC_OFFSET_CCR: 965366f6083SPeter Grehan default: 966366f6083SPeter Grehan // Read only. 967366f6083SPeter Grehan break; 968366f6083SPeter Grehan } 969366f6083SPeter Grehan 970366f6083SPeter Grehan return (retval); 971366f6083SPeter Grehan } 972366f6083SPeter Grehan 973366f6083SPeter Grehan struct vlapic * 974366f6083SPeter Grehan vlapic_init(struct vm *vm, int vcpuid) 975366f6083SPeter Grehan { 976366f6083SPeter Grehan struct vlapic *vlapic; 977366f6083SPeter Grehan 978366f6083SPeter Grehan vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO); 979366f6083SPeter Grehan vlapic->vm = vm; 980366f6083SPeter Grehan vlapic->vcpuid = vcpuid; 9812d3a73edSNeel Natu 982*becd9849SNeel Natu /* 983*becd9849SNeel Natu * If the vlapic is configured in x2apic mode then it will be 984*becd9849SNeel Natu * accessed in the critical section via the MSR emulation code. 985*becd9849SNeel Natu * 986*becd9849SNeel Natu * Therefore the timer mutex must be a spinlock because blockable 987*becd9849SNeel Natu * mutexes cannot be acquired in a critical section. 988*becd9849SNeel Natu */ 989*becd9849SNeel Natu mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN); 990fb03ca4eSNeel Natu callout_init(&vlapic->callout, 1); 991fb03ca4eSNeel Natu 992a2da7af6SNeel Natu vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED; 9932d3a73edSNeel Natu 9942d3a73edSNeel Natu if (vcpuid == 0) 9952d3a73edSNeel Natu vlapic->msr_apicbase |= APICBASE_BSP; 9962d3a73edSNeel Natu 99703cd0501SNeel Natu vlapic_reset(vlapic); 998366f6083SPeter Grehan 999366f6083SPeter Grehan return (vlapic); 1000366f6083SPeter Grehan } 1001366f6083SPeter Grehan 1002366f6083SPeter Grehan void 1003366f6083SPeter Grehan vlapic_cleanup(struct vlapic *vlapic) 1004366f6083SPeter Grehan { 100503cd0501SNeel Natu 1006fb03ca4eSNeel Natu callout_drain(&vlapic->callout); 1007366f6083SPeter Grehan free(vlapic, M_VLAPIC); 1008366f6083SPeter Grehan } 10092d3a73edSNeel Natu 10102d3a73edSNeel Natu uint64_t 10112d3a73edSNeel Natu vlapic_get_apicbase(struct vlapic *vlapic) 10122d3a73edSNeel Natu { 10132d3a73edSNeel Natu 10142d3a73edSNeel Natu return (vlapic->msr_apicbase); 10152d3a73edSNeel Natu } 10162d3a73edSNeel Natu 10172d3a73edSNeel Natu void 10182d3a73edSNeel Natu vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val) 10192d3a73edSNeel Natu { 1020a2da7af6SNeel Natu int err; 1021a2da7af6SNeel Natu enum x2apic_state state; 1022a2da7af6SNeel Natu 1023a2da7af6SNeel Natu err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state); 1024a2da7af6SNeel Natu if (err) 1025a2da7af6SNeel Natu panic("vlapic_set_apicbase: err %d fetching x2apic state", err); 1026a2da7af6SNeel Natu 1027a2da7af6SNeel Natu if (state == X2APIC_DISABLED) 1028a2da7af6SNeel Natu val &= ~APICBASE_X2APIC; 10292d3a73edSNeel Natu 10302d3a73edSNeel Natu vlapic->msr_apicbase = val; 10312d3a73edSNeel Natu } 103273820fb0SNeel Natu 103373820fb0SNeel Natu void 103473820fb0SNeel Natu vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 103573820fb0SNeel Natu { 103673820fb0SNeel Natu struct vlapic *vlapic; 103773820fb0SNeel Natu 103873820fb0SNeel Natu vlapic = vm_lapic(vm, vcpuid); 103973820fb0SNeel Natu 1040485f986aSNeel Natu if (state == X2APIC_DISABLED) 104173820fb0SNeel Natu vlapic->msr_apicbase &= ~APICBASE_X2APIC; 104273820fb0SNeel Natu } 10431c052192SNeel Natu 10441c052192SNeel Natu bool 10451c052192SNeel Natu vlapic_enabled(struct vlapic *vlapic) 10461c052192SNeel Natu { 10471c052192SNeel Natu struct LAPIC *lapic = &vlapic->apic; 10481c052192SNeel Natu 10491c052192SNeel Natu if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 && 10501c052192SNeel Natu (lapic->svr & APIC_SVR_ENABLE) != 0) 10511c052192SNeel Natu return (true); 10521c052192SNeel Natu else 10531c052192SNeel Natu return (false); 10541c052192SNeel Natu } 1055