1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1996, by Steve Passe
5 * All rights reserved.
6 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. The name of the developer may NOT be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 * 3. Neither the name of the author nor the names of any co-contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 /*
33 * Local APIC support on Pentium and later processors.
34 */
35
36 #include <sys/cdefs.h>
37 #include "opt_atpic.h"
38
39 #include "opt_ddb.h"
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/asan.h>
44 #include <sys/bus.h>
45 #include <sys/kernel.h>
46 #include <sys/lock.h>
47 #include <sys/malloc.h>
48 #include <sys/msan.h>
49 #include <sys/mutex.h>
50 #include <sys/pcpu.h>
51 #include <sys/proc.h>
52 #include <sys/refcount.h>
53 #include <sys/sched.h>
54 #include <sys/smp.h>
55 #include <sys/sysctl.h>
56 #include <sys/timeet.h>
57 #include <sys/timetc.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61
62 #include <x86/apicreg.h>
63 #include <machine/clock.h>
64 #include <machine/cpufunc.h>
65 #include <machine/cputypes.h>
66 #include <machine/fpu.h>
67 #include <machine/frame.h>
68 #include <machine/intr_machdep.h>
69 #include <x86/apicvar.h>
70 #include <x86/mca.h>
71 #include <machine/md_var.h>
72 #include <machine/smp.h>
73 #include <machine/specialreg.h>
74 #include <x86/init.h>
75 #include <x86/kvm.h>
76 #include <contrib/xen/arch-x86/cpuid.h>
77 #include <x86/bhyve.h>
78 #include <dev/hyperv/vmbus/x86/hyperv_reg.h>
79
80 #ifdef DDB
81 #include <sys/interrupt.h>
82 #include <ddb/ddb.h>
83 #endif
84
85 #ifdef __amd64__
86 #define SDT_APIC SDT_SYSIGT
87 #define GSEL_APIC 0
88 #else
89 #define SDT_APIC SDT_SYS386IGT
90 #define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL)
91 #endif
92
93 static MALLOC_DEFINE(M_LAPIC, "local_apic", "Local APIC items");
94
95 /* Sanity checks on IDT vectors. */
96 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
97 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
98 CTASSERT(APIC_LOCAL_INTS == 240);
99 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);
100
101 /*
102 * I/O interrupts use non-negative IRQ values. These values are used
103 * to mark unused IDT entries or IDT entries reserved for a non-I/O
104 * interrupt.
105 */
106 #define IRQ_FREE -1
107 #define IRQ_TIMER -2
108 #define IRQ_SYSCALL -3
109 #define IRQ_DTRACE_RET -4
110 #define IRQ_EVTCHN -5
111
112 enum lat_timer_mode {
113 LAT_MODE_UNDEF = 0,
114 LAT_MODE_PERIODIC = 1,
115 LAT_MODE_ONESHOT = 2,
116 LAT_MODE_DEADLINE = 3,
117 };
118
119 /*
120 * Support for local APICs. Local APICs manage interrupts on each
121 * individual processor as opposed to I/O APICs which receive interrupts
122 * from I/O devices and then forward them on to the local APICs.
123 *
124 * Local APICs can also send interrupts to each other thus providing the
125 * mechanism for IPIs.
126 */
127
128 struct lvt {
129 u_int lvt_edgetrigger:1;
130 u_int lvt_activehi:1;
131 u_int lvt_masked:1;
132 u_int lvt_active:1;
133 u_int lvt_mode:16;
134 u_int lvt_vector:8;
135 u_int lvt_reg;
136 const char *lvt_desc;
137 };
138
139 struct lapic {
140 struct lvt la_lvts[APIC_LVT_MAX + 1];
141 struct lvt la_elvts[APIC_ELVT_MAX + 1];
142 u_int la_id:8;
143 u_int la_cluster:4;
144 u_int la_cluster_id:2;
145 u_int la_present:1;
146 u_long *la_timer_count;
147 uint64_t la_timer_period;
148 enum lat_timer_mode la_timer_mode;
149 uint32_t lvt_timer_base;
150 uint32_t lvt_timer_last;
151 /* Include IDT_SYSCALL to make indexing easier. */
152 int la_ioint_irqs[APIC_NUM_IOINTS + 1];
153 } static *lapics;
154
155 /* Global defaults for local APIC LVT entries. */
156 static struct lvt lvts[] = {
157 /* LINT0: masked ExtINT */
158 [APIC_LVT_LINT0] = {
159 .lvt_edgetrigger = 1,
160 .lvt_activehi = 1,
161 .lvt_masked = 1,
162 .lvt_active = 1,
163 .lvt_mode = APIC_LVT_DM_EXTINT,
164 .lvt_vector = 0,
165 .lvt_reg = LAPIC_LVT_LINT0,
166 .lvt_desc = "LINT0",
167 },
168 /* LINT1: NMI */
169 [APIC_LVT_LINT1] = {
170 .lvt_edgetrigger = 1,
171 .lvt_activehi = 1,
172 .lvt_masked = 0,
173 .lvt_active = 1,
174 .lvt_mode = APIC_LVT_DM_NMI,
175 .lvt_vector = 0,
176 .lvt_reg = LAPIC_LVT_LINT1,
177 .lvt_desc = "LINT1",
178 },
179 [APIC_LVT_TIMER] = {
180 .lvt_edgetrigger = 1,
181 .lvt_activehi = 1,
182 .lvt_masked = 1,
183 .lvt_active = 1,
184 .lvt_mode = APIC_LVT_DM_FIXED,
185 .lvt_vector = APIC_TIMER_INT,
186 .lvt_reg = LAPIC_LVT_TIMER,
187 .lvt_desc = "TIMER",
188 },
189 [APIC_LVT_ERROR] = {
190 .lvt_edgetrigger = 1,
191 .lvt_activehi = 1,
192 .lvt_masked = 0,
193 .lvt_active = 1,
194 .lvt_mode = APIC_LVT_DM_FIXED,
195 .lvt_vector = APIC_ERROR_INT,
196 .lvt_reg = LAPIC_LVT_ERROR,
197 .lvt_desc = "ERROR",
198 },
199 [APIC_LVT_PMC] = {
200 .lvt_edgetrigger = 1,
201 .lvt_activehi = 1,
202 .lvt_masked = 1,
203 .lvt_active = 1,
204 .lvt_mode = APIC_LVT_DM_NMI,
205 .lvt_vector = 0,
206 .lvt_reg = LAPIC_LVT_PCINT,
207 .lvt_desc = "PMC",
208 },
209 [APIC_LVT_THERMAL] = {
210 .lvt_edgetrigger = 1,
211 .lvt_activehi = 1,
212 .lvt_masked = 1,
213 .lvt_active = 1,
214 .lvt_mode = APIC_LVT_DM_FIXED,
215 .lvt_vector = APIC_THERMAL_INT,
216 .lvt_reg = LAPIC_LVT_THERMAL,
217 .lvt_desc = "THERM",
218 },
219 [APIC_LVT_CMCI] = {
220 .lvt_edgetrigger = 1,
221 .lvt_activehi = 1,
222 .lvt_masked = 1,
223 .lvt_active = 1,
224 .lvt_mode = APIC_LVT_DM_FIXED,
225 .lvt_vector = APIC_CMC_INT,
226 .lvt_reg = LAPIC_LVT_CMCI,
227 .lvt_desc = "CMCI",
228 },
229 };
230
231 /* Global defaults for AMD local APIC ELVT entries. */
232 static struct lvt elvts[] = {
233 [APIC_ELVT_IBS] = {
234 .lvt_edgetrigger = 1,
235 .lvt_activehi = 1,
236 .lvt_masked = 1,
237 .lvt_active = 1,
238 .lvt_mode = APIC_LVT_DM_NMI,
239 .lvt_vector = 0,
240 .lvt_reg = LAPIC_EXT_LVT0,
241 .lvt_desc = "IBS",
242 },
243 [APIC_ELVT_MCA] = {
244 .lvt_edgetrigger = 1,
245 .lvt_activehi = 1,
246 .lvt_masked = 1,
247 .lvt_active = 0,
248 .lvt_mode = APIC_LVT_DM_FIXED,
249 .lvt_vector = APIC_CMC_INT,
250 .lvt_reg = LAPIC_EXT_LVT1,
251 .lvt_desc = "MCA",
252 },
253 [APIC_ELVT_DEI] = {
254 .lvt_edgetrigger = 1,
255 .lvt_activehi = 1,
256 .lvt_masked = 1,
257 .lvt_active = 0,
258 .lvt_mode = APIC_LVT_DM_FIXED,
259 .lvt_vector = 0,
260 .lvt_reg = LAPIC_EXT_LVT2,
261 .lvt_desc = "ELVT2",
262 },
263 [APIC_ELVT_SBI] = {
264 .lvt_edgetrigger = 1,
265 .lvt_activehi = 1,
266 .lvt_masked = 1,
267 .lvt_active = 0,
268 .lvt_mode = APIC_LVT_DM_FIXED,
269 .lvt_vector = 0,
270 .lvt_reg = LAPIC_EXT_LVT3,
271 .lvt_desc = "ELVT3",
272 },
273 };
274
275 static inthand_t *ioint_handlers[] = {
276 NULL, /* 0 - 31 */
277 IDTVEC(apic_isr1), /* 32 - 63 */
278 IDTVEC(apic_isr2), /* 64 - 95 */
279 IDTVEC(apic_isr3), /* 96 - 127 */
280 IDTVEC(apic_isr4), /* 128 - 159 */
281 IDTVEC(apic_isr5), /* 160 - 191 */
282 IDTVEC(apic_isr6), /* 192 - 223 */
283 IDTVEC(apic_isr7), /* 224 - 255 */
284 };
285
286 static inthand_t *ioint_pti_handlers[] = {
287 NULL, /* 0 - 31 */
288 IDTVEC(apic_isr1_pti), /* 32 - 63 */
289 IDTVEC(apic_isr2_pti), /* 64 - 95 */
290 IDTVEC(apic_isr3_pti), /* 96 - 127 */
291 IDTVEC(apic_isr4_pti), /* 128 - 159 */
292 IDTVEC(apic_isr5_pti), /* 160 - 191 */
293 IDTVEC(apic_isr6_pti), /* 192 - 223 */
294 IDTVEC(apic_isr7_pti), /* 224 - 255 */
295 };
296
297 static u_int32_t lapic_timer_divisors[] = {
298 APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
299 APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
300 };
301
302 extern inthand_t IDTVEC(rsvd_pti), IDTVEC(rsvd);
303
304 volatile char *lapic_map;
305 vm_paddr_t lapic_paddr = DEFAULT_APIC_BASE;
306 int x2apic_mode;
307 int lapic_eoi_suppression;
308 static int lapic_timer_tsc_deadline;
309 static u_long lapic_timer_divisor, count_freq;
310 static struct eventtimer lapic_et;
311 #ifdef SMP
312 static uint64_t lapic_ipi_wait_mult;
313 static int __read_mostly lapic_ds_idle_timeout = 1000000;
314 #endif
315 unsigned int max_apic_id;
316 static int pcint_refcnt = 0;
317
318 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
319 "APIC options");
320 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
321 SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
322 &lapic_eoi_suppression, 0, "");
323 SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
324 &lapic_timer_tsc_deadline, 0, "");
325 #ifdef SMP
326 SYSCTL_INT(_hw_apic, OID_AUTO, ds_idle_timeout, CTLFLAG_RWTUN,
327 &lapic_ds_idle_timeout, 0,
328 "timeout (in us) for APIC Delivery Status to become Idle (xAPIC only)");
329 #endif
330
331 static void lapic_calibrate_initcount(struct lapic *la);
332
333 /*
334 * Calculate the max index of the present LVT entry from the value of
335 * the LAPIC version register.
336 */
337 static int
lapic_maxlvt(uint32_t version)338 lapic_maxlvt(uint32_t version)
339 {
340 return ((version & APIC_VER_MAXLVT) >> MAXLVTSHIFT);
341 }
342
343 /*
344 * Use __nosanitizethread to exempt the LAPIC I/O accessors from KCSan
345 * instrumentation. Otherwise, if x2APIC is not available, use of the global
346 * lapic_map will generate a KCSan false positive. While the mapping is
347 * shared among all CPUs, the physical access will always take place on the
348 * local CPU's APIC, so there isn't in fact a race here. Furthermore, the
349 * KCSan warning printf can cause a panic if issued during LAPIC access,
350 * due to attempted recursive use of event timer resources.
351 */
352
353 static uint32_t __nosanitizethread
lapic_read32(enum LAPIC_REGISTERS reg)354 lapic_read32(enum LAPIC_REGISTERS reg)
355 {
356 uint32_t res;
357
358 if (x2apic_mode) {
359 res = rdmsr32(MSR_APIC_000 + reg);
360 } else {
361 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL);
362 }
363 return (res);
364 }
365
366 static void __nosanitizethread
lapic_write32(enum LAPIC_REGISTERS reg,uint32_t val)367 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
368 {
369
370 if (x2apic_mode) {
371 mfence();
372 lfence();
373 wrmsr(MSR_APIC_000 + reg, val);
374 } else {
375 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
376 }
377 }
378
379 static void __nosanitizethread
lapic_write32_nofence(enum LAPIC_REGISTERS reg,uint32_t val)380 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
381 {
382
383 if (x2apic_mode) {
384 wrmsr(MSR_APIC_000 + reg, val);
385 } else {
386 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
387 }
388 }
389
390 #ifdef SMP
391 static uint64_t
lapic_read_icr_lo(void)392 lapic_read_icr_lo(void)
393 {
394
395 return (lapic_read32(LAPIC_ICR_LO));
396 }
397
398 static void
lapic_write_icr(uint32_t vhi,uint32_t vlo)399 lapic_write_icr(uint32_t vhi, uint32_t vlo)
400 {
401 register_t saveintr;
402 uint64_t v;
403
404 if (x2apic_mode) {
405 v = ((uint64_t)vhi << 32) | vlo;
406 mfence();
407 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
408 } else {
409 saveintr = intr_disable();
410 lapic_write32(LAPIC_ICR_HI, vhi);
411 lapic_write32(LAPIC_ICR_LO, vlo);
412 intr_restore(saveintr);
413 }
414 }
415
416 static void
lapic_write_icr_lo(uint32_t vlo)417 lapic_write_icr_lo(uint32_t vlo)
418 {
419
420 if (x2apic_mode) {
421 mfence();
422 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, vlo);
423 } else {
424 lapic_write32(LAPIC_ICR_LO, vlo);
425 }
426 }
427
428 static void
lapic_write_self_ipi(uint32_t vector)429 lapic_write_self_ipi(uint32_t vector)
430 {
431
432 KASSERT(x2apic_mode, ("SELF IPI write in xAPIC mode"));
433 wrmsr(MSR_APIC_000 + LAPIC_SELF_IPI, vector);
434 }
435 #endif /* SMP */
436
437 static void
lapic_enable_x2apic(void)438 lapic_enable_x2apic(void)
439 {
440 uint64_t apic_base;
441
442 apic_base = rdmsr(MSR_APICBASE);
443 apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
444 wrmsr(MSR_APICBASE, apic_base);
445 }
446
447 bool
lapic_is_x2apic(void)448 lapic_is_x2apic(void)
449 {
450 uint64_t apic_base;
451
452 apic_base = rdmsr(MSR_APICBASE);
453 return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) ==
454 (APICBASE_X2APIC | APICBASE_ENABLED));
455 }
456
457 static void lapic_early_mask_vecs(void);
458 static void lapic_enable(void);
459 static void lapic_resume(struct pic *pic, bool suspend_cancelled);
460 static void lapic_timer_oneshot(struct lapic *);
461 static void lapic_timer_oneshot_nointr(struct lapic *, uint32_t);
462 static void lapic_timer_periodic(struct lapic *);
463 static void lapic_timer_deadline(struct lapic *);
464 static void lapic_timer_stop(struct lapic *);
465 static void lapic_timer_set_divisor(u_int divisor);
466 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
467 static int lapic_et_start(struct eventtimer *et,
468 sbintime_t first, sbintime_t period);
469 static int lapic_et_stop(struct eventtimer *et);
470 static u_int apic_idt_to_irq(u_int apic_id, u_int vector);
471 static void lapic_set_tpr(u_int vector);
472
473 struct pic lapic_pic = { .pic_resume = lapic_resume };
474
475 static uint32_t
lvt_mode_impl(struct lapic * la,struct lvt * lvt,u_int pin,uint32_t value)476 lvt_mode_impl(struct lapic *la, struct lvt *lvt, u_int pin, uint32_t value)
477 {
478
479 value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
480 APIC_LVT_VECTOR);
481 if (lvt->lvt_edgetrigger == 0)
482 value |= APIC_LVT_TM;
483 if (lvt->lvt_activehi == 0)
484 value |= APIC_LVT_IIPP_INTALO;
485 if (lvt->lvt_masked)
486 value |= APIC_LVT_M;
487 value |= lvt->lvt_mode;
488 switch (lvt->lvt_mode) {
489 case APIC_LVT_DM_NMI:
490 case APIC_LVT_DM_SMI:
491 case APIC_LVT_DM_INIT:
492 case APIC_LVT_DM_EXTINT:
493 if (!lvt->lvt_edgetrigger) {
494 if (bootverbose) {
495 printf(
496 "lapic%u: Forcing LINT%u to edge trigger\n",
497 la->la_id, pin);
498 }
499 value &= ~APIC_LVT_TM;
500 }
501 /* Use a vector of 0. */
502 break;
503 case APIC_LVT_DM_FIXED:
504 value |= lvt->lvt_vector;
505 break;
506 default:
507 panic("bad APIC LVT delivery mode: %#x\n", value);
508 }
509 return (value);
510 }
511
512 static uint32_t
lvt_mode(struct lapic * la,u_int pin,uint32_t value)513 lvt_mode(struct lapic *la, u_int pin, uint32_t value)
514 {
515 struct lvt *lvt;
516
517 KASSERT(pin <= APIC_LVT_MAX,
518 ("%s: pin %u out of range", __func__, pin));
519 if (la->la_lvts[pin].lvt_active)
520 lvt = &la->la_lvts[pin];
521 else
522 lvt = &lvts[pin];
523
524 return (lvt_mode_impl(la, lvt, pin, value));
525 }
526
527 static uint32_t
elvt_mode(struct lapic * la,u_int idx,uint32_t value)528 elvt_mode(struct lapic *la, u_int idx, uint32_t value)
529 {
530 struct lvt *elvt;
531
532 KASSERT(idx <= APIC_ELVT_MAX,
533 ("%s: idx %u out of range", __func__, idx));
534
535 if (la->la_elvts[idx].lvt_active)
536 elvt = &la->la_elvts[idx];
537 else
538 elvt = &elvts[idx];
539 KASSERT(elvt->lvt_active, ("%s: ELVT%u is not active", __func__, idx));
540 KASSERT(elvt->lvt_edgetrigger,
541 ("%s: ELVT%u is not edge triggered", __func__, idx));
542 KASSERT(elvt->lvt_activehi,
543 ("%s: ELVT%u is not active high", __func__, idx));
544 return (lvt_mode_impl(la, elvt, idx, value));
545 }
546
547 /*
548 * Map the local APIC and setup necessary interrupt vectors.
549 */
550 void
lapic_init(vm_paddr_t addr)551 lapic_init(vm_paddr_t addr)
552 {
553 #ifdef SMP
554 uint64_t r, r1, r2, rx;
555 #endif
556 uint32_t ver;
557 int i;
558 bool arat;
559
560 TSENTER();
561
562 /*
563 * Enable x2APIC mode if possible. Map the local APIC
564 * registers page.
565 *
566 * Keep the LAPIC registers page mapped uncached for x2APIC
567 * mode too, to have direct map page attribute set to
568 * uncached. This is needed to work around CPU errata present
569 * on all Intel processors.
570 */
571 KASSERT(trunc_page(addr) == addr,
572 ("local APIC not aligned on a page boundary"));
573 lapic_paddr = addr;
574 lapic_map = pmap_mapdev(addr, PAGE_SIZE);
575 if (x2apic_mode) {
576 lapic_enable_x2apic();
577 lapic_map = NULL;
578 }
579
580 /* Setup the spurious interrupt handler. */
581 setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
582 GSEL_APIC);
583
584 /* Perform basic initialization of the BSP's local APIC. */
585 lapic_enable();
586 lapic_early_mask_vecs();
587
588 /* Set BSP's per-CPU local APIC ID. */
589 PCPU_SET(apic_id, lapic_id());
590
591 /* Local APIC timer interrupt. */
592 setidt(APIC_TIMER_INT, pti ? IDTVEC(timerint_pti) : IDTVEC(timerint),
593 SDT_APIC, SEL_KPL, GSEL_APIC);
594
595 /* Local APIC error interrupt. */
596 setidt(APIC_ERROR_INT, pti ? IDTVEC(errorint_pti) : IDTVEC(errorint),
597 SDT_APIC, SEL_KPL, GSEL_APIC);
598
599 /* XXX: Thermal interrupt */
600
601 /* Local APIC CMCI. */
602 setidt(APIC_CMC_INT, pti ? IDTVEC(cmcint_pti) : IDTVEC(cmcint),
603 SDT_APIC, SEL_KPL, GSEL_APIC);
604
605 if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
606 /* Set if APIC timer runs in C3. */
607 arat = (cpu_power_eax & CPUTPM1_ARAT);
608
609 bzero(&lapic_et, sizeof(lapic_et));
610 lapic_et.et_name = "LAPIC";
611 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
612 ET_FLAGS_PERCPU;
613 lapic_et.et_quality = 600;
614 if (!arat) {
615 lapic_et.et_flags |= ET_FLAGS_C3STOP;
616 lapic_et.et_quality = 100;
617 }
618 if ((cpu_feature & CPUID_TSC) != 0 &&
619 (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
620 tsc_is_invariant && tsc_freq != 0) {
621 lapic_timer_tsc_deadline = 1;
622 TUNABLE_INT_FETCH("hw.apic.timer_tsc_deadline",
623 &lapic_timer_tsc_deadline);
624 }
625
626 lapic_et.et_frequency = 0;
627 /* We don't know frequency yet, so trying to guess. */
628 lapic_et.et_min_period = 0x00001000LL;
629 lapic_et.et_max_period = SBT_1S;
630 lapic_et.et_start = lapic_et_start;
631 lapic_et.et_stop = lapic_et_stop;
632 lapic_et.et_priv = NULL;
633 et_register(&lapic_et);
634 }
635
636 /*
637 * Set lapic_eoi_suppression after lapic_enable(), to not
638 * enable suppression in the hardware prematurely. Note that
639 * we by default enable suppression even when system only has
640 * one IO-APIC, since EOI is broadcasted to all APIC agents,
641 * including CPUs, otherwise.
642 *
643 * It seems that at least some KVM versions report
644 * EOI_SUPPRESSION bit, but auto-EOI does not work.
645 */
646 ver = lapic_read32(LAPIC_VERSION);
647 if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
648 lapic_eoi_suppression = 1;
649 if (vm_guest == VM_GUEST_KVM) {
650 if (bootverbose)
651 printf(
652 "KVM -- disabling lapic eoi suppression\n");
653 lapic_eoi_suppression = 0;
654 }
655 TUNABLE_INT_FETCH("hw.apic.eoi_suppression",
656 &lapic_eoi_suppression);
657 }
658
659 #ifdef SMP
660 #define LOOPS 1000
661 /*
662 * Calibrate the busy loop waiting for IPI ack in xAPIC mode.
663 * lapic_ipi_wait_mult contains the number of iterations which
664 * approximately delay execution for 1 microsecond (the
665 * argument to lapic_ipi_wait() is in microseconds).
666 *
667 * We assume that TSC is present and already measured.
668 * Possible TSC frequency jumps are irrelevant to the
669 * calibration loop below, the CPU clock management code is
670 * not yet started, and we do not enter sleep states.
671 */
672 KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0,
673 ("TSC not initialized"));
674 if (!x2apic_mode) {
675 r = rdtsc();
676 for (rx = 0; rx < LOOPS; rx++) {
677 (void)lapic_read_icr_lo();
678 ia32_pause();
679 }
680 r = rdtsc() - r;
681 r1 = tsc_freq * LOOPS;
682 r2 = r * 1000000;
683 lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1;
684 if (bootverbose) {
685 printf("LAPIC: ipi_wait() us multiplier %ju (r %ju "
686 "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult,
687 (uintmax_t)r, (uintmax_t)tsc_freq);
688 }
689 }
690 #undef LOOPS
691 #endif /* SMP */
692
693 TSEXIT();
694 }
695
696 /*
697 * Create a local APIC instance.
698 */
699 void
lapic_create(u_int apic_id,int boot_cpu)700 lapic_create(u_int apic_id, int boot_cpu)
701 {
702 int i;
703
704 if (apic_id > max_apic_id) {
705 printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
706 if (boot_cpu)
707 panic("Can't ignore BSP");
708 return;
709 }
710 KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
711 apic_id));
712
713 /*
714 * Assume no local LVT overrides and a cluster of 0 and
715 * intra-cluster ID of 0.
716 */
717 lapics[apic_id].la_present = 1;
718 lapics[apic_id].la_id = apic_id;
719 for (i = 0; i <= APIC_LVT_MAX; i++) {
720 lapics[apic_id].la_lvts[i] = lvts[i];
721 lapics[apic_id].la_lvts[i].lvt_active = 0;
722 }
723 for (i = 0; i <= APIC_ELVT_MAX; i++) {
724 lapics[apic_id].la_elvts[i] = elvts[i];
725 lapics[apic_id].la_elvts[i].lvt_active = 0;
726 }
727 for (i = 0; i <= APIC_NUM_IOINTS; i++)
728 lapics[apic_id].la_ioint_irqs[i] = IRQ_FREE;
729 lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
730 lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
731 IRQ_TIMER;
732 #ifdef KDTRACE_HOOKS
733 lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] =
734 IRQ_DTRACE_RET;
735 #endif
736 #ifdef XENHVM
737 lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN;
738 #endif
739
740 #ifdef SMP
741 cpu_add(apic_id, boot_cpu);
742 #endif
743 }
744
745 static inline uint32_t
amd_read_ext_features(void)746 amd_read_ext_features(void)
747 {
748 uint32_t version;
749
750 if (cpu_vendor_id != CPU_VENDOR_AMD &&
751 cpu_vendor_id != CPU_VENDOR_HYGON)
752 return (0);
753 version = lapic_read32(LAPIC_VERSION);
754 if ((version & APIC_VER_AMD_EXT_SPACE) != 0)
755 return (lapic_read32(LAPIC_EXT_FEATURES));
756 else
757 return (0);
758 }
759
760 static inline uint32_t
amd_read_elvt_count(void)761 amd_read_elvt_count(void)
762 {
763 uint32_t extf;
764 uint32_t count;
765
766 extf = amd_read_ext_features();
767 count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT;
768 count = min(count, APIC_ELVT_MAX + 1);
769 return (count);
770 }
771
772 /*
773 * Dump contents of local APIC registers
774 */
775 void
lapic_dump(const char * str)776 lapic_dump(const char* str)
777 {
778 uint32_t version;
779 uint32_t maxlvt;
780 uint32_t extf;
781 int elvt_count;
782 int i;
783
784 version = lapic_read32(LAPIC_VERSION);
785 maxlvt = lapic_maxlvt(version);
786 printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
787 printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x",
788 lapic_read32(LAPIC_ID), version,
789 lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR));
790 if ((cpu_feature2 & CPUID2_X2APIC) != 0)
791 printf(" x2APIC: %d", x2apic_mode);
792 printf("\n lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
793 lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1),
794 lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR));
795 printf(" timer: 0x%08x therm: 0x%08x err: 0x%08x",
796 lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL),
797 lapic_read32(LAPIC_LVT_ERROR));
798 if (maxlvt >= APIC_LVT_PMC)
799 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT));
800 printf("\n");
801 if (maxlvt >= APIC_LVT_CMCI)
802 printf(" cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI));
803 extf = amd_read_ext_features();
804 if (extf != 0) {
805 printf(" AMD ext features: 0x%08x", extf);
806 elvt_count = amd_read_elvt_count();
807 for (i = 0; i < elvt_count; i++)
808 printf("%s elvt%d: 0x%08x", (i % 4) ? "" : "\n ", i,
809 lapic_read32(LAPIC_EXT_LVT0 + i));
810 printf("\n");
811 }
812 }
813
814 void
lapic_xapic_mode(void)815 lapic_xapic_mode(void)
816 {
817 register_t saveintr;
818
819 saveintr = intr_disable();
820 if (x2apic_mode)
821 lapic_enable_x2apic();
822 intr_restore(saveintr);
823 }
824
825 static void
lapic_early_mask_vec(const struct lvt * l)826 lapic_early_mask_vec(const struct lvt *l)
827 {
828 uint32_t v;
829
830 if (l->lvt_masked != 0) {
831 v = lapic_read32(l->lvt_reg);
832 v |= APIC_LVT_M;
833 lapic_write32(l->lvt_reg, v);
834 }
835 }
836
837 /* Done on BSP only */
838 static void
lapic_early_mask_vecs(void)839 lapic_early_mask_vecs(void)
840 {
841 int elvt_count, lvts_count, i;
842 uint32_t version;
843
844 version = lapic_read32(LAPIC_VERSION);
845 lvts_count = min(nitems(lvts), lapic_maxlvt(version) + 1);
846 for (i = 0; i < lvts_count; i++)
847 lapic_early_mask_vec(&lvts[i]);
848
849 elvt_count = amd_read_elvt_count();
850 for (i = 0; i < elvt_count; i++)
851 lapic_early_mask_vec(&elvts[i]);
852 }
853
854 void
lapic_setup(int boot)855 lapic_setup(int boot)
856 {
857 struct lapic *la;
858 uint32_t version;
859 uint32_t maxlvt;
860 register_t saveintr;
861 int elvt_count;
862 int i;
863
864 saveintr = intr_disable();
865
866 la = &lapics[lapic_id()];
867 KASSERT(la->la_present, ("missing APIC structure"));
868 version = lapic_read32(LAPIC_VERSION);
869 maxlvt = lapic_maxlvt(version);
870
871 /* Initialize the TPR to allow all interrupts. */
872 lapic_set_tpr(0);
873
874 /* Setup spurious vector and enable the local APIC. */
875 lapic_enable();
876
877 /* Program LINT[01] LVT entries. */
878 lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0,
879 lapic_read32(LAPIC_LVT_LINT0)));
880 lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1,
881 lapic_read32(LAPIC_LVT_LINT1)));
882
883 /* Program the PMC LVT entry if present. */
884 if (maxlvt >= APIC_LVT_PMC) {
885 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
886 LAPIC_LVT_PCINT));
887 }
888
889 /*
890 * Program the timer LVT. Calibration is deferred until it is certain
891 * that we have a reliable timecounter.
892 */
893 la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
894 lapic_read32(LAPIC_LVT_TIMER));
895 la->lvt_timer_last = la->lvt_timer_base;
896 lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
897
898 if (boot)
899 la->la_timer_mode = LAT_MODE_UNDEF;
900 else if (la->la_timer_mode != LAT_MODE_UNDEF) {
901 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
902 lapic_id()));
903 switch (la->la_timer_mode) {
904 case LAT_MODE_PERIODIC:
905 lapic_timer_set_divisor(lapic_timer_divisor);
906 lapic_timer_periodic(la);
907 break;
908 case LAT_MODE_ONESHOT:
909 lapic_timer_set_divisor(lapic_timer_divisor);
910 lapic_timer_oneshot(la);
911 break;
912 case LAT_MODE_DEADLINE:
913 lapic_timer_deadline(la);
914 break;
915 default:
916 panic("corrupted la_timer_mode %p %d", la,
917 la->la_timer_mode);
918 }
919 }
920
921 /* Program error LVT and clear any existing errors. */
922 lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR,
923 lapic_read32(LAPIC_LVT_ERROR)));
924 lapic_write32(LAPIC_ESR, 0);
925
926 /* XXX: Thermal LVT */
927
928 /* Program the CMCI LVT entry if present. */
929 if (maxlvt >= APIC_LVT_CMCI) {
930 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI,
931 lapic_read32(LAPIC_LVT_CMCI)));
932 }
933
934 elvt_count = amd_read_elvt_count();
935 for (i = 0; i < elvt_count; i++) {
936 if (la->la_elvts[i].lvt_active)
937 lapic_write32(LAPIC_EXT_LVT0 + i,
938 elvt_mode(la, i, lapic_read32(LAPIC_EXT_LVT0 + i)));
939 }
940
941 intr_restore(saveintr);
942 }
943
944 static void
lapic_intrcnt(void * dummy __unused)945 lapic_intrcnt(void *dummy __unused)
946 {
947 struct pcpu *pc;
948 struct lapic *la;
949 char buf[MAXCOMLEN + 1];
950
951 /* If there are no APICs, skip this function. */
952 if (lapics == NULL)
953 return;
954
955 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
956 la = &lapics[pc->pc_apic_id];
957 if (!la->la_present)
958 continue;
959
960 snprintf(buf, sizeof(buf), "cpu%d:timer", pc->pc_cpuid);
961 intrcnt_add(buf, &la->la_timer_count);
962 }
963 }
964 SYSINIT(lapic_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, lapic_intrcnt, NULL);
965
966 void
lapic_reenable_pcint(void)967 lapic_reenable_pcint(void)
968 {
969 uint32_t value;
970
971 if (refcount_load(&pcint_refcnt) == 0)
972 return;
973
974 value = lapic_read32(LAPIC_LVT_PCINT);
975 value &= ~APIC_LVT_M;
976 lapic_write32(LAPIC_LVT_PCINT, value);
977
978 if ((amd_feature2 & AMDID2_IBS) != 0) {
979 value = lapic_read32(LAPIC_EXT_LVT0);
980 value &= ~APIC_LVT_M;
981 lapic_write32(LAPIC_EXT_LVT0, value);
982 }
983 }
984
985 static void
lapic_update_pcint(void * dummy)986 lapic_update_pcint(void *dummy)
987 {
988 struct lapic *la;
989
990 la = &lapics[lapic_id()];
991 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
992 lapic_read32(LAPIC_LVT_PCINT)));
993
994 if ((amd_feature2 & AMDID2_IBS) != 0) {
995 lapic_write32(LAPIC_EXT_LVT0, elvt_mode(la, APIC_ELVT_IBS,
996 lapic_read32(LAPIC_EXT_LVT0)));
997 }
998 }
999
1000 void
lapic_calibrate_timer(void)1001 lapic_calibrate_timer(void)
1002 {
1003 struct lapic *la;
1004 register_t intr;
1005
1006 #ifdef DEV_ATPIC
1007 /* Fail if the local APIC is not present. */
1008 if (!x2apic_mode && lapic_map == NULL)
1009 return;
1010 #endif
1011
1012 intr = intr_disable();
1013 la = &lapics[lapic_id()];
1014
1015 lapic_calibrate_initcount(la);
1016
1017 intr_restore(intr);
1018
1019 if (lapic_timer_tsc_deadline && bootverbose) {
1020 printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
1021 (uintmax_t)tsc_freq);
1022 }
1023 }
1024
1025 int
lapic_enable_pcint(void)1026 lapic_enable_pcint(void)
1027 {
1028 u_int32_t maxlvt;
1029
1030 #ifdef DEV_ATPIC
1031 /* Fail if the local APIC is not present. */
1032 if (!x2apic_mode && lapic_map == NULL)
1033 return (0);
1034 #endif
1035
1036 /* Fail if the PMC LVT is not present. */
1037 maxlvt = lapic_maxlvt(lapic_read32(LAPIC_VERSION));
1038 if (maxlvt < APIC_LVT_PMC)
1039 return (0);
1040 if (refcount_acquire(&pcint_refcnt) > 0)
1041 return (1);
1042 lvts[APIC_LVT_PMC].lvt_masked = 0;
1043
1044 if ((amd_feature2 & AMDID2_IBS) != 0)
1045 elvts[APIC_ELVT_IBS].lvt_masked = 0;
1046
1047 MPASS(mp_ncpus == 1 || smp_started);
1048 smp_rendezvous(NULL, lapic_update_pcint, NULL, NULL);
1049 return (1);
1050 }
1051
1052 void
lapic_disable_pcint(void)1053 lapic_disable_pcint(void)
1054 {
1055 u_int32_t maxlvt;
1056
1057 #ifdef DEV_ATPIC
1058 /* Fail if the local APIC is not present. */
1059 if (!x2apic_mode && lapic_map == NULL)
1060 return;
1061 #endif
1062
1063 /* Fail if the PMC LVT is not present. */
1064 maxlvt = lapic_maxlvt(lapic_read32(LAPIC_VERSION));
1065 if (maxlvt < APIC_LVT_PMC)
1066 return;
1067 if (!refcount_release(&pcint_refcnt))
1068 return;
1069 lvts[APIC_LVT_PMC].lvt_masked = 1;
1070 elvts[APIC_ELVT_IBS].lvt_masked = 1;
1071
1072 #ifdef SMP
1073 /* The APs should always be started when hwpmc is unloaded. */
1074 KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early"));
1075 #endif
1076 smp_rendezvous(NULL, lapic_update_pcint, NULL, NULL);
1077 }
1078
1079 static int
lapic_calibrate_initcount_cpuid_vm(void)1080 lapic_calibrate_initcount_cpuid_vm(void)
1081 {
1082 u_int regs[4];
1083 uint64_t freq;
1084
1085 /* Get value from CPUID leaf if possible. */
1086 if (vm_guest == VM_GUEST_NO)
1087 return (false);
1088 if (hv_high < 0x40000010)
1089 return (false);
1090 do_cpuid(0x40000010, regs);
1091 freq = (uint64_t)(regs[1]) * 1000;
1092
1093 /* Pick timer divisor. */
1094 lapic_timer_divisor = 2;
1095 do {
1096 if (freq / lapic_timer_divisor < APIC_TIMER_MAX_COUNT)
1097 break;
1098 lapic_timer_divisor <<= 1;
1099 } while (lapic_timer_divisor <= 128);
1100 if (lapic_timer_divisor > 128)
1101 return (false);
1102
1103 /* Record divided frequency. */
1104 count_freq = freq / lapic_timer_divisor;
1105 return (count_freq != 0);
1106 }
1107
1108 static uint64_t
cb_lapic_getcount(void)1109 cb_lapic_getcount(void)
1110 {
1111
1112 return (APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER));
1113 }
1114
1115 static void
lapic_calibrate_initcount(struct lapic * la)1116 lapic_calibrate_initcount(struct lapic *la)
1117 {
1118 uint64_t freq;
1119
1120 if (lapic_calibrate_initcount_cpuid_vm())
1121 goto done;
1122
1123 /* Calibrate the APIC timer frequency. */
1124 lapic_timer_set_divisor(2);
1125 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT);
1126 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
1127 freq = clockcalib(cb_lapic_getcount, "lapic");
1128 fpu_kern_leave(curthread, NULL);
1129
1130 /* Pick a different divisor if necessary. */
1131 lapic_timer_divisor = 2;
1132 do {
1133 if (freq * 2 / lapic_timer_divisor < APIC_TIMER_MAX_COUNT)
1134 break;
1135 lapic_timer_divisor <<= 1;
1136 } while (lapic_timer_divisor <= 128);
1137 if (lapic_timer_divisor > 128)
1138 panic("lapic: Divisor too big");
1139 count_freq = freq * 2 / lapic_timer_divisor;
1140 done:
1141 if (bootverbose) {
1142 printf("lapic: Divisor %lu, Frequency %lu Hz\n",
1143 lapic_timer_divisor, count_freq);
1144 }
1145 }
1146
1147 static void
lapic_change_mode(struct eventtimer * et,struct lapic * la,enum lat_timer_mode newmode)1148 lapic_change_mode(struct eventtimer *et, struct lapic *la,
1149 enum lat_timer_mode newmode)
1150 {
1151 if (la->la_timer_mode == newmode)
1152 return;
1153 switch (newmode) {
1154 case LAT_MODE_PERIODIC:
1155 lapic_timer_set_divisor(lapic_timer_divisor);
1156 et->et_frequency = count_freq;
1157 break;
1158 case LAT_MODE_DEADLINE:
1159 et->et_frequency = tsc_freq;
1160 break;
1161 case LAT_MODE_ONESHOT:
1162 lapic_timer_set_divisor(lapic_timer_divisor);
1163 et->et_frequency = count_freq;
1164 break;
1165 default:
1166 panic("lapic_change_mode %d", newmode);
1167 }
1168 la->la_timer_mode = newmode;
1169 et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
1170 et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
1171 }
1172
1173 static int
lapic_et_start(struct eventtimer * et,sbintime_t first,sbintime_t period)1174 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
1175 {
1176 struct lapic *la;
1177
1178 la = &lapics[PCPU_GET(apic_id)];
1179 if (period != 0) {
1180 lapic_change_mode(et, la, LAT_MODE_PERIODIC);
1181 la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
1182 32;
1183 lapic_timer_periodic(la);
1184 } else if (lapic_timer_tsc_deadline) {
1185 lapic_change_mode(et, la, LAT_MODE_DEADLINE);
1186 la->la_timer_period = (et->et_frequency * first) >> 32;
1187 lapic_timer_deadline(la);
1188 } else {
1189 lapic_change_mode(et, la, LAT_MODE_ONESHOT);
1190 la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
1191 32;
1192 lapic_timer_oneshot(la);
1193 }
1194 return (0);
1195 }
1196
1197 static int
lapic_et_stop(struct eventtimer * et)1198 lapic_et_stop(struct eventtimer *et)
1199 {
1200 struct lapic *la;
1201
1202 la = &lapics[PCPU_GET(apic_id)];
1203 lapic_timer_stop(la);
1204 la->la_timer_mode = LAT_MODE_UNDEF;
1205 return (0);
1206 }
1207
1208 void
lapic_disable(void)1209 lapic_disable(void)
1210 {
1211 uint32_t value;
1212
1213 /* Software disable the local APIC. */
1214 value = lapic_read32(LAPIC_SVR);
1215 value &= ~APIC_SVR_SWEN;
1216 lapic_write32(LAPIC_SVR, value);
1217 }
1218
1219 static void
lapic_enable(void)1220 lapic_enable(void)
1221 {
1222 uint32_t value;
1223
1224 /* Program the spurious vector to enable the local APIC. */
1225 value = lapic_read32(LAPIC_SVR);
1226 value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
1227 value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
1228 if (lapic_eoi_suppression)
1229 value |= APIC_SVR_EOI_SUPPRESSION;
1230 lapic_write32(LAPIC_SVR, value);
1231 }
1232
1233 /* Reset the local APIC on the BSP during resume. */
1234 static void
lapic_resume(struct pic * pic,bool suspend_cancelled)1235 lapic_resume(struct pic *pic, bool suspend_cancelled)
1236 {
1237
1238 lapic_setup(0);
1239 }
1240
1241 int
lapic_id(void)1242 lapic_id(void)
1243 {
1244 uint32_t v;
1245
1246 KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped"));
1247 v = lapic_read32(LAPIC_ID);
1248 if (!x2apic_mode)
1249 v >>= APIC_ID_SHIFT;
1250 return (v);
1251 }
1252
1253 int
lapic_intr_pending(u_int vector)1254 lapic_intr_pending(u_int vector)
1255 {
1256 uint32_t irr;
1257
1258 /*
1259 * The IRR registers are an array of registers each of which
1260 * only describes 32 interrupts in the low 32 bits. Thus, we
1261 * divide the vector by 32 to get the register index.
1262 * Finally, we modulus the vector by 32 to determine the
1263 * individual bit to test.
1264 */
1265 irr = lapic_read32(LAPIC_IRR0 + vector / 32);
1266 return (irr & 1 << (vector % 32));
1267 }
1268
1269 void
lapic_set_logical_id(u_int apic_id,u_int cluster,u_int cluster_id)1270 lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
1271 {
1272 struct lapic *la;
1273
1274 KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
1275 __func__, apic_id));
1276 KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
1277 __func__, cluster));
1278 KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
1279 ("%s: intra cluster id %u too big", __func__, cluster_id));
1280 la = &lapics[apic_id];
1281 la->la_cluster = cluster;
1282 la->la_cluster_id = cluster_id;
1283 }
1284
1285 int
lapic_set_lvt_mask(u_int apic_id,u_int pin,u_char masked)1286 lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
1287 {
1288
1289 if (pin > APIC_LVT_MAX)
1290 return (EINVAL);
1291 if (apic_id == APIC_ID_ALL) {
1292 lvts[pin].lvt_masked = masked;
1293 if (bootverbose)
1294 printf("lapic:");
1295 } else {
1296 KASSERT(lapics[apic_id].la_present,
1297 ("%s: missing APIC %u", __func__, apic_id));
1298 lapics[apic_id].la_lvts[pin].lvt_masked = masked;
1299 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1300 if (bootverbose)
1301 printf("lapic%u:", apic_id);
1302 }
1303 if (bootverbose)
1304 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
1305 return (0);
1306 }
1307
1308 int
lapic_set_lvt_mode(u_int apic_id,u_int pin,u_int32_t mode)1309 lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
1310 {
1311 struct lvt *lvt;
1312
1313 if (pin > APIC_LVT_MAX)
1314 return (EINVAL);
1315 if (apic_id == APIC_ID_ALL) {
1316 lvt = &lvts[pin];
1317 if (bootverbose)
1318 printf("lapic:");
1319 } else {
1320 KASSERT(lapics[apic_id].la_present,
1321 ("%s: missing APIC %u", __func__, apic_id));
1322 lvt = &lapics[apic_id].la_lvts[pin];
1323 lvt->lvt_active = 1;
1324 if (bootverbose)
1325 printf("lapic%u:", apic_id);
1326 }
1327 lvt->lvt_mode = mode;
1328 switch (mode) {
1329 case APIC_LVT_DM_NMI:
1330 case APIC_LVT_DM_SMI:
1331 case APIC_LVT_DM_INIT:
1332 case APIC_LVT_DM_EXTINT:
1333 lvt->lvt_edgetrigger = 1;
1334 lvt->lvt_activehi = 1;
1335 if (mode == APIC_LVT_DM_EXTINT)
1336 lvt->lvt_masked = 1;
1337 else
1338 lvt->lvt_masked = 0;
1339 break;
1340 default:
1341 panic("Unsupported delivery mode: 0x%x\n", mode);
1342 }
1343 if (bootverbose) {
1344 printf(" Routing ");
1345 switch (mode) {
1346 case APIC_LVT_DM_NMI:
1347 printf("NMI");
1348 break;
1349 case APIC_LVT_DM_SMI:
1350 printf("SMI");
1351 break;
1352 case APIC_LVT_DM_INIT:
1353 printf("INIT");
1354 break;
1355 case APIC_LVT_DM_EXTINT:
1356 printf("ExtINT");
1357 break;
1358 }
1359 printf(" -> LINT%u\n", pin);
1360 }
1361 return (0);
1362 }
1363
1364 int
lapic_set_lvt_polarity(u_int apic_id,u_int pin,enum intr_polarity pol)1365 lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol)
1366 {
1367
1368 if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM)
1369 return (EINVAL);
1370 if (apic_id == APIC_ID_ALL) {
1371 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH);
1372 if (bootverbose)
1373 printf("lapic:");
1374 } else {
1375 KASSERT(lapics[apic_id].la_present,
1376 ("%s: missing APIC %u", __func__, apic_id));
1377 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1378 lapics[apic_id].la_lvts[pin].lvt_activehi =
1379 (pol == INTR_POLARITY_HIGH);
1380 if (bootverbose)
1381 printf("lapic%u:", apic_id);
1382 }
1383 if (bootverbose)
1384 printf(" LINT%u polarity: %s\n", pin,
1385 pol == INTR_POLARITY_HIGH ? "high" : "low");
1386 return (0);
1387 }
1388
1389 int
lapic_set_lvt_triggermode(u_int apic_id,u_int pin,enum intr_trigger trigger)1390 lapic_set_lvt_triggermode(u_int apic_id, u_int pin,
1391 enum intr_trigger trigger)
1392 {
1393
1394 if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM)
1395 return (EINVAL);
1396 if (apic_id == APIC_ID_ALL) {
1397 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE);
1398 if (bootverbose)
1399 printf("lapic:");
1400 } else {
1401 KASSERT(lapics[apic_id].la_present,
1402 ("%s: missing APIC %u", __func__, apic_id));
1403 lapics[apic_id].la_lvts[pin].lvt_edgetrigger =
1404 (trigger == INTR_TRIGGER_EDGE);
1405 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1406 if (bootverbose)
1407 printf("lapic%u:", apic_id);
1408 }
1409 if (bootverbose)
1410 printf(" LINT%u trigger: %s\n", pin,
1411 trigger == INTR_TRIGGER_EDGE ? "edge" : "level");
1412 return (0);
1413 }
1414
1415 /*
1416 * Adjust the TPR of the current CPU so that it blocks all interrupts below
1417 * the passed in vector.
1418 */
1419 static void
lapic_set_tpr(u_int vector)1420 lapic_set_tpr(u_int vector)
1421 {
1422 #ifdef CHEAP_TPR
1423 lapic_write32(LAPIC_TPR, vector);
1424 #else
1425 uint32_t tpr;
1426
1427 tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO;
1428 tpr |= vector;
1429 lapic_write32(LAPIC_TPR, tpr);
1430 #endif
1431 }
1432
1433 void
lapic_eoi(void)1434 lapic_eoi(void)
1435 {
1436
1437 lapic_write32_nofence(LAPIC_EOI, 0);
1438 }
1439
1440 void
lapic_handle_intr(int vector,struct trapframe * frame)1441 lapic_handle_intr(int vector, struct trapframe *frame)
1442 {
1443 struct intsrc *isrc;
1444
1445 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
1446 kmsan_mark(&vector, sizeof(vector), KMSAN_STATE_INITED);
1447 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
1448 trap_check_kstack();
1449
1450 isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
1451 vector));
1452 KASSERT(isrc != NULL,
1453 ("lapic_handle_intr: vector %d unrecognized at lapic %u",
1454 vector, PCPU_GET(apic_id)));
1455 intr_execute_handlers(isrc, frame);
1456 }
1457
1458 void
lapic_handle_timer(struct trapframe * frame)1459 lapic_handle_timer(struct trapframe *frame)
1460 {
1461 struct lapic *la;
1462 struct trapframe *oldframe;
1463 struct thread *td;
1464
1465 /* Send EOI first thing. */
1466 lapic_eoi();
1467
1468 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
1469 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
1470 trap_check_kstack();
1471
1472 if (!sched_do_timer_accounting())
1473 return;
1474
1475 /* Look up our local APIC structure for the tick counters. */
1476 la = &lapics[PCPU_GET(apic_id)];
1477 (*la->la_timer_count)++;
1478 critical_enter();
1479 if (lapic_et.et_active) {
1480 td = curthread;
1481 td->td_intr_nesting_level++;
1482 oldframe = td->td_intr_frame;
1483 td->td_intr_frame = frame;
1484 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg);
1485 td->td_intr_frame = oldframe;
1486 td->td_intr_nesting_level--;
1487 }
1488 critical_exit();
1489 }
1490
1491 static void
lapic_timer_set_divisor(u_int divisor)1492 lapic_timer_set_divisor(u_int divisor)
1493 {
1494
1495 KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor));
1496 KASSERT(ffs(divisor) <= nitems(lapic_timer_divisors),
1497 ("lapic: invalid divisor %u", divisor));
1498 lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]);
1499 }
1500
1501 static void
lapic_timer_oneshot(struct lapic * la)1502 lapic_timer_oneshot(struct lapic *la)
1503 {
1504 uint32_t value;
1505
1506 value = la->lvt_timer_base;
1507 value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1508 value |= APIC_LVTT_TM_ONE_SHOT;
1509 la->lvt_timer_last = value;
1510 lapic_write32(LAPIC_LVT_TIMER, value);
1511 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1512 }
1513
1514 static void
lapic_timer_oneshot_nointr(struct lapic * la,uint32_t count)1515 lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count)
1516 {
1517 uint32_t value;
1518
1519 value = la->lvt_timer_base;
1520 value &= ~APIC_LVTT_TM;
1521 value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M;
1522 la->lvt_timer_last = value;
1523 lapic_write32(LAPIC_LVT_TIMER, value);
1524 lapic_write32(LAPIC_ICR_TIMER, count);
1525 }
1526
1527 static void
lapic_timer_periodic(struct lapic * la)1528 lapic_timer_periodic(struct lapic *la)
1529 {
1530 uint32_t value;
1531
1532 value = la->lvt_timer_base;
1533 value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1534 value |= APIC_LVTT_TM_PERIODIC;
1535 la->lvt_timer_last = value;
1536 lapic_write32(LAPIC_LVT_TIMER, value);
1537 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1538 }
1539
1540 static void
lapic_timer_deadline(struct lapic * la)1541 lapic_timer_deadline(struct lapic *la)
1542 {
1543 uint32_t value;
1544
1545 value = la->lvt_timer_base;
1546 value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1547 value |= APIC_LVTT_TM_TSCDLT;
1548 if (value != la->lvt_timer_last) {
1549 la->lvt_timer_last = value;
1550 lapic_write32_nofence(LAPIC_LVT_TIMER, value);
1551 if (!x2apic_mode)
1552 mfence();
1553 }
1554 wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc());
1555 }
1556
1557 static void
lapic_timer_stop(struct lapic * la)1558 lapic_timer_stop(struct lapic *la)
1559 {
1560 uint32_t value;
1561
1562 if (la->la_timer_mode == LAT_MODE_DEADLINE) {
1563 wrmsr(MSR_TSC_DEADLINE, 0);
1564 mfence();
1565 } else {
1566 value = la->lvt_timer_base;
1567 value &= ~APIC_LVTT_TM;
1568 value |= APIC_LVT_M;
1569 la->lvt_timer_last = value;
1570 lapic_write32(LAPIC_LVT_TIMER, value);
1571 }
1572 }
1573
1574 void
lapic_handle_cmc(void)1575 lapic_handle_cmc(void)
1576 {
1577 trap_check_kstack();
1578
1579 lapic_eoi();
1580 cmc_intr();
1581 }
1582
1583 /*
1584 * Called from the mca_init() to activate the CMC interrupt if this CPU is
1585 * responsible for monitoring any MC banks for CMC events. Since mca_init()
1586 * is called prior to lapic_setup() during boot, this just needs to unmask
1587 * this CPU's LVT_CMCI entry.
1588 */
1589 void
lapic_enable_cmc(void)1590 lapic_enable_cmc(void)
1591 {
1592 u_int apic_id;
1593
1594 #ifdef DEV_ATPIC
1595 if (!x2apic_mode && lapic_map == NULL)
1596 return;
1597 #endif
1598 apic_id = PCPU_GET(apic_id);
1599 KASSERT(lapics[apic_id].la_present,
1600 ("%s: missing APIC %u", __func__, apic_id));
1601 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0;
1602 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1;
1603 }
1604
1605 int
lapic_enable_mca_elvt(void)1606 lapic_enable_mca_elvt(void)
1607 {
1608 u_int apic_id;
1609 uint32_t value;
1610 int elvt_count;
1611
1612 #ifdef DEV_ATPIC
1613 if (lapic_map == NULL)
1614 return (-1);
1615 #endif
1616
1617 apic_id = PCPU_GET(apic_id);
1618 KASSERT(lapics[apic_id].la_present,
1619 ("%s: missing APIC %u", __func__, apic_id));
1620 elvt_count = amd_read_elvt_count();
1621 if (elvt_count <= APIC_ELVT_MCA)
1622 return (-1);
1623
1624 value = lapic_read32(LAPIC_EXT_LVT0 + APIC_ELVT_MCA);
1625 if ((value & APIC_LVT_M) == 0) {
1626 if (bootverbose)
1627 printf("AMD MCE Thresholding Extended LVT is already active\n");
1628 return (APIC_ELVT_MCA);
1629 }
1630 lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_masked = 0;
1631 lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_active = 1;
1632 return (APIC_ELVT_MCA);
1633 }
1634
1635 void
lapic_handle_error(void)1636 lapic_handle_error(void)
1637 {
1638 uint32_t esr;
1639
1640 trap_check_kstack();
1641
1642 /*
1643 * Read the contents of the error status register. Write to
1644 * the register first before reading from it to force the APIC
1645 * to update its value to indicate any errors that have
1646 * occurred since the previous write to the register.
1647 */
1648 lapic_write32(LAPIC_ESR, 0);
1649 esr = lapic_read32(LAPIC_ESR);
1650
1651 printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr);
1652 lapic_eoi();
1653 }
1654
1655 u_int
apic_cpuid(u_int apic_id)1656 apic_cpuid(u_int apic_id)
1657 {
1658 #ifdef SMP
1659 return apic_cpuids[apic_id];
1660 #else
1661 return 0;
1662 #endif
1663 }
1664
1665 /* Request a free IDT vector to be used by the specified IRQ. */
1666 u_int
apic_alloc_vector(u_int apic_id,u_int irq)1667 apic_alloc_vector(u_int apic_id, u_int irq)
1668 {
1669 u_int vector;
1670
1671 KASSERT(irq < num_io_irqs, ("Invalid IRQ %u", irq));
1672
1673 /*
1674 * Search for a free vector. Currently we just use a very simple
1675 * algorithm to find the first free vector.
1676 */
1677 mtx_lock_spin(&icu_lock);
1678 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1679 if (lapics[apic_id].la_ioint_irqs[vector] != IRQ_FREE)
1680 continue;
1681 lapics[apic_id].la_ioint_irqs[vector] = irq;
1682 mtx_unlock_spin(&icu_lock);
1683 return (vector + APIC_IO_INTS);
1684 }
1685 mtx_unlock_spin(&icu_lock);
1686 return (0);
1687 }
1688
1689 /*
1690 * Request 'count' free contiguous IDT vectors to be used by 'count'
1691 * IRQs. 'count' must be a power of two and the vectors will be
1692 * aligned on a boundary of 'align'. If the request cannot be
1693 * satisfied, 0 is returned.
1694 */
1695 u_int
apic_alloc_vectors(u_int apic_id,u_int * irqs,u_int count,u_int align)1696 apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
1697 {
1698 u_int first, run, vector;
1699
1700 KASSERT(powerof2(count), ("bad count"));
1701 KASSERT(powerof2(align), ("bad align"));
1702 KASSERT(align >= count, ("align < count"));
1703 #ifdef INVARIANTS
1704 for (run = 0; run < count; run++)
1705 KASSERT(irqs[run] < num_io_irqs, ("Invalid IRQ %u at index %u",
1706 irqs[run], run));
1707 #endif
1708
1709 /*
1710 * Search for 'count' free vectors. As with apic_alloc_vector(),
1711 * this just uses a simple first fit algorithm.
1712 */
1713 run = 0;
1714 first = 0;
1715 mtx_lock_spin(&icu_lock);
1716 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1717 /* Vector is in use, end run. */
1718 if (lapics[apic_id].la_ioint_irqs[vector] != IRQ_FREE) {
1719 run = 0;
1720 first = 0;
1721 continue;
1722 }
1723
1724 /* Start a new run if run == 0 and vector is aligned. */
1725 if (run == 0) {
1726 if (((vector + APIC_IO_INTS) & (align - 1)) != 0)
1727 continue;
1728 first = vector;
1729 }
1730 run++;
1731
1732 /* Keep looping if the run isn't long enough yet. */
1733 if (run < count)
1734 continue;
1735
1736 /* Found a run, assign IRQs and return the first vector. */
1737 for (vector = 0; vector < count; vector++)
1738 lapics[apic_id].la_ioint_irqs[first + vector] =
1739 irqs[vector];
1740 mtx_unlock_spin(&icu_lock);
1741 return (first + APIC_IO_INTS);
1742 }
1743 mtx_unlock_spin(&icu_lock);
1744 printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count);
1745 return (0);
1746 }
1747
1748 /*
1749 * Enable a vector for a particular apic_id. Since all lapics share idt
1750 * entries and ioint_handlers this enables the vector on all lapics. lapics
1751 * which do not have the vector configured would report spurious interrupts
1752 * should it fire.
1753 */
1754 void
apic_enable_vector(u_int apic_id,u_int vector)1755 apic_enable_vector(u_int apic_id, u_int vector)
1756 {
1757
1758 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1759 KASSERT(ioint_handlers[vector / 32] != NULL,
1760 ("No ISR handler for vector %u", vector));
1761 #ifdef KDTRACE_HOOKS
1762 KASSERT(vector != IDT_DTRACE_RET,
1763 ("Attempt to overwrite DTrace entry"));
1764 #endif
1765 setidt(vector, (pti ? ioint_pti_handlers : ioint_handlers)[vector / 32],
1766 SDT_APIC, SEL_KPL, GSEL_APIC);
1767 }
1768
1769 void
apic_disable_vector(u_int apic_id,u_int vector)1770 apic_disable_vector(u_int apic_id, u_int vector)
1771 {
1772
1773 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1774 #ifdef KDTRACE_HOOKS
1775 KASSERT(vector != IDT_DTRACE_RET,
1776 ("Attempt to overwrite DTrace entry"));
1777 #endif
1778 KASSERT(ioint_handlers[vector / 32] != NULL,
1779 ("No ISR handler for vector %u", vector));
1780 #ifdef notyet
1781 /*
1782 * We can not currently clear the idt entry because other cpus
1783 * may have a valid vector at this offset.
1784 */
1785 setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_APIC,
1786 SEL_KPL, GSEL_APIC);
1787 #endif
1788 }
1789
1790 /* Release an APIC vector when it's no longer in use. */
1791 void
apic_free_vector(u_int apic_id,u_int vector,u_int irq)1792 apic_free_vector(u_int apic_id, u_int vector, u_int irq)
1793 {
1794 struct thread *td;
1795
1796 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1797 vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1798 ("Vector %u does not map to an IRQ line", vector));
1799 KASSERT(irq < num_io_irqs, ("Invalid IRQ %u", irq));
1800 KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] ==
1801 irq, ("IRQ mismatch"));
1802 #ifdef KDTRACE_HOOKS
1803 KASSERT(vector != IDT_DTRACE_RET,
1804 ("Attempt to overwrite DTrace entry"));
1805 #endif
1806
1807 /*
1808 * Bind us to the cpu that owned the vector before freeing it so
1809 * we don't lose an interrupt delivery race.
1810 */
1811 td = curthread;
1812 if (!rebooting) {
1813 thread_lock(td);
1814 if (sched_is_bound(td))
1815 panic("apic_free_vector: Thread already bound.\n");
1816 sched_bind(td, apic_cpuid(apic_id));
1817 thread_unlock(td);
1818 }
1819 mtx_lock_spin(&icu_lock);
1820 lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = IRQ_FREE;
1821 mtx_unlock_spin(&icu_lock);
1822 if (!rebooting) {
1823 thread_lock(td);
1824 sched_unbind(td);
1825 thread_unlock(td);
1826 }
1827 }
1828
1829 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */
1830 static u_int
apic_idt_to_irq(u_int apic_id,u_int vector)1831 apic_idt_to_irq(u_int apic_id, u_int vector)
1832 {
1833 int irq;
1834
1835 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1836 vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1837 ("Vector %u does not map to an IRQ line", vector));
1838 #ifdef KDTRACE_HOOKS
1839 KASSERT(vector != IDT_DTRACE_RET,
1840 ("Attempt to overwrite DTrace entry"));
1841 #endif
1842 irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS];
1843 if (irq < 0)
1844 irq = 0;
1845 return (irq);
1846 }
1847
1848 #ifdef DDB
1849 /*
1850 * Dump data about APIC IDT vector mappings.
1851 */
DB_SHOW_COMMAND_FLAGS(apic,db_show_apic,DB_CMD_MEMSAFE)1852 DB_SHOW_COMMAND_FLAGS(apic, db_show_apic, DB_CMD_MEMSAFE)
1853 {
1854 struct intsrc *isrc;
1855 int i, verbose;
1856 u_int apic_id;
1857 u_int irq;
1858
1859 if (strcmp(modif, "vv") == 0)
1860 verbose = 2;
1861 else if (strcmp(modif, "v") == 0)
1862 verbose = 1;
1863 else
1864 verbose = 0;
1865 for (apic_id = 0; apic_id <= max_apic_id; apic_id++) {
1866 if (lapics[apic_id].la_present == 0)
1867 continue;
1868 db_printf("Interrupts bound to lapic %u\n", apic_id);
1869 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
1870 irq = lapics[apic_id].la_ioint_irqs[i];
1871 if (irq == IRQ_FREE || irq == IRQ_SYSCALL)
1872 continue;
1873 #ifdef KDTRACE_HOOKS
1874 if (irq == IRQ_DTRACE_RET)
1875 continue;
1876 #endif
1877 #ifdef XENHVM
1878 if (irq == IRQ_EVTCHN)
1879 continue;
1880 #endif
1881 db_printf("vec 0x%2x -> ", i + APIC_IO_INTS);
1882 if (irq == IRQ_TIMER)
1883 db_printf("lapic timer\n");
1884 else if (irq < num_io_irqs) {
1885 isrc = intr_lookup_source(irq);
1886 if (isrc == NULL || verbose == 0)
1887 db_printf("IRQ %u\n", irq);
1888 else
1889 db_dump_intr_event(isrc->is_event,
1890 verbose == 2);
1891 } else
1892 db_printf("IRQ %u ???\n", irq);
1893 }
1894 }
1895 }
1896
1897 static void
dump_mask(const char * prefix,uint32_t v,int base)1898 dump_mask(const char *prefix, uint32_t v, int base)
1899 {
1900 int i, first;
1901
1902 first = 1;
1903 for (i = 0; i < 32; i++)
1904 if (v & (1 << i)) {
1905 if (first) {
1906 db_printf("%s:", prefix);
1907 first = 0;
1908 }
1909 db_printf(" %02x", base + i);
1910 }
1911 if (!first)
1912 db_printf("\n");
1913 }
1914
1915 /* Show info from the lapic regs for this CPU. */
DB_SHOW_COMMAND_FLAGS(lapic,db_show_lapic,DB_CMD_MEMSAFE)1916 DB_SHOW_COMMAND_FLAGS(lapic, db_show_lapic, DB_CMD_MEMSAFE)
1917 {
1918 const struct lvt *l;
1919 int elvt_count, lvts_count, i;
1920 uint32_t v, vr;
1921
1922 db_printf("lapic ID = %d\n", lapic_id());
1923 v = lapic_read32(LAPIC_VERSION);
1924 db_printf("version = %d.%d (%#x) \n", (v & APIC_VER_VERSION) >> 4,
1925 v & 0xf, v);
1926 db_printf("max LVT = %d\n", lapic_maxlvt(v));
1927 vr = lapic_read32(LAPIC_SVR);
1928 db_printf("SVR = %02x (%s)\n", vr & APIC_SVR_VECTOR,
1929 vr & APIC_SVR_ENABLE ? "enabled" : "disabled");
1930 db_printf("TPR = %02x\n", lapic_read32(LAPIC_TPR));
1931
1932 lvts_count = min(nitems(lvts), lapic_maxlvt(v) + 1);
1933 for (i = 0; i < lvts_count; i++) {
1934 l = &lvts[i];
1935 db_printf("LVT%d (reg %#x %-5s) = %#010x\n", i, l->lvt_reg,
1936 l->lvt_desc, lapic_read32(l->lvt_reg));
1937 }
1938
1939 elvt_count = amd_read_elvt_count();
1940 for (i = 0; i < elvt_count; i++) {
1941 l = &elvts[i];
1942 db_printf("ELVT%d (reg %#x %-5s) = %#010x\n", i, l->lvt_reg,
1943 l->lvt_desc, lapic_read32(l->lvt_reg));
1944 }
1945
1946 #define dump_field(prefix, regn, index) \
1947 dump_mask(__XSTRING(prefix ## index), \
1948 lapic_read32(LAPIC_ ## regn ## index), \
1949 index * 32)
1950
1951 db_printf("In-service Interrupts:\n");
1952 dump_field(isr, ISR, 0);
1953 dump_field(isr, ISR, 1);
1954 dump_field(isr, ISR, 2);
1955 dump_field(isr, ISR, 3);
1956 dump_field(isr, ISR, 4);
1957 dump_field(isr, ISR, 5);
1958 dump_field(isr, ISR, 6);
1959 dump_field(isr, ISR, 7);
1960
1961 db_printf("TMR Interrupts:\n");
1962 dump_field(tmr, TMR, 0);
1963 dump_field(tmr, TMR, 1);
1964 dump_field(tmr, TMR, 2);
1965 dump_field(tmr, TMR, 3);
1966 dump_field(tmr, TMR, 4);
1967 dump_field(tmr, TMR, 5);
1968 dump_field(tmr, TMR, 6);
1969 dump_field(tmr, TMR, 7);
1970
1971 db_printf("IRR Interrupts:\n");
1972 dump_field(irr, IRR, 0);
1973 dump_field(irr, IRR, 1);
1974 dump_field(irr, IRR, 2);
1975 dump_field(irr, IRR, 3);
1976 dump_field(irr, IRR, 4);
1977 dump_field(irr, IRR, 5);
1978 dump_field(irr, IRR, 6);
1979 dump_field(irr, IRR, 7);
1980
1981 #undef dump_field
1982 }
1983 #endif
1984
1985 /*
1986 * APIC probing support code. This includes code to manage enumerators.
1987 */
1988
1989 static SLIST_HEAD(, apic_enumerator) enumerators =
1990 SLIST_HEAD_INITIALIZER(enumerators);
1991 static struct apic_enumerator *best_enum;
1992
1993 void
apic_register_enumerator(struct apic_enumerator * enumerator)1994 apic_register_enumerator(struct apic_enumerator *enumerator)
1995 {
1996 #ifdef INVARIANTS
1997 struct apic_enumerator *apic_enum;
1998
1999 SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
2000 if (apic_enum == enumerator)
2001 panic("%s: Duplicate register of %s", __func__,
2002 enumerator->apic_name);
2003 }
2004 #endif
2005 SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
2006 }
2007
2008 /*
2009 * We have to look for CPU's very, very early because certain subsystems
2010 * want to know how many CPU's we have extremely early on in the boot
2011 * process.
2012 */
2013 static void
apic_init(void * dummy __unused)2014 apic_init(void *dummy __unused)
2015 {
2016 struct apic_enumerator *enumerator;
2017 int retval, best;
2018
2019 /* We only support built in local APICs. */
2020 if (!(cpu_feature & CPUID_APIC))
2021 return;
2022
2023 /* Don't probe if APIC mode is disabled. */
2024 if (resource_disabled("apic", 0))
2025 return;
2026
2027 /* Probe all the enumerators to find the best match. */
2028 best_enum = NULL;
2029 best = 0;
2030 SLIST_FOREACH(enumerator, &enumerators, apic_next) {
2031 retval = enumerator->apic_probe();
2032 if (retval > 0)
2033 continue;
2034 if (best_enum == NULL || best < retval) {
2035 best_enum = enumerator;
2036 best = retval;
2037 }
2038 }
2039 if (best_enum == NULL) {
2040 if (bootverbose)
2041 printf("APIC: Could not find any APICs.\n");
2042 #ifndef DEV_ATPIC
2043 panic("running without device atpic requires a local APIC");
2044 #endif
2045 return;
2046 }
2047
2048 if (bootverbose)
2049 printf("APIC: Using the %s enumerator.\n",
2050 best_enum->apic_name);
2051
2052 #ifdef I686_CPU
2053 /*
2054 * To work around an errata, we disable the local APIC on some
2055 * CPUs during early startup. We need to turn the local APIC back
2056 * on on such CPUs now.
2057 */
2058 ppro_reenable_apic();
2059 #endif
2060
2061 /* Probe the CPU's in the system. */
2062 retval = best_enum->apic_probe_cpus();
2063 if (retval != 0)
2064 printf("%s: Failed to probe CPUs: returned %d\n",
2065 best_enum->apic_name, retval);
2066
2067 }
2068 SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL);
2069
2070 /*
2071 * Setup the local APIC. We have to do this prior to starting up the APs
2072 * in the SMP case.
2073 */
2074 static void
apic_setup_local(void * dummy __unused)2075 apic_setup_local(void *dummy __unused)
2076 {
2077 int retval;
2078
2079 if (best_enum == NULL)
2080 return;
2081
2082 lapics = malloc(sizeof(*lapics) * (max_apic_id + 1), M_LAPIC,
2083 M_WAITOK | M_ZERO);
2084
2085 /* Initialize the local APIC. */
2086 retval = best_enum->apic_setup_local();
2087 if (retval != 0)
2088 printf("%s: Failed to setup the local APIC: returned %d\n",
2089 best_enum->apic_name, retval);
2090 }
2091 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL);
2092
2093 /* Are we in a VM which supports the Extended Destination ID standard? */
2094 int apic_ext_dest_id = -1;
2095 SYSCTL_INT(_machdep, OID_AUTO, apic_ext_dest_id, CTLFLAG_RDTUN, &apic_ext_dest_id, 0,
2096 "Use APIC Extended Destination IDs");
2097
2098 /* Detect support for Extended Destination IDs. */
2099 static void
detect_extended_dest_id(void)2100 detect_extended_dest_id(void)
2101 {
2102 u_int regs[4];
2103
2104 /* Check if we support extended destination IDs. */
2105 switch (vm_guest) {
2106 case VM_GUEST_XEN:
2107 cpuid_count(hv_base + 4, 0, regs);
2108 if (regs[0] & XEN_HVM_CPUID_EXT_DEST_ID)
2109 apic_ext_dest_id = 1;
2110 break;
2111 case VM_GUEST_HV:
2112 cpuid_count(CPUID_LEAF_HV_STACK_INTERFACE, 0, regs);
2113 if (regs[0] != HYPERV_STACK_INTERFACE_EAX_SIG)
2114 break;
2115 cpuid_count(CPUID_LEAF_HV_STACK_PROPERTIES, 0, regs);
2116 if (regs[0] & HYPERV_PROPERTIES_EXT_DEST_ID)
2117 apic_ext_dest_id = 1;
2118 break;
2119 case VM_GUEST_KVM:
2120 kvm_cpuid_get_features(regs);
2121 if (regs[0] & KVM_FEATURE_MSI_EXT_DEST_ID)
2122 apic_ext_dest_id = 1;
2123 break;
2124 case VM_GUEST_BHYVE:
2125 if (hv_high < CPUID_BHYVE_FEATURES)
2126 break;
2127 cpuid_count(CPUID_BHYVE_FEATURES, 0, regs);
2128 if (regs[0] & CPUID_BHYVE_FEAT_EXT_DEST_ID)
2129 apic_ext_dest_id = 1;
2130 break;
2131 }
2132 }
2133
2134 /*
2135 * Setup the I/O APICs.
2136 */
2137 static void
apic_setup_io(void * dummy __unused)2138 apic_setup_io(void *dummy __unused)
2139 {
2140 int retval;
2141
2142 if (best_enum == NULL)
2143 return;
2144
2145 /* Check hypervisor support for extended destination IDs. */
2146 if (apic_ext_dest_id == -1)
2147 detect_extended_dest_id();
2148
2149 /*
2150 * Local APIC must be registered before other PICs and pseudo PICs
2151 * for proper suspend/resume order.
2152 */
2153 intr_register_pic(&lapic_pic);
2154
2155 retval = best_enum->apic_setup_io();
2156 if (retval != 0)
2157 printf("%s: Failed to setup I/O APICs: returned %d\n",
2158 best_enum->apic_name, retval);
2159
2160 /*
2161 * Finish setting up the local APIC on the BSP once we know
2162 * how to properly program the LINT pins. In particular, this
2163 * enables the EOI suppression mode, if LAPIC supports it and
2164 * user did not disable the mode.
2165 */
2166 lapic_setup(1);
2167 if (bootverbose)
2168 lapic_dump("BSP");
2169
2170 /* Enable the MSI "pic". */
2171 msi_init();
2172
2173 #ifdef XENHVM
2174 xen_intr_alloc_irqs();
2175 #endif
2176 }
2177 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL);
2178
2179 #ifdef SMP
2180 /*
2181 * Inter Processor Interrupt functions. The lapic_ipi_*() functions are
2182 * private to the MD code. The public interface for the rest of the
2183 * kernel is defined in mp_machdep.c.
2184 */
2185
2186 /*
2187 * Wait delay microseconds for IPI to be sent. If delay is -1, we
2188 * wait forever.
2189 */
2190 int
lapic_ipi_wait(int delay)2191 lapic_ipi_wait(int delay)
2192 {
2193 uint64_t rx;
2194
2195 /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */
2196 if (x2apic_mode)
2197 return (1);
2198
2199 for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) {
2200 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) ==
2201 APIC_DELSTAT_IDLE)
2202 return (1);
2203 ia32_pause();
2204 }
2205 return (0);
2206 }
2207
2208 void
lapic_ipi_raw(register_t icrlo,u_int dest)2209 lapic_ipi_raw(register_t icrlo, u_int dest)
2210 {
2211 uint32_t icrhi;
2212
2213 /* XXX: Need more sanity checking of icrlo? */
2214 KASSERT(x2apic_mode || lapic_map != NULL,
2215 ("%s called too early", __func__));
2216 KASSERT(x2apic_mode ||
2217 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
2218 ("%s: invalid dest field", __func__));
2219 KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
2220 ("%s: reserved bits set in ICR LO register", __func__));
2221
2222 if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
2223 if (x2apic_mode)
2224 icrhi = dest;
2225 else
2226 icrhi = dest << APIC_ID_SHIFT;
2227 lapic_write_icr(icrhi, icrlo);
2228 } else {
2229 lapic_write_icr_lo(icrlo);
2230 }
2231 }
2232
2233 #ifdef DETECT_DEADLOCK
2234 #define AFTER_SPIN 50
2235 #endif
2236
2237 static void
native_lapic_ipi_vectored(u_int vector,int dest)2238 native_lapic_ipi_vectored(u_int vector, int dest)
2239 {
2240 register_t icrlo, destfield;
2241
2242 KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
2243 ("%s: invalid vector %d", __func__, vector));
2244
2245 destfield = 0;
2246 switch (dest) {
2247 case APIC_IPI_DEST_SELF:
2248 if (x2apic_mode && vector < IPI_NMI_FIRST) {
2249 lapic_write_self_ipi(vector);
2250 return;
2251 }
2252 icrlo = APIC_DEST_SELF;
2253 break;
2254 case APIC_IPI_DEST_ALL:
2255 icrlo = APIC_DEST_ALLISELF;
2256 break;
2257 case APIC_IPI_DEST_OTHERS:
2258 icrlo = APIC_DEST_ALLESELF;
2259 break;
2260 default:
2261 icrlo = 0;
2262 KASSERT(x2apic_mode ||
2263 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
2264 ("%s: invalid destination 0x%x", __func__, dest));
2265 destfield = dest;
2266 }
2267
2268 /*
2269 * NMI IPIs are just fake vectors used to send a NMI. Use special rules
2270 * regarding NMIs if passed, otherwise specify the vector.
2271 */
2272 if (vector >= IPI_NMI_FIRST)
2273 icrlo |= APIC_DELMODE_NMI;
2274 else
2275 icrlo |= vector | APIC_DELMODE_FIXED;
2276 icrlo |= APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
2277
2278 /* Wait for an earlier IPI to finish. */
2279 if (!lapic_ipi_wait(lapic_ds_idle_timeout)) {
2280 if (KERNEL_PANICKED())
2281 return;
2282 else
2283 panic("APIC: Previous IPI is stuck");
2284 }
2285
2286 lapic_ipi_raw(icrlo, destfield);
2287
2288 #ifdef DETECT_DEADLOCK
2289 /* Wait for IPI to be delivered. */
2290 if (!lapic_ipi_wait(AFTER_SPIN)) {
2291 #ifdef needsattention
2292 /*
2293 * XXX FIXME:
2294 *
2295 * The above function waits for the message to actually be
2296 * delivered. It breaks out after an arbitrary timeout
2297 * since the message should eventually be delivered (at
2298 * least in theory) and that if it wasn't we would catch
2299 * the failure with the check above when the next IPI is
2300 * sent.
2301 *
2302 * We could skip this wait entirely, EXCEPT it probably
2303 * protects us from other routines that assume that the
2304 * message was delivered and acted upon when this function
2305 * returns.
2306 */
2307 printf("APIC: IPI might be stuck\n");
2308 #else /* !needsattention */
2309 /* Wait until mesage is sent without a timeout. */
2310 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND)
2311 ia32_pause();
2312 #endif /* needsattention */
2313 }
2314 #endif /* DETECT_DEADLOCK */
2315 }
2316
2317 void (*ipi_vectored)(u_int, int) = &native_lapic_ipi_vectored;
2318 #endif /* SMP */
2319
2320 /*
2321 * Since the IDT is shared by all CPUs the IPI slot update needs to be globally
2322 * visible.
2323 *
2324 * Consider the case where an IPI is generated immediately after allocation:
2325 * vector = lapic_ipi_alloc(ipifunc);
2326 * ipi_selected(other_cpus, vector);
2327 *
2328 * In xAPIC mode a write to ICR_LO has serializing semantics because the
2329 * APIC page is mapped as an uncached region. In x2APIC mode there is an
2330 * explicit 'mfence' before the ICR MSR is written. Therefore in both cases
2331 * the IDT slot update is globally visible before the IPI is delivered.
2332 */
2333 int
lapic_ipi_alloc(inthand_t * ipifunc)2334 lapic_ipi_alloc(inthand_t *ipifunc)
2335 {
2336 struct gate_descriptor *ip;
2337 long func;
2338 int idx, vector;
2339
2340 KASSERT(ipifunc != &IDTVEC(rsvd) && ipifunc != &IDTVEC(rsvd_pti),
2341 ("invalid ipifunc %p", ipifunc));
2342
2343 vector = -1;
2344 mtx_lock_spin(&icu_lock);
2345 for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
2346 ip = &idt[idx];
2347 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
2348 #ifdef __i386__
2349 func -= setidt_disp;
2350 #endif
2351 if ((!pti && func == (uintptr_t)&IDTVEC(rsvd)) ||
2352 (pti && func == (uintptr_t)&IDTVEC(rsvd_pti))) {
2353 vector = idx;
2354 setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
2355 break;
2356 }
2357 }
2358 mtx_unlock_spin(&icu_lock);
2359 return (vector);
2360 }
2361
2362 void
lapic_ipi_free(int vector)2363 lapic_ipi_free(int vector)
2364 {
2365 struct gate_descriptor *ip;
2366 long func __diagused;
2367
2368 KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST,
2369 ("%s: invalid vector %d", __func__, vector));
2370
2371 mtx_lock_spin(&icu_lock);
2372 ip = &idt[vector];
2373 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
2374 #ifdef __i386__
2375 func -= setidt_disp;
2376 #endif
2377 KASSERT(func != (uintptr_t)&IDTVEC(rsvd) &&
2378 func != (uintptr_t)&IDTVEC(rsvd_pti),
2379 ("invalid idtfunc %#lx", func));
2380 setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_APIC,
2381 SEL_KPL, GSEL_APIC);
2382 mtx_unlock_spin(&icu_lock);
2383 }
2384