1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1996, by Steve Passe
5 * All rights reserved.
6 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. The name of the developer may NOT be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 * 3. Neither the name of the author nor the names of any co-contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 /*
33 * Local APIC support on Pentium and later processors.
34 */
35
36 #include <sys/cdefs.h>
37 #include "opt_atpic.h"
38
39 #include "opt_ddb.h"
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/asan.h>
44 #include <sys/bus.h>
45 #include <sys/kernel.h>
46 #include <sys/lock.h>
47 #include <sys/malloc.h>
48 #include <sys/msan.h>
49 #include <sys/mutex.h>
50 #include <sys/pcpu.h>
51 #include <sys/proc.h>
52 #include <sys/refcount.h>
53 #include <sys/sched.h>
54 #include <sys/smp.h>
55 #include <sys/sysctl.h>
56 #include <sys/timeet.h>
57 #include <sys/timetc.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61
62 #include <x86/apicreg.h>
63 #include <machine/atomic.h>
64 #include <machine/clock.h>
65 #include <machine/cpufunc.h>
66 #include <machine/cputypes.h>
67 #include <machine/fpu.h>
68 #include <machine/frame.h>
69 #include <machine/intr_machdep.h>
70 #include <x86/apicvar.h>
71 #include <x86/mca.h>
72 #include <machine/md_var.h>
73 #include <machine/smp.h>
74 #include <machine/specialreg.h>
75 #include <x86/init.h>
76 #include <x86/kvm.h>
77 #include <contrib/xen/arch-x86/cpuid.h>
78 #include <x86/bhyve.h>
79 #include <dev/hyperv/vmbus/x86/hyperv_reg.h>
80
81 #ifdef DDB
82 #include <sys/interrupt.h>
83 #include <ddb/ddb.h>
84 #endif
85
86 #ifdef __amd64__
87 #define SDT_APIC SDT_SYSIGT
88 #define GSEL_APIC 0
89 #else
90 #define SDT_APIC SDT_SYS386IGT
91 #define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL)
92 #endif
93
94 static MALLOC_DEFINE(M_LAPIC, "local_apic", "Local APIC items");
95
96 /* Sanity checks on IDT vectors. */
97 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
98 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
99 CTASSERT(APIC_LOCAL_INTS == 240);
100 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);
101
102 /*
103 * I/O interrupts use non-negative IRQ values. These values are used
104 * to mark unused IDT entries or IDT entries reserved for a non-I/O
105 * interrupt.
106 */
107 #define IRQ_FREE -1
108 #define IRQ_TIMER -2
109 #define IRQ_SYSCALL -3
110 #define IRQ_DTRACE_RET -4
111 #define IRQ_EVTCHN -5
112
113 enum lat_timer_mode {
114 LAT_MODE_UNDEF = 0,
115 LAT_MODE_PERIODIC = 1,
116 LAT_MODE_ONESHOT = 2,
117 LAT_MODE_DEADLINE = 3,
118 };
119
120 /*
121 * Support for local APICs. Local APICs manage interrupts on each
122 * individual processor as opposed to I/O APICs which receive interrupts
123 * from I/O devices and then forward them on to the local APICs.
124 *
125 * Local APICs can also send interrupts to each other thus providing the
126 * mechanism for IPIs.
127 */
128
129 struct lvt {
130 u_int lvt_edgetrigger:1;
131 u_int lvt_activehi:1;
132 u_int lvt_masked:1;
133 u_int lvt_active:1;
134 u_int lvt_mode:16;
135 u_int lvt_vector:8;
136 u_int lvt_reg;
137 const char *lvt_desc;
138 };
139
140 struct lapic {
141 struct lvt la_lvts[APIC_LVT_MAX + 1];
142 struct lvt la_elvts[APIC_ELVT_MAX + 1];
143 u_int la_id:8;
144 u_int la_cluster:4;
145 u_int la_cluster_id:2;
146 u_int la_present:1;
147 u_long *la_timer_count;
148 uint64_t la_timer_period;
149 enum lat_timer_mode la_timer_mode;
150 uint32_t lvt_timer_base;
151 uint32_t lvt_timer_last;
152 /* Include IDT_SYSCALL to make indexing easier. */
153 int la_ioint_irqs[APIC_NUM_IOINTS + 1];
154 } static *lapics;
155
156 /* Global defaults for local APIC LVT entries. */
157 static struct lvt lvts[] = {
158 /* LINT0: masked ExtINT */
159 [APIC_LVT_LINT0] = {
160 .lvt_edgetrigger = 1,
161 .lvt_activehi = 1,
162 .lvt_masked = 1,
163 .lvt_active = 1,
164 .lvt_mode = APIC_LVT_DM_EXTINT,
165 .lvt_vector = 0,
166 .lvt_reg = LAPIC_LVT_LINT0,
167 .lvt_desc = "LINT0",
168 },
169 /* LINT1: NMI */
170 [APIC_LVT_LINT1] = {
171 .lvt_edgetrigger = 1,
172 .lvt_activehi = 1,
173 .lvt_masked = 0,
174 .lvt_active = 1,
175 .lvt_mode = APIC_LVT_DM_NMI,
176 .lvt_vector = 0,
177 .lvt_reg = LAPIC_LVT_LINT1,
178 .lvt_desc = "LINT1",
179 },
180 [APIC_LVT_TIMER] = {
181 .lvt_edgetrigger = 1,
182 .lvt_activehi = 1,
183 .lvt_masked = 1,
184 .lvt_active = 1,
185 .lvt_mode = APIC_LVT_DM_FIXED,
186 .lvt_vector = APIC_TIMER_INT,
187 .lvt_reg = LAPIC_LVT_TIMER,
188 .lvt_desc = "TIMER",
189 },
190 [APIC_LVT_ERROR] = {
191 .lvt_edgetrigger = 1,
192 .lvt_activehi = 1,
193 .lvt_masked = 0,
194 .lvt_active = 1,
195 .lvt_mode = APIC_LVT_DM_FIXED,
196 .lvt_vector = APIC_ERROR_INT,
197 .lvt_reg = LAPIC_LVT_ERROR,
198 .lvt_desc = "ERROR",
199 },
200 [APIC_LVT_PMC] = {
201 .lvt_edgetrigger = 1,
202 .lvt_activehi = 1,
203 .lvt_masked = 1,
204 .lvt_active = 1,
205 .lvt_mode = APIC_LVT_DM_NMI,
206 .lvt_vector = 0,
207 .lvt_reg = LAPIC_LVT_PCINT,
208 .lvt_desc = "PMC",
209 },
210 [APIC_LVT_THERMAL] = {
211 .lvt_edgetrigger = 1,
212 .lvt_activehi = 1,
213 .lvt_masked = 1,
214 .lvt_active = 1,
215 .lvt_mode = APIC_LVT_DM_FIXED,
216 .lvt_vector = APIC_THERMAL_INT,
217 .lvt_reg = LAPIC_LVT_THERMAL,
218 .lvt_desc = "THERM",
219 },
220 [APIC_LVT_CMCI] = {
221 .lvt_edgetrigger = 1,
222 .lvt_activehi = 1,
223 .lvt_masked = 1,
224 .lvt_active = 1,
225 .lvt_mode = APIC_LVT_DM_FIXED,
226 .lvt_vector = APIC_CMC_INT,
227 .lvt_reg = LAPIC_LVT_CMCI,
228 .lvt_desc = "CMCI",
229 },
230 };
231
232 /* Global defaults for AMD local APIC ELVT entries. */
233 static struct lvt elvts[] = {
234 [APIC_ELVT_IBS] = {
235 .lvt_edgetrigger = 1,
236 .lvt_activehi = 1,
237 .lvt_masked = 1,
238 .lvt_active = 1,
239 .lvt_mode = APIC_LVT_DM_NMI,
240 .lvt_vector = 0,
241 .lvt_reg = LAPIC_EXT_LVT0,
242 .lvt_desc = "IBS",
243 },
244 [APIC_ELVT_MCA] = {
245 .lvt_edgetrigger = 1,
246 .lvt_activehi = 1,
247 .lvt_masked = 1,
248 .lvt_active = 0,
249 .lvt_mode = APIC_LVT_DM_FIXED,
250 .lvt_vector = APIC_CMC_INT,
251 .lvt_reg = LAPIC_EXT_LVT1,
252 .lvt_desc = "MCA",
253 },
254 [APIC_ELVT_DEI] = {
255 .lvt_edgetrigger = 1,
256 .lvt_activehi = 1,
257 .lvt_masked = 1,
258 .lvt_active = 0,
259 .lvt_mode = APIC_LVT_DM_FIXED,
260 .lvt_vector = 0,
261 .lvt_reg = LAPIC_EXT_LVT2,
262 .lvt_desc = "ELVT2",
263 },
264 [APIC_ELVT_SBI] = {
265 .lvt_edgetrigger = 1,
266 .lvt_activehi = 1,
267 .lvt_masked = 1,
268 .lvt_active = 0,
269 .lvt_mode = APIC_LVT_DM_FIXED,
270 .lvt_vector = 0,
271 .lvt_reg = LAPIC_EXT_LVT3,
272 .lvt_desc = "ELVT3",
273 },
274 };
275
276 static inthand_t *ioint_handlers[] = {
277 NULL, /* 0 - 31 */
278 IDTVEC(apic_isr1), /* 32 - 63 */
279 IDTVEC(apic_isr2), /* 64 - 95 */
280 IDTVEC(apic_isr3), /* 96 - 127 */
281 IDTVEC(apic_isr4), /* 128 - 159 */
282 IDTVEC(apic_isr5), /* 160 - 191 */
283 IDTVEC(apic_isr6), /* 192 - 223 */
284 IDTVEC(apic_isr7), /* 224 - 255 */
285 };
286
287 static inthand_t *ioint_pti_handlers[] = {
288 NULL, /* 0 - 31 */
289 IDTVEC(apic_isr1_pti), /* 32 - 63 */
290 IDTVEC(apic_isr2_pti), /* 64 - 95 */
291 IDTVEC(apic_isr3_pti), /* 96 - 127 */
292 IDTVEC(apic_isr4_pti), /* 128 - 159 */
293 IDTVEC(apic_isr5_pti), /* 160 - 191 */
294 IDTVEC(apic_isr6_pti), /* 192 - 223 */
295 IDTVEC(apic_isr7_pti), /* 224 - 255 */
296 };
297
298 static u_int32_t lapic_timer_divisors[] = {
299 APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
300 APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
301 };
302
303 extern inthand_t IDTVEC(rsvd_pti), IDTVEC(rsvd);
304
305 volatile char *lapic_map;
306 vm_paddr_t lapic_paddr = DEFAULT_APIC_BASE;
307 int x2apic_mode;
308 int lapic_eoi_suppression;
309 static int lapic_timer_tsc_deadline;
310 static u_long lapic_timer_divisor, count_freq;
311 static struct eventtimer lapic_et;
312 #ifdef SMP
313 static uint64_t lapic_ipi_wait_mult;
314 static int __read_mostly lapic_ds_idle_timeout = 1000000;
315 #endif
316 unsigned int max_apic_id;
317 static lapic_thermal_handler_t *lapic_thermal_function;
318 static void *lapic_thermal_function_arg;
319 static int pcint_refcnt = 0;
320
321 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
322 "APIC options");
323 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
324 SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
325 &lapic_eoi_suppression, 0, "");
326 SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
327 &lapic_timer_tsc_deadline, 0, "");
328 #ifdef SMP
329 SYSCTL_INT(_hw_apic, OID_AUTO, ds_idle_timeout, CTLFLAG_RWTUN,
330 &lapic_ds_idle_timeout, 0,
331 "timeout (in us) for APIC Delivery Status to become Idle (xAPIC only)");
332 #endif
333
334 static void lapic_calibrate_initcount(struct lapic *la);
335
336 /*
337 * Use __nosanitizethread to exempt the LAPIC I/O accessors from KCSan
338 * instrumentation. Otherwise, if x2APIC is not available, use of the global
339 * lapic_map will generate a KCSan false positive. While the mapping is
340 * shared among all CPUs, the physical access will always take place on the
341 * local CPU's APIC, so there isn't in fact a race here. Furthermore, the
342 * KCSan warning printf can cause a panic if issued during LAPIC access,
343 * due to attempted recursive use of event timer resources.
344 */
345
346 static uint32_t __nosanitizethread
lapic_read32(enum LAPIC_REGISTERS reg)347 lapic_read32(enum LAPIC_REGISTERS reg)
348 {
349 uint32_t res;
350
351 if (x2apic_mode) {
352 res = rdmsr32(MSR_APIC_000 + reg);
353 } else {
354 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL);
355 }
356 return (res);
357 }
358
359 static void __nosanitizethread
lapic_write32(enum LAPIC_REGISTERS reg,uint32_t val)360 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
361 {
362
363 if (x2apic_mode) {
364 mfence();
365 lfence();
366 wrmsr(MSR_APIC_000 + reg, val);
367 } else {
368 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
369 }
370 }
371
372 static void __nosanitizethread
lapic_write32_nofence(enum LAPIC_REGISTERS reg,uint32_t val)373 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
374 {
375
376 if (x2apic_mode) {
377 wrmsr(MSR_APIC_000 + reg, val);
378 } else {
379 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
380 }
381 }
382
383 static inline uint32_t __pure2
lapic_version(void)384 lapic_version(void)
385 {
386 return (lapic_read32(LAPIC_VERSION));
387 }
388
389 /*
390 * Calculate the max index of the present LVT entry from the value of
391 * the LAPIC version register.
392 */
393 static inline int __pure2
lapic_version_maxlvt(uint32_t version)394 lapic_version_maxlvt(uint32_t version)
395 {
396 return ((version & APIC_VER_MAXLVT) >> MAXLVTSHIFT);
397 }
398
399 static inline int __pure2
lapic_maxlvt(void)400 lapic_maxlvt(void)
401 {
402 return (lapic_version_maxlvt(lapic_version()));
403 }
404
405 #ifdef SMP
406 static uint64_t
lapic_read_icr_lo(void)407 lapic_read_icr_lo(void)
408 {
409
410 return (lapic_read32(LAPIC_ICR_LO));
411 }
412
413 static void
lapic_write_icr(uint32_t vhi,uint32_t vlo)414 lapic_write_icr(uint32_t vhi, uint32_t vlo)
415 {
416 register_t saveintr;
417 uint64_t v;
418
419 if (x2apic_mode) {
420 v = ((uint64_t)vhi << 32) | vlo;
421 mfence();
422 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
423 } else {
424 saveintr = intr_disable();
425 lapic_write32(LAPIC_ICR_HI, vhi);
426 lapic_write32(LAPIC_ICR_LO, vlo);
427 intr_restore(saveintr);
428 }
429 }
430
431 static void
lapic_write_icr_lo(uint32_t vlo)432 lapic_write_icr_lo(uint32_t vlo)
433 {
434
435 if (x2apic_mode) {
436 mfence();
437 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, vlo);
438 } else {
439 lapic_write32(LAPIC_ICR_LO, vlo);
440 }
441 }
442
443 static void
lapic_write_self_ipi(uint32_t vector)444 lapic_write_self_ipi(uint32_t vector)
445 {
446
447 KASSERT(x2apic_mode, ("SELF IPI write in xAPIC mode"));
448 wrmsr(MSR_APIC_000 + LAPIC_SELF_IPI, vector);
449 }
450 #endif /* SMP */
451
452 static void
lapic_enable_x2apic(void)453 lapic_enable_x2apic(void)
454 {
455 uint64_t apic_base;
456
457 apic_base = rdmsr(MSR_APICBASE);
458 apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
459 wrmsr(MSR_APICBASE, apic_base);
460 }
461
462 bool
lapic_is_x2apic(void)463 lapic_is_x2apic(void)
464 {
465 uint64_t apic_base;
466
467 apic_base = rdmsr(MSR_APICBASE);
468 return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) ==
469 (APICBASE_X2APIC | APICBASE_ENABLED));
470 }
471
472 static void lapic_early_mask_vecs(void);
473 static void lapic_enable(void);
474 static void lapic_resume(struct pic *pic, bool suspend_cancelled);
475 static void lapic_timer_oneshot(struct lapic *);
476 static void lapic_timer_oneshot_nointr(struct lapic *, uint32_t);
477 static void lapic_timer_periodic(struct lapic *);
478 static void lapic_timer_deadline(struct lapic *);
479 static void lapic_timer_stop(struct lapic *);
480 static void lapic_timer_set_divisor(u_int divisor);
481 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
482 static int lapic_et_start(struct eventtimer *et,
483 sbintime_t first, sbintime_t period);
484 static int lapic_et_stop(struct eventtimer *et);
485 static u_int apic_idt_to_irq(u_int apic_id, u_int vector);
486 static void lapic_set_tpr(u_int vector);
487
488 struct pic lapic_pic = { .pic_resume = lapic_resume };
489
490 static uint32_t
lvt_mode_impl(struct lapic * la,struct lvt * lvt,u_int pin,uint32_t value)491 lvt_mode_impl(struct lapic *la, struct lvt *lvt, u_int pin, uint32_t value)
492 {
493
494 value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
495 APIC_LVT_VECTOR);
496 if (lvt->lvt_edgetrigger == 0)
497 value |= APIC_LVT_TM;
498 if (lvt->lvt_activehi == 0)
499 value |= APIC_LVT_IIPP_INTALO;
500 if (lvt->lvt_masked)
501 value |= APIC_LVT_M;
502 value |= lvt->lvt_mode;
503 switch (lvt->lvt_mode) {
504 case APIC_LVT_DM_NMI:
505 case APIC_LVT_DM_SMI:
506 case APIC_LVT_DM_INIT:
507 case APIC_LVT_DM_EXTINT:
508 if (!lvt->lvt_edgetrigger) {
509 if (bootverbose) {
510 printf(
511 "lapic%u: Forcing LINT%u to edge trigger\n",
512 la->la_id, pin);
513 }
514 value &= ~APIC_LVT_TM;
515 }
516 /* Use a vector of 0. */
517 break;
518 case APIC_LVT_DM_FIXED:
519 value |= lvt->lvt_vector;
520 break;
521 default:
522 panic("bad APIC LVT delivery mode: %#x\n", value);
523 }
524 return (value);
525 }
526
527 static uint32_t
lvt_mode(struct lapic * la,u_int pin,uint32_t value)528 lvt_mode(struct lapic *la, u_int pin, uint32_t value)
529 {
530 struct lvt *lvt;
531
532 KASSERT(pin <= APIC_LVT_MAX,
533 ("%s: pin %u out of range", __func__, pin));
534 if (la->la_lvts[pin].lvt_active)
535 lvt = &la->la_lvts[pin];
536 else
537 lvt = &lvts[pin];
538
539 return (lvt_mode_impl(la, lvt, pin, value));
540 }
541
542 static uint32_t
elvt_mode(struct lapic * la,u_int idx,uint32_t value)543 elvt_mode(struct lapic *la, u_int idx, uint32_t value)
544 {
545 struct lvt *elvt;
546
547 KASSERT(idx <= APIC_ELVT_MAX,
548 ("%s: idx %u out of range", __func__, idx));
549
550 if (la->la_elvts[idx].lvt_active)
551 elvt = &la->la_elvts[idx];
552 else
553 elvt = &elvts[idx];
554 KASSERT(elvt->lvt_active, ("%s: ELVT%u is not active", __func__, idx));
555 KASSERT(elvt->lvt_edgetrigger,
556 ("%s: ELVT%u is not edge triggered", __func__, idx));
557 KASSERT(elvt->lvt_activehi,
558 ("%s: ELVT%u is not active high", __func__, idx));
559 return (lvt_mode_impl(la, elvt, idx, value));
560 }
561
562 /*
563 * Map the local APIC and setup necessary interrupt vectors.
564 */
565 void
lapic_init(vm_paddr_t addr)566 lapic_init(vm_paddr_t addr)
567 {
568 #ifdef SMP
569 uint64_t r, r1, r2, rx;
570 #endif
571 uint32_t ver;
572 int i;
573 bool arat;
574
575 TSENTER();
576
577 /*
578 * Enable x2APIC mode if possible. Map the local APIC
579 * registers page.
580 *
581 * Keep the LAPIC registers page mapped uncached for x2APIC
582 * mode too, to have direct map page attribute set to
583 * uncached. This is needed to work around CPU errata present
584 * on all Intel processors.
585 */
586 KASSERT(trunc_page(addr) == addr,
587 ("local APIC not aligned on a page boundary"));
588 lapic_paddr = addr;
589 lapic_map = pmap_mapdev(addr, PAGE_SIZE);
590 if (x2apic_mode) {
591 lapic_enable_x2apic();
592 lapic_map = NULL;
593 }
594
595 /* Setup the spurious interrupt handler. */
596 setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
597 GSEL_APIC);
598
599 /* Perform basic initialization of the BSP's local APIC. */
600 lapic_enable();
601 lapic_early_mask_vecs();
602
603 /* Set BSP's per-CPU local APIC ID. */
604 PCPU_SET(apic_id, lapic_id());
605
606 /* Local APIC timer interrupt. */
607 setidt(APIC_TIMER_INT, pti ? IDTVEC(timerint_pti) : IDTVEC(timerint),
608 SDT_APIC, SEL_KPL, GSEL_APIC);
609
610 /* Local APIC error interrupt. */
611 setidt(APIC_ERROR_INT, pti ? IDTVEC(errorint_pti) : IDTVEC(errorint),
612 SDT_APIC, SEL_KPL, GSEL_APIC);
613
614 /* Thermal interrupt */
615 setidt(APIC_THERMAL_INT, pti ? IDTVEC(thermalint_pti) : IDTVEC(thermalint),
616 SDT_APIC, SEL_KPL, GSEL_APIC);
617
618 /* Local APIC CMCI. */
619 setidt(APIC_CMC_INT, pti ? IDTVEC(cmcint_pti) : IDTVEC(cmcint),
620 SDT_APIC, SEL_KPL, GSEL_APIC);
621
622 if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
623 /* Set if APIC timer runs in C3. */
624 arat = (cpu_power_eax & CPUTPM1_ARAT);
625
626 bzero(&lapic_et, sizeof(lapic_et));
627 lapic_et.et_name = "LAPIC";
628 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
629 ET_FLAGS_PERCPU;
630 lapic_et.et_quality = 600;
631 if (!arat) {
632 lapic_et.et_flags |= ET_FLAGS_C3STOP;
633 lapic_et.et_quality = 100;
634 }
635 if ((cpu_feature & CPUID_TSC) != 0 &&
636 (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
637 tsc_is_invariant && tsc_freq != 0) {
638 lapic_timer_tsc_deadline = 1;
639 TUNABLE_INT_FETCH("hw.apic.timer_tsc_deadline",
640 &lapic_timer_tsc_deadline);
641 }
642
643 lapic_et.et_frequency = 0;
644 /* We don't know frequency yet, so trying to guess. */
645 lapic_et.et_min_period = 0x00001000LL;
646 lapic_et.et_max_period = SBT_1S;
647 lapic_et.et_start = lapic_et_start;
648 lapic_et.et_stop = lapic_et_stop;
649 lapic_et.et_priv = NULL;
650 et_register(&lapic_et);
651 }
652
653 /*
654 * Set lapic_eoi_suppression after lapic_enable(), to not
655 * enable suppression in the hardware prematurely. Note that
656 * we by default enable suppression even when system only has
657 * one IO-APIC, since EOI is broadcasted to all APIC agents,
658 * including CPUs, otherwise.
659 *
660 * It seems that at least some KVM versions report
661 * EOI_SUPPRESSION bit, but auto-EOI does not work.
662 */
663 ver = lapic_version();
664 if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
665 lapic_eoi_suppression = 1;
666 if (vm_guest == VM_GUEST_KVM) {
667 if (bootverbose)
668 printf(
669 "KVM -- disabling lapic eoi suppression\n");
670 lapic_eoi_suppression = 0;
671 }
672 TUNABLE_INT_FETCH("hw.apic.eoi_suppression",
673 &lapic_eoi_suppression);
674 }
675
676 #ifdef SMP
677 #define LOOPS 1000
678 /*
679 * Calibrate the busy loop waiting for IPI ack in xAPIC mode.
680 * lapic_ipi_wait_mult contains the number of iterations which
681 * approximately delay execution for 1 microsecond (the
682 * argument to lapic_ipi_wait() is in microseconds).
683 *
684 * We assume that TSC is present and already measured.
685 * Possible TSC frequency jumps are irrelevant to the
686 * calibration loop below, the CPU clock management code is
687 * not yet started, and we do not enter sleep states.
688 */
689 KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0,
690 ("TSC not initialized"));
691 if (!x2apic_mode) {
692 r = rdtsc();
693 for (rx = 0; rx < LOOPS; rx++) {
694 (void)lapic_read_icr_lo();
695 ia32_pause();
696 }
697 r = rdtsc() - r;
698 r1 = tsc_freq * LOOPS;
699 r2 = r * 1000000;
700 lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1;
701 if (bootverbose) {
702 printf("LAPIC: ipi_wait() us multiplier %ju (r %ju "
703 "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult,
704 (uintmax_t)r, (uintmax_t)tsc_freq);
705 }
706 }
707 #undef LOOPS
708 #endif /* SMP */
709
710 TSEXIT();
711 }
712
713 /*
714 * Create a local APIC instance.
715 */
716 void
lapic_create(u_int apic_id,int boot_cpu)717 lapic_create(u_int apic_id, int boot_cpu)
718 {
719 int i;
720
721 if (apic_id > max_apic_id) {
722 printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
723 if (boot_cpu)
724 panic("Can't ignore BSP");
725 return;
726 }
727 KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
728 apic_id));
729
730 /*
731 * Assume no local LVT overrides and a cluster of 0 and
732 * intra-cluster ID of 0.
733 */
734 lapics[apic_id].la_present = 1;
735 lapics[apic_id].la_id = apic_id;
736 for (i = 0; i <= APIC_LVT_MAX; i++) {
737 lapics[apic_id].la_lvts[i] = lvts[i];
738 lapics[apic_id].la_lvts[i].lvt_active = 0;
739 }
740 for (i = 0; i <= APIC_ELVT_MAX; i++) {
741 lapics[apic_id].la_elvts[i] = elvts[i];
742 lapics[apic_id].la_elvts[i].lvt_active = 0;
743 }
744 for (i = 0; i <= APIC_NUM_IOINTS; i++)
745 lapics[apic_id].la_ioint_irqs[i] = IRQ_FREE;
746 lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
747 lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
748 IRQ_TIMER;
749 #ifdef KDTRACE_HOOKS
750 lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] =
751 IRQ_DTRACE_RET;
752 #endif
753 #ifdef XENHVM
754 lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN;
755 #endif
756
757 #ifdef SMP
758 cpu_add(apic_id, boot_cpu);
759 #endif
760 }
761
762 static inline uint32_t
amd_read_ext_features(void)763 amd_read_ext_features(void)
764 {
765 uint32_t version;
766
767 if (cpu_vendor_id != CPU_VENDOR_AMD &&
768 cpu_vendor_id != CPU_VENDOR_HYGON)
769 return (0);
770 version = lapic_version();
771 if ((version & APIC_VER_AMD_EXT_SPACE) != 0)
772 return (lapic_read32(LAPIC_EXT_FEATURES));
773 else
774 return (0);
775 }
776
777 static inline uint32_t
amd_read_elvt_count(void)778 amd_read_elvt_count(void)
779 {
780 uint32_t extf;
781 uint32_t count;
782
783 extf = amd_read_ext_features();
784 count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT;
785 count = min(count, APIC_ELVT_MAX + 1);
786 return (count);
787 }
788
789 /*
790 * Dump contents of local APIC registers
791 */
792 void
lapic_dump(const char * str)793 lapic_dump(const char* str)
794 {
795 const uint32_t version = lapic_version();
796 const int maxlvt = lapic_version_maxlvt(version);
797 uint32_t extf;
798 int elvt_count;
799 int i;
800
801 printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
802 printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x",
803 lapic_read32(LAPIC_ID), version,
804 lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR));
805 if ((cpu_feature2 & CPUID2_X2APIC) != 0)
806 printf(" x2APIC: %d", x2apic_mode);
807 printf("\n lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
808 lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1),
809 lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR));
810 printf(" timer: 0x%08x err: 0x%08x", lapic_read32(LAPIC_LVT_TIMER),
811 lapic_read32(LAPIC_LVT_ERROR));
812 if (maxlvt >= APIC_LVT_THERMAL)
813 printf(" therm: 0x%08x", lapic_read32(LAPIC_LVT_THERMAL));
814 if (maxlvt >= APIC_LVT_PMC)
815 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT));
816 printf("\n");
817 if (maxlvt >= APIC_LVT_CMCI)
818 printf(" cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI));
819 extf = amd_read_ext_features();
820 if (extf != 0) {
821 printf(" AMD ext features: 0x%08x", extf);
822 elvt_count = amd_read_elvt_count();
823 for (i = 0; i < elvt_count; i++)
824 printf("%s elvt%d: 0x%08x", (i % 4) ? "" : "\n ", i,
825 lapic_read32(LAPIC_EXT_LVT0 + i));
826 printf("\n");
827 }
828 }
829
830 void
lapic_xapic_mode(void)831 lapic_xapic_mode(void)
832 {
833 register_t saveintr;
834
835 saveintr = intr_disable();
836 if (x2apic_mode)
837 lapic_enable_x2apic();
838 intr_restore(saveintr);
839 }
840
841 static void
lapic_early_mask_vec(const struct lvt * l)842 lapic_early_mask_vec(const struct lvt *l)
843 {
844 uint32_t v;
845
846 if (l->lvt_masked != 0) {
847 v = lapic_read32(l->lvt_reg);
848 v |= APIC_LVT_M;
849 lapic_write32(l->lvt_reg, v);
850 }
851 }
852
853 /* Done on BSP only */
854 static void
lapic_early_mask_vecs(void)855 lapic_early_mask_vecs(void)
856 {
857 int elvt_count, lvts_count, i;
858
859 lvts_count = min(nitems(lvts), lapic_maxlvt() + 1);
860 for (i = 0; i < lvts_count; i++)
861 lapic_early_mask_vec(&lvts[i]);
862
863 elvt_count = amd_read_elvt_count();
864 for (i = 0; i < elvt_count; i++)
865 lapic_early_mask_vec(&elvts[i]);
866 }
867
868 void
lapic_setup(int boot)869 lapic_setup(int boot)
870 {
871 const uint32_t version = lapic_version();
872 const uint32_t maxlvt = lapic_version_maxlvt(version);
873 struct lapic *la;
874 register_t saveintr;
875 int elvt_count;
876 int i;
877
878 saveintr = intr_disable();
879
880 la = &lapics[lapic_id()];
881 KASSERT(la->la_present, ("missing APIC structure"));
882
883 /* Initialize the TPR to allow all interrupts. */
884 lapic_set_tpr(0);
885
886 /* Setup spurious vector and enable the local APIC. */
887 lapic_enable();
888
889 /* Program LINT[01] LVT entries. */
890 lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0,
891 lapic_read32(LAPIC_LVT_LINT0)));
892 lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1,
893 lapic_read32(LAPIC_LVT_LINT1)));
894
895 /* Program the PMC LVT entry if present. */
896 if (maxlvt >= APIC_LVT_PMC) {
897 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
898 LAPIC_LVT_PCINT));
899 }
900
901 /*
902 * Program the timer LVT. Calibration is deferred until it is certain
903 * that we have a reliable timecounter.
904 */
905 la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
906 lapic_read32(LAPIC_LVT_TIMER));
907 la->lvt_timer_last = la->lvt_timer_base;
908 lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
909
910 if (boot)
911 la->la_timer_mode = LAT_MODE_UNDEF;
912 else if (la->la_timer_mode != LAT_MODE_UNDEF) {
913 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
914 lapic_id()));
915 switch (la->la_timer_mode) {
916 case LAT_MODE_PERIODIC:
917 lapic_timer_set_divisor(lapic_timer_divisor);
918 lapic_timer_periodic(la);
919 break;
920 case LAT_MODE_ONESHOT:
921 lapic_timer_set_divisor(lapic_timer_divisor);
922 lapic_timer_oneshot(la);
923 break;
924 case LAT_MODE_DEADLINE:
925 lapic_timer_deadline(la);
926 break;
927 default:
928 panic("corrupted la_timer_mode %p %d", la,
929 la->la_timer_mode);
930 }
931 }
932
933 /* Program error LVT and clear any existing errors. */
934 lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR,
935 lapic_read32(LAPIC_LVT_ERROR)));
936 lapic_write32(LAPIC_ESR, 0);
937
938 /* Thermal LVT */
939 if (maxlvt >= APIC_LVT_THERMAL)
940 lapic_write32(LAPIC_LVT_THERMAL, lvt_mode(la, APIC_LVT_THERMAL,
941 lapic_read32(LAPIC_LVT_THERMAL)));
942
943 /* Program the CMCI LVT entry if present. */
944 if (maxlvt >= APIC_LVT_CMCI) {
945 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI,
946 lapic_read32(LAPIC_LVT_CMCI)));
947 }
948
949 elvt_count = amd_read_elvt_count();
950 for (i = 0; i < elvt_count; i++) {
951 if (la->la_elvts[i].lvt_active)
952 lapic_write32(LAPIC_EXT_LVT0 + i,
953 elvt_mode(la, i, lapic_read32(LAPIC_EXT_LVT0 + i)));
954 }
955
956 intr_restore(saveintr);
957 }
958
959 static void
lapic_intrcnt(void * dummy __unused)960 lapic_intrcnt(void *dummy __unused)
961 {
962 struct pcpu *pc;
963 struct lapic *la;
964 char buf[MAXCOMLEN + 1];
965
966 /* If there are no APICs, skip this function. */
967 if (lapics == NULL)
968 return;
969
970 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
971 la = &lapics[pc->pc_apic_id];
972 if (!la->la_present)
973 continue;
974
975 snprintf(buf, sizeof(buf), "cpu%d:timer", pc->pc_cpuid);
976 intrcnt_add(buf, &la->la_timer_count);
977 }
978 }
979 SYSINIT(lapic_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, lapic_intrcnt, NULL);
980
981 void
lapic_reenable_pcint(void)982 lapic_reenable_pcint(void)
983 {
984 uint32_t value;
985
986 if (refcount_load(&pcint_refcnt) == 0)
987 return;
988
989 value = lapic_read32(LAPIC_LVT_PCINT);
990 value &= ~APIC_LVT_M;
991 lapic_write32(LAPIC_LVT_PCINT, value);
992
993 if ((amd_feature2 & AMDID2_IBS) != 0) {
994 value = lapic_read32(LAPIC_EXT_LVT0);
995 value &= ~APIC_LVT_M;
996 lapic_write32(LAPIC_EXT_LVT0, value);
997 }
998 }
999
1000 static void
lapic_update_pcint(void * dummy)1001 lapic_update_pcint(void *dummy)
1002 {
1003 struct lapic *la;
1004
1005 la = &lapics[lapic_id()];
1006 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
1007 lapic_read32(LAPIC_LVT_PCINT)));
1008
1009 if ((amd_feature2 & AMDID2_IBS) != 0) {
1010 lapic_write32(LAPIC_EXT_LVT0, elvt_mode(la, APIC_ELVT_IBS,
1011 lapic_read32(LAPIC_EXT_LVT0)));
1012 }
1013 }
1014
1015 void
lapic_calibrate_timer(void)1016 lapic_calibrate_timer(void)
1017 {
1018 struct lapic *la;
1019 register_t intr;
1020
1021 #ifdef DEV_ATPIC
1022 /* Fail if the local APIC is not present. */
1023 if (!x2apic_mode && lapic_map == NULL)
1024 return;
1025 #endif
1026
1027 intr = intr_disable();
1028 la = &lapics[lapic_id()];
1029
1030 lapic_calibrate_initcount(la);
1031
1032 intr_restore(intr);
1033
1034 if (lapic_timer_tsc_deadline && bootverbose) {
1035 printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
1036 (uintmax_t)tsc_freq);
1037 }
1038 }
1039
1040 int
lapic_enable_pcint(void)1041 lapic_enable_pcint(void)
1042 {
1043 #ifdef DEV_ATPIC
1044 /* Fail if the local APIC is not present. */
1045 if (!x2apic_mode && lapic_map == NULL)
1046 return (0);
1047 #endif
1048
1049 /* Fail if the PMC LVT is not present. */
1050 if (lapic_maxlvt() < APIC_LVT_PMC)
1051 return (0);
1052 if (refcount_acquire(&pcint_refcnt) > 0)
1053 return (1);
1054 lvts[APIC_LVT_PMC].lvt_masked = 0;
1055
1056 if ((amd_feature2 & AMDID2_IBS) != 0)
1057 elvts[APIC_ELVT_IBS].lvt_masked = 0;
1058
1059 MPASS(mp_ncpus == 1 || smp_started);
1060 smp_rendezvous(NULL, lapic_update_pcint, NULL, NULL);
1061 return (1);
1062 }
1063
1064 void
lapic_disable_pcint(void)1065 lapic_disable_pcint(void)
1066 {
1067 #ifdef DEV_ATPIC
1068 /* Fail if the local APIC is not present. */
1069 if (!x2apic_mode && lapic_map == NULL)
1070 return;
1071 #endif
1072
1073 /* Fail if the PMC LVT is not present. */
1074 if (lapic_maxlvt() < APIC_LVT_PMC)
1075 return;
1076 if (!refcount_release(&pcint_refcnt))
1077 return;
1078 lvts[APIC_LVT_PMC].lvt_masked = 1;
1079 elvts[APIC_ELVT_IBS].lvt_masked = 1;
1080
1081 #ifdef SMP
1082 /* The APs should always be started when hwpmc is unloaded. */
1083 KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early"));
1084 #endif
1085 smp_rendezvous(NULL, lapic_update_pcint, NULL, NULL);
1086 }
1087
1088 static int
lapic_calibrate_initcount_cpuid_vm(void)1089 lapic_calibrate_initcount_cpuid_vm(void)
1090 {
1091 u_int regs[4];
1092 uint64_t freq;
1093
1094 /* Get value from CPUID leaf if possible. */
1095 if (vm_guest == VM_GUEST_NO)
1096 return (false);
1097 if (hv_high < 0x40000010)
1098 return (false);
1099 do_cpuid(0x40000010, regs);
1100 freq = (uint64_t)(regs[1]) * 1000;
1101
1102 /* Pick timer divisor. */
1103 lapic_timer_divisor = 2;
1104 do {
1105 if (freq / lapic_timer_divisor < APIC_TIMER_MAX_COUNT)
1106 break;
1107 lapic_timer_divisor <<= 1;
1108 } while (lapic_timer_divisor <= 128);
1109 if (lapic_timer_divisor > 128)
1110 return (false);
1111
1112 /* Record divided frequency. */
1113 count_freq = freq / lapic_timer_divisor;
1114 return (count_freq != 0);
1115 }
1116
1117 static uint64_t
cb_lapic_getcount(void)1118 cb_lapic_getcount(void)
1119 {
1120
1121 return (APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER));
1122 }
1123
1124 static void
lapic_calibrate_initcount(struct lapic * la)1125 lapic_calibrate_initcount(struct lapic *la)
1126 {
1127 uint64_t freq;
1128
1129 if (lapic_calibrate_initcount_cpuid_vm())
1130 goto done;
1131
1132 /* Calibrate the APIC timer frequency. */
1133 lapic_timer_set_divisor(2);
1134 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT);
1135 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
1136 freq = clockcalib(cb_lapic_getcount, "lapic");
1137 fpu_kern_leave(curthread, NULL);
1138
1139 /* Pick a different divisor if necessary. */
1140 lapic_timer_divisor = 2;
1141 do {
1142 if (freq * 2 / lapic_timer_divisor < APIC_TIMER_MAX_COUNT)
1143 break;
1144 lapic_timer_divisor <<= 1;
1145 } while (lapic_timer_divisor <= 128);
1146 if (lapic_timer_divisor > 128)
1147 panic("lapic: Divisor too big");
1148 count_freq = freq * 2 / lapic_timer_divisor;
1149 done:
1150 if (bootverbose) {
1151 printf("lapic: Divisor %lu, Frequency %lu Hz\n",
1152 lapic_timer_divisor, count_freq);
1153 }
1154 }
1155
1156 static void
lapic_change_mode(struct eventtimer * et,struct lapic * la,enum lat_timer_mode newmode)1157 lapic_change_mode(struct eventtimer *et, struct lapic *la,
1158 enum lat_timer_mode newmode)
1159 {
1160 if (la->la_timer_mode == newmode)
1161 return;
1162 switch (newmode) {
1163 case LAT_MODE_PERIODIC:
1164 lapic_timer_set_divisor(lapic_timer_divisor);
1165 et->et_frequency = count_freq;
1166 break;
1167 case LAT_MODE_DEADLINE:
1168 et->et_frequency = tsc_freq;
1169 break;
1170 case LAT_MODE_ONESHOT:
1171 lapic_timer_set_divisor(lapic_timer_divisor);
1172 et->et_frequency = count_freq;
1173 break;
1174 default:
1175 panic("lapic_change_mode %d", newmode);
1176 }
1177 la->la_timer_mode = newmode;
1178 et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
1179 et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
1180 }
1181
1182 static int
lapic_et_start(struct eventtimer * et,sbintime_t first,sbintime_t period)1183 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
1184 {
1185 struct lapic *la;
1186
1187 la = &lapics[PCPU_GET(apic_id)];
1188 if (period != 0) {
1189 lapic_change_mode(et, la, LAT_MODE_PERIODIC);
1190 la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
1191 32;
1192 lapic_timer_periodic(la);
1193 } else if (lapic_timer_tsc_deadline) {
1194 lapic_change_mode(et, la, LAT_MODE_DEADLINE);
1195 la->la_timer_period = (et->et_frequency * first) >> 32;
1196 lapic_timer_deadline(la);
1197 } else {
1198 lapic_change_mode(et, la, LAT_MODE_ONESHOT);
1199 la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
1200 32;
1201 lapic_timer_oneshot(la);
1202 }
1203 return (0);
1204 }
1205
1206 static int
lapic_et_stop(struct eventtimer * et)1207 lapic_et_stop(struct eventtimer *et)
1208 {
1209 struct lapic *la;
1210
1211 la = &lapics[PCPU_GET(apic_id)];
1212 lapic_timer_stop(la);
1213 la->la_timer_mode = LAT_MODE_UNDEF;
1214 return (0);
1215 }
1216
1217 void
lapic_disable(void)1218 lapic_disable(void)
1219 {
1220 uint32_t value;
1221
1222 /* Software disable the local APIC. */
1223 value = lapic_read32(LAPIC_SVR);
1224 value &= ~APIC_SVR_SWEN;
1225 lapic_write32(LAPIC_SVR, value);
1226 }
1227
1228 static void
lapic_enable(void)1229 lapic_enable(void)
1230 {
1231 uint32_t value;
1232
1233 /* Program the spurious vector to enable the local APIC. */
1234 value = lapic_read32(LAPIC_SVR);
1235 value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
1236 value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
1237 if (lapic_eoi_suppression)
1238 value |= APIC_SVR_EOI_SUPPRESSION;
1239 lapic_write32(LAPIC_SVR, value);
1240 }
1241
1242 /* Reset the local APIC on the BSP during resume. */
1243 static void
lapic_resume(struct pic * pic,bool suspend_cancelled)1244 lapic_resume(struct pic *pic, bool suspend_cancelled)
1245 {
1246
1247 lapic_setup(0);
1248 }
1249
1250 int
lapic_id(void)1251 lapic_id(void)
1252 {
1253 uint32_t v;
1254
1255 KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped"));
1256 v = lapic_read32(LAPIC_ID);
1257 if (!x2apic_mode)
1258 v >>= APIC_ID_SHIFT;
1259 return (v);
1260 }
1261
1262 int
lapic_intr_pending(u_int vector)1263 lapic_intr_pending(u_int vector)
1264 {
1265 uint32_t irr;
1266
1267 /*
1268 * The IRR registers are an array of registers each of which
1269 * only describes 32 interrupts in the low 32 bits. Thus, we
1270 * divide the vector by 32 to get the register index.
1271 * Finally, we modulus the vector by 32 to determine the
1272 * individual bit to test.
1273 */
1274 irr = lapic_read32(LAPIC_IRR0 + vector / 32);
1275 return (irr & 1 << (vector % 32));
1276 }
1277
1278 void
lapic_set_logical_id(u_int apic_id,u_int cluster,u_int cluster_id)1279 lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
1280 {
1281 struct lapic *la;
1282
1283 KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
1284 __func__, apic_id));
1285 KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
1286 __func__, cluster));
1287 KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
1288 ("%s: intra cluster id %u too big", __func__, cluster_id));
1289 la = &lapics[apic_id];
1290 la->la_cluster = cluster;
1291 la->la_cluster_id = cluster_id;
1292 }
1293
1294 int
lapic_set_lvt_mask(u_int apic_id,u_int pin,u_char masked)1295 lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
1296 {
1297
1298 if (pin > APIC_LVT_MAX)
1299 return (EINVAL);
1300 if (apic_id == APIC_ID_ALL) {
1301 lvts[pin].lvt_masked = masked;
1302 if (bootverbose)
1303 printf("lapic:");
1304 } else {
1305 KASSERT(lapics[apic_id].la_present,
1306 ("%s: missing APIC %u", __func__, apic_id));
1307 lapics[apic_id].la_lvts[pin].lvt_masked = masked;
1308 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1309 if (bootverbose)
1310 printf("lapic%u:", apic_id);
1311 }
1312 if (bootverbose)
1313 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
1314 return (0);
1315 }
1316
1317 int
lapic_set_lvt_mode(u_int apic_id,u_int pin,u_int32_t mode)1318 lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
1319 {
1320 struct lvt *lvt;
1321
1322 if (pin > APIC_LVT_MAX)
1323 return (EINVAL);
1324 if (apic_id == APIC_ID_ALL) {
1325 lvt = &lvts[pin];
1326 if (bootverbose)
1327 printf("lapic:");
1328 } else {
1329 KASSERT(lapics[apic_id].la_present,
1330 ("%s: missing APIC %u", __func__, apic_id));
1331 lvt = &lapics[apic_id].la_lvts[pin];
1332 lvt->lvt_active = 1;
1333 if (bootverbose)
1334 printf("lapic%u:", apic_id);
1335 }
1336 lvt->lvt_mode = mode;
1337 switch (mode) {
1338 case APIC_LVT_DM_NMI:
1339 case APIC_LVT_DM_SMI:
1340 case APIC_LVT_DM_INIT:
1341 case APIC_LVT_DM_EXTINT:
1342 lvt->lvt_edgetrigger = 1;
1343 lvt->lvt_activehi = 1;
1344 if (mode == APIC_LVT_DM_EXTINT)
1345 lvt->lvt_masked = 1;
1346 else
1347 lvt->lvt_masked = 0;
1348 break;
1349 default:
1350 panic("Unsupported delivery mode: 0x%x\n", mode);
1351 }
1352 if (bootverbose) {
1353 printf(" Routing ");
1354 switch (mode) {
1355 case APIC_LVT_DM_NMI:
1356 printf("NMI");
1357 break;
1358 case APIC_LVT_DM_SMI:
1359 printf("SMI");
1360 break;
1361 case APIC_LVT_DM_INIT:
1362 printf("INIT");
1363 break;
1364 case APIC_LVT_DM_EXTINT:
1365 printf("ExtINT");
1366 break;
1367 }
1368 printf(" -> LINT%u\n", pin);
1369 }
1370 return (0);
1371 }
1372
1373 int
lapic_set_lvt_polarity(u_int apic_id,u_int pin,enum intr_polarity pol)1374 lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol)
1375 {
1376
1377 if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM)
1378 return (EINVAL);
1379 if (apic_id == APIC_ID_ALL) {
1380 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH);
1381 if (bootverbose)
1382 printf("lapic:");
1383 } else {
1384 KASSERT(lapics[apic_id].la_present,
1385 ("%s: missing APIC %u", __func__, apic_id));
1386 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1387 lapics[apic_id].la_lvts[pin].lvt_activehi =
1388 (pol == INTR_POLARITY_HIGH);
1389 if (bootverbose)
1390 printf("lapic%u:", apic_id);
1391 }
1392 if (bootverbose)
1393 printf(" LINT%u polarity: %s\n", pin,
1394 pol == INTR_POLARITY_HIGH ? "high" : "low");
1395 return (0);
1396 }
1397
1398 int
lapic_set_lvt_triggermode(u_int apic_id,u_int pin,enum intr_trigger trigger)1399 lapic_set_lvt_triggermode(u_int apic_id, u_int pin,
1400 enum intr_trigger trigger)
1401 {
1402
1403 if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM)
1404 return (EINVAL);
1405 if (apic_id == APIC_ID_ALL) {
1406 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE);
1407 if (bootverbose)
1408 printf("lapic:");
1409 } else {
1410 KASSERT(lapics[apic_id].la_present,
1411 ("%s: missing APIC %u", __func__, apic_id));
1412 lapics[apic_id].la_lvts[pin].lvt_edgetrigger =
1413 (trigger == INTR_TRIGGER_EDGE);
1414 lapics[apic_id].la_lvts[pin].lvt_active = 1;
1415 if (bootverbose)
1416 printf("lapic%u:", apic_id);
1417 }
1418 if (bootverbose)
1419 printf(" LINT%u trigger: %s\n", pin,
1420 trigger == INTR_TRIGGER_EDGE ? "edge" : "level");
1421 return (0);
1422 }
1423
1424 /*
1425 * Adjust the TPR of the current CPU so that it blocks all interrupts below
1426 * the passed in vector.
1427 */
1428 static void
lapic_set_tpr(u_int vector)1429 lapic_set_tpr(u_int vector)
1430 {
1431 #ifdef CHEAP_TPR
1432 lapic_write32(LAPIC_TPR, vector);
1433 #else
1434 uint32_t tpr;
1435
1436 tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO;
1437 tpr |= vector;
1438 lapic_write32(LAPIC_TPR, tpr);
1439 #endif
1440 }
1441
1442 void
lapic_eoi(void)1443 lapic_eoi(void)
1444 {
1445
1446 lapic_write32_nofence(LAPIC_EOI, 0);
1447 }
1448
1449 void
lapic_handle_intr(int vector,struct trapframe * frame)1450 lapic_handle_intr(int vector, struct trapframe *frame)
1451 {
1452 struct intsrc *isrc;
1453
1454 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
1455 kmsan_mark(&vector, sizeof(vector), KMSAN_STATE_INITED);
1456 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
1457 trap_check_kstack();
1458
1459 isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
1460 vector));
1461 KASSERT(isrc != NULL,
1462 ("lapic_handle_intr: vector %d unrecognized at lapic %u",
1463 vector, PCPU_GET(apic_id)));
1464 intr_execute_handlers(isrc, frame);
1465 }
1466
1467 void
lapic_handle_timer(struct trapframe * frame)1468 lapic_handle_timer(struct trapframe *frame)
1469 {
1470 struct lapic *la;
1471 struct trapframe *oldframe;
1472 struct thread *td;
1473
1474 /* Send EOI first thing. */
1475 lapic_eoi();
1476
1477 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
1478 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
1479 trap_check_kstack();
1480
1481 /* Look up our local APIC structure for the tick counters. */
1482 la = &lapics[PCPU_GET(apic_id)];
1483 (*la->la_timer_count)++;
1484 critical_enter();
1485 if (lapic_et.et_active) {
1486 td = curthread;
1487 td->td_intr_nesting_level++;
1488 oldframe = td->td_intr_frame;
1489 td->td_intr_frame = frame;
1490 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg);
1491 td->td_intr_frame = oldframe;
1492 td->td_intr_nesting_level--;
1493 }
1494 critical_exit();
1495 }
1496
1497 static void
lapic_timer_set_divisor(u_int divisor)1498 lapic_timer_set_divisor(u_int divisor)
1499 {
1500
1501 KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor));
1502 KASSERT(ffs(divisor) <= nitems(lapic_timer_divisors),
1503 ("lapic: invalid divisor %u", divisor));
1504 lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]);
1505 }
1506
1507 static void
lapic_timer_oneshot(struct lapic * la)1508 lapic_timer_oneshot(struct lapic *la)
1509 {
1510 uint32_t value;
1511
1512 value = la->lvt_timer_base;
1513 value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1514 value |= APIC_LVTT_TM_ONE_SHOT;
1515 la->lvt_timer_last = value;
1516 lapic_write32(LAPIC_LVT_TIMER, value);
1517 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1518 }
1519
1520 static void
lapic_timer_oneshot_nointr(struct lapic * la,uint32_t count)1521 lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count)
1522 {
1523 uint32_t value;
1524
1525 value = la->lvt_timer_base;
1526 value &= ~APIC_LVTT_TM;
1527 value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M;
1528 la->lvt_timer_last = value;
1529 lapic_write32(LAPIC_LVT_TIMER, value);
1530 lapic_write32(LAPIC_ICR_TIMER, count);
1531 }
1532
1533 static void
lapic_timer_periodic(struct lapic * la)1534 lapic_timer_periodic(struct lapic *la)
1535 {
1536 uint32_t value;
1537
1538 value = la->lvt_timer_base;
1539 value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1540 value |= APIC_LVTT_TM_PERIODIC;
1541 la->lvt_timer_last = value;
1542 lapic_write32(LAPIC_LVT_TIMER, value);
1543 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period);
1544 }
1545
1546 static void
lapic_timer_deadline(struct lapic * la)1547 lapic_timer_deadline(struct lapic *la)
1548 {
1549 uint32_t value;
1550
1551 value = la->lvt_timer_base;
1552 value &= ~(APIC_LVTT_TM | APIC_LVT_M);
1553 value |= APIC_LVTT_TM_TSCDLT;
1554 if (value != la->lvt_timer_last) {
1555 la->lvt_timer_last = value;
1556 lapic_write32_nofence(LAPIC_LVT_TIMER, value);
1557 if (!x2apic_mode)
1558 mfence();
1559 }
1560 wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc());
1561 }
1562
1563 static void
lapic_timer_stop(struct lapic * la)1564 lapic_timer_stop(struct lapic *la)
1565 {
1566 uint32_t value;
1567
1568 if (la->la_timer_mode == LAT_MODE_DEADLINE) {
1569 wrmsr(MSR_TSC_DEADLINE, 0);
1570 mfence();
1571 } else {
1572 value = la->lvt_timer_base;
1573 value &= ~APIC_LVTT_TM;
1574 value |= APIC_LVT_M;
1575 la->lvt_timer_last = value;
1576 lapic_write32(LAPIC_LVT_TIMER, value);
1577 }
1578 }
1579
1580 void
lapic_handle_cmc(void)1581 lapic_handle_cmc(void)
1582 {
1583 trap_check_kstack();
1584
1585 lapic_eoi();
1586 cmc_intr();
1587 }
1588
1589 /*
1590 * Called from the mca_init() to activate the CMC interrupt if this CPU is
1591 * responsible for monitoring any MC banks for CMC events. Since mca_init()
1592 * is called prior to lapic_setup() during boot, this just needs to unmask
1593 * this CPU's LVT_CMCI entry.
1594 */
1595 void
lapic_enable_cmc(void)1596 lapic_enable_cmc(void)
1597 {
1598 u_int apic_id;
1599
1600 #ifdef DEV_ATPIC
1601 if (!x2apic_mode && lapic_map == NULL)
1602 return;
1603 #endif
1604 apic_id = PCPU_GET(apic_id);
1605 KASSERT(lapics[apic_id].la_present,
1606 ("%s: missing APIC %u", __func__, apic_id));
1607 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0;
1608 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1;
1609 }
1610
1611 int
lapic_enable_mca_elvt(void)1612 lapic_enable_mca_elvt(void)
1613 {
1614 u_int apic_id;
1615 uint32_t value;
1616 int elvt_count;
1617
1618 #ifdef DEV_ATPIC
1619 if (lapic_map == NULL)
1620 return (-1);
1621 #endif
1622
1623 apic_id = PCPU_GET(apic_id);
1624 KASSERT(lapics[apic_id].la_present,
1625 ("%s: missing APIC %u", __func__, apic_id));
1626 elvt_count = amd_read_elvt_count();
1627 if (elvt_count <= APIC_ELVT_MCA)
1628 return (-1);
1629
1630 value = lapic_read32(LAPIC_EXT_LVT0 + APIC_ELVT_MCA);
1631 if ((value & APIC_LVT_M) == 0) {
1632 if (bootverbose)
1633 printf("AMD MCE Thresholding Extended LVT is already active\n");
1634 return (APIC_ELVT_MCA);
1635 }
1636 lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_masked = 0;
1637 lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_active = 1;
1638 return (APIC_ELVT_MCA);
1639 }
1640
1641 void
lapic_handle_thermal(void)1642 lapic_handle_thermal(void)
1643 {
1644 lapic_thermal_handler_t *func;
1645
1646 func = (lapic_thermal_handler_t *)atomic_load_acq_ptr(
1647 (uintptr_t *)&lapic_thermal_function);
1648
1649 if (func != NULL)
1650 func(PCPU_GET(cpuid), lapic_thermal_function_arg);
1651
1652 lapic_eoi();
1653 }
1654
1655 static void
lapic_update_thermal(void * dummy __unused)1656 lapic_update_thermal(void *dummy __unused)
1657 {
1658 struct lapic *la;
1659
1660 la = &lapics[lapic_id()];
1661 lapic_write32(LAPIC_LVT_THERMAL, lvt_mode(la, APIC_LVT_THERMAL,
1662 lapic_read32(LAPIC_LVT_THERMAL)));
1663 }
1664
1665 bool
lapic_enable_thermal(lapic_thermal_handler_t * func,void * func_arg)1666 lapic_enable_thermal(lapic_thermal_handler_t *func, void *func_arg)
1667 {
1668 #ifdef DEV_ATPIC
1669 /* Fail if the local APIC is not present. */
1670 if (!x2apic_mode && lapic_map == NULL)
1671 return (false);
1672 #endif
1673
1674 if (lapic_maxlvt() < APIC_LVT_THERMAL)
1675 return (false);
1676
1677 lapic_thermal_function_arg = func_arg;
1678 atomic_store_rel_ptr((uintptr_t *)&lapic_thermal_function,
1679 (uintptr_t)func);
1680
1681 lvts[APIC_LVT_THERMAL].lvt_masked = 0;
1682
1683 MPASS(mp_ncpus == 1 || smp_started);
1684 smp_rendezvous(NULL, lapic_update_thermal, NULL, NULL);
1685
1686 return (true);
1687 }
1688
1689 void
lapic_disable_thermal(void)1690 lapic_disable_thermal(void)
1691 {
1692 #ifdef DEV_ATPIC
1693 /* Fail if the local APIC is not present. */
1694 if (!x2apic_mode && lapic_map == NULL)
1695 return;
1696 #endif
1697
1698 if (lapic_maxlvt() < APIC_LVT_THERMAL)
1699 return;
1700
1701 lvts[APIC_LVT_THERMAL].lvt_masked = 1;
1702
1703 #ifdef SMP
1704 KASSERT(mp_ncpus == 1 || smp_started, ("thermal driver unloaded too early"));
1705 #endif
1706 smp_rendezvous(NULL, lapic_update_thermal, NULL, NULL);
1707
1708 atomic_store_rel_ptr((uintptr_t *)&lapic_thermal_function,
1709 (uintptr_t)NULL);
1710 }
1711
1712 void
lapic_handle_error(void)1713 lapic_handle_error(void)
1714 {
1715 uint32_t esr;
1716
1717 trap_check_kstack();
1718
1719 /*
1720 * Read the contents of the error status register. Write to
1721 * the register first before reading from it to force the APIC
1722 * to update its value to indicate any errors that have
1723 * occurred since the previous write to the register.
1724 */
1725 lapic_write32(LAPIC_ESR, 0);
1726 esr = lapic_read32(LAPIC_ESR);
1727
1728 printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr);
1729 lapic_eoi();
1730 }
1731
1732 u_int
apic_cpuid(u_int apic_id)1733 apic_cpuid(u_int apic_id)
1734 {
1735 #ifdef SMP
1736 return apic_cpuids[apic_id];
1737 #else
1738 return 0;
1739 #endif
1740 }
1741
1742 /* Request a free IDT vector to be used by the specified IRQ. */
1743 u_int
apic_alloc_vector(u_int apic_id,u_int irq)1744 apic_alloc_vector(u_int apic_id, u_int irq)
1745 {
1746 u_int vector;
1747
1748 KASSERT(irq < num_io_irqs, ("Invalid IRQ %u", irq));
1749
1750 /*
1751 * Search for a free vector. Currently we just use a very simple
1752 * algorithm to find the first free vector.
1753 */
1754 mtx_lock_spin(&icu_lock);
1755 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1756 if (lapics[apic_id].la_ioint_irqs[vector] != IRQ_FREE)
1757 continue;
1758 lapics[apic_id].la_ioint_irqs[vector] = irq;
1759 mtx_unlock_spin(&icu_lock);
1760 return (vector + APIC_IO_INTS);
1761 }
1762 mtx_unlock_spin(&icu_lock);
1763 return (0);
1764 }
1765
1766 /*
1767 * Request 'count' free contiguous IDT vectors to be used by 'count'
1768 * IRQs. 'count' must be a power of two and the vectors will be
1769 * aligned on a boundary of 'align'. If the request cannot be
1770 * satisfied, 0 is returned.
1771 */
1772 u_int
apic_alloc_vectors(u_int apic_id,u_int * irqs,u_int count,u_int align)1773 apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
1774 {
1775 u_int first, run, vector;
1776
1777 KASSERT(powerof2(count), ("bad count"));
1778 KASSERT(powerof2(align), ("bad align"));
1779 KASSERT(align >= count, ("align < count"));
1780 #ifdef INVARIANTS
1781 for (run = 0; run < count; run++)
1782 KASSERT(irqs[run] < num_io_irqs, ("Invalid IRQ %u at index %u",
1783 irqs[run], run));
1784 #endif
1785
1786 /*
1787 * Search for 'count' free vectors. As with apic_alloc_vector(),
1788 * this just uses a simple first fit algorithm.
1789 */
1790 run = 0;
1791 first = 0;
1792 mtx_lock_spin(&icu_lock);
1793 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
1794 /* Vector is in use, end run. */
1795 if (lapics[apic_id].la_ioint_irqs[vector] != IRQ_FREE) {
1796 run = 0;
1797 first = 0;
1798 continue;
1799 }
1800
1801 /* Start a new run if run == 0 and vector is aligned. */
1802 if (run == 0) {
1803 if (((vector + APIC_IO_INTS) & (align - 1)) != 0)
1804 continue;
1805 first = vector;
1806 }
1807 run++;
1808
1809 /* Keep looping if the run isn't long enough yet. */
1810 if (run < count)
1811 continue;
1812
1813 /* Found a run, assign IRQs and return the first vector. */
1814 for (vector = 0; vector < count; vector++)
1815 lapics[apic_id].la_ioint_irqs[first + vector] =
1816 irqs[vector];
1817 mtx_unlock_spin(&icu_lock);
1818 return (first + APIC_IO_INTS);
1819 }
1820 mtx_unlock_spin(&icu_lock);
1821 printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count);
1822 return (0);
1823 }
1824
1825 /*
1826 * Enable a vector for a particular apic_id. Since all lapics share idt
1827 * entries and ioint_handlers this enables the vector on all lapics. lapics
1828 * which do not have the vector configured would report spurious interrupts
1829 * should it fire.
1830 */
1831 void
apic_enable_vector(u_int apic_id,u_int vector)1832 apic_enable_vector(u_int apic_id, u_int vector)
1833 {
1834
1835 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1836 KASSERT(ioint_handlers[vector / 32] != NULL,
1837 ("No ISR handler for vector %u", vector));
1838 #ifdef KDTRACE_HOOKS
1839 KASSERT(vector != IDT_DTRACE_RET,
1840 ("Attempt to overwrite DTrace entry"));
1841 #endif
1842 setidt(vector, (pti ? ioint_pti_handlers : ioint_handlers)[vector / 32],
1843 SDT_APIC, SEL_KPL, GSEL_APIC);
1844 }
1845
1846 void
apic_disable_vector(u_int apic_id,u_int vector)1847 apic_disable_vector(u_int apic_id, u_int vector)
1848 {
1849
1850 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
1851 #ifdef KDTRACE_HOOKS
1852 KASSERT(vector != IDT_DTRACE_RET,
1853 ("Attempt to overwrite DTrace entry"));
1854 #endif
1855 KASSERT(ioint_handlers[vector / 32] != NULL,
1856 ("No ISR handler for vector %u", vector));
1857 #ifdef notyet
1858 /*
1859 * We can not currently clear the idt entry because other cpus
1860 * may have a valid vector at this offset.
1861 */
1862 setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_APIC,
1863 SEL_KPL, GSEL_APIC);
1864 #endif
1865 }
1866
1867 /* Release an APIC vector when it's no longer in use. */
1868 void
apic_free_vector(u_int apic_id,u_int vector,u_int irq)1869 apic_free_vector(u_int apic_id, u_int vector, u_int irq)
1870 {
1871 struct thread *td;
1872
1873 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1874 vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1875 ("Vector %u does not map to an IRQ line", vector));
1876 KASSERT(irq < num_io_irqs, ("Invalid IRQ %u", irq));
1877 KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] ==
1878 irq, ("IRQ mismatch"));
1879 #ifdef KDTRACE_HOOKS
1880 KASSERT(vector != IDT_DTRACE_RET,
1881 ("Attempt to overwrite DTrace entry"));
1882 #endif
1883
1884 /*
1885 * Bind us to the cpu that owned the vector before freeing it so
1886 * we don't lose an interrupt delivery race.
1887 */
1888 td = curthread;
1889 if (!rebooting) {
1890 thread_lock(td);
1891 if (sched_is_bound(td))
1892 panic("apic_free_vector: Thread already bound.\n");
1893 sched_bind(td, apic_cpuid(apic_id));
1894 thread_unlock(td);
1895 }
1896 mtx_lock_spin(&icu_lock);
1897 lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = IRQ_FREE;
1898 mtx_unlock_spin(&icu_lock);
1899 if (!rebooting) {
1900 thread_lock(td);
1901 sched_unbind(td);
1902 thread_unlock(td);
1903 }
1904 }
1905
1906 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */
1907 static u_int
apic_idt_to_irq(u_int apic_id,u_int vector)1908 apic_idt_to_irq(u_int apic_id, u_int vector)
1909 {
1910 int irq;
1911
1912 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
1913 vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
1914 ("Vector %u does not map to an IRQ line", vector));
1915 #ifdef KDTRACE_HOOKS
1916 KASSERT(vector != IDT_DTRACE_RET,
1917 ("Attempt to overwrite DTrace entry"));
1918 #endif
1919 irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS];
1920 if (irq < 0)
1921 irq = 0;
1922 return (irq);
1923 }
1924
1925 #ifdef DDB
1926 /*
1927 * Dump data about APIC IDT vector mappings.
1928 */
DB_SHOW_COMMAND_FLAGS(apic,db_show_apic,DB_CMD_MEMSAFE)1929 DB_SHOW_COMMAND_FLAGS(apic, db_show_apic, DB_CMD_MEMSAFE)
1930 {
1931 struct intsrc *isrc;
1932 int i, verbose;
1933 u_int apic_id;
1934 u_int irq;
1935
1936 if (strcmp(modif, "vv") == 0)
1937 verbose = 2;
1938 else if (strcmp(modif, "v") == 0)
1939 verbose = 1;
1940 else
1941 verbose = 0;
1942 for (apic_id = 0; apic_id <= max_apic_id; apic_id++) {
1943 if (lapics[apic_id].la_present == 0)
1944 continue;
1945 db_printf("Interrupts bound to lapic %u\n", apic_id);
1946 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
1947 irq = lapics[apic_id].la_ioint_irqs[i];
1948 if (irq == IRQ_FREE || irq == IRQ_SYSCALL)
1949 continue;
1950 #ifdef KDTRACE_HOOKS
1951 if (irq == IRQ_DTRACE_RET)
1952 continue;
1953 #endif
1954 #ifdef XENHVM
1955 if (irq == IRQ_EVTCHN)
1956 continue;
1957 #endif
1958 db_printf("vec 0x%2x -> ", i + APIC_IO_INTS);
1959 if (irq == IRQ_TIMER)
1960 db_printf("lapic timer\n");
1961 else if (irq < num_io_irqs) {
1962 isrc = intr_lookup_source(irq);
1963 if (isrc == NULL || verbose == 0)
1964 db_printf("IRQ %u\n", irq);
1965 else
1966 db_dump_intr_event(isrc->is_event,
1967 verbose == 2);
1968 } else
1969 db_printf("IRQ %u ???\n", irq);
1970 }
1971 }
1972 }
1973
1974 static void
dump_mask(const char * prefix,uint32_t v,int base)1975 dump_mask(const char *prefix, uint32_t v, int base)
1976 {
1977 int i, first;
1978
1979 first = 1;
1980 for (i = 0; i < 32; i++)
1981 if (v & (1 << i)) {
1982 if (first) {
1983 db_printf("%s:", prefix);
1984 first = 0;
1985 }
1986 db_printf(" %02x", base + i);
1987 }
1988 if (!first)
1989 db_printf("\n");
1990 }
1991
1992 /* Show info from the lapic regs for this CPU. */
DB_SHOW_COMMAND_FLAGS(lapic,db_show_lapic,DB_CMD_MEMSAFE)1993 DB_SHOW_COMMAND_FLAGS(lapic, db_show_lapic, DB_CMD_MEMSAFE)
1994 {
1995 const struct lvt *l;
1996 int elvt_count, lvts_count, i;
1997 const uint32_t v = lapic_version();
1998 const int maxlvt = lapic_version_maxlvt(v);
1999 const uint32_t vr = lapic_read32(LAPIC_SVR);
2000
2001 db_printf("lapic ID = %d\n", lapic_id());
2002 db_printf("version = %d.%d (%#x) \n", (v & APIC_VER_VERSION) >> 4,
2003 v & 0xf, v);
2004 db_printf("max LVT = %d\n", maxlvt);
2005 db_printf("SVR = %02x (%s)\n", vr & APIC_SVR_VECTOR,
2006 vr & APIC_SVR_ENABLE ? "enabled" : "disabled");
2007 db_printf("TPR = %02x\n", lapic_read32(LAPIC_TPR));
2008
2009 lvts_count = min(nitems(lvts), maxlvt + 1);
2010 for (i = 0; i < lvts_count; i++) {
2011 l = &lvts[i];
2012 db_printf("LVT%d (reg %#x %-5s) = %#010x\n", i, l->lvt_reg,
2013 l->lvt_desc, lapic_read32(l->lvt_reg));
2014 }
2015
2016 elvt_count = amd_read_elvt_count();
2017 for (i = 0; i < elvt_count; i++) {
2018 l = &elvts[i];
2019 db_printf("ELVT%d (reg %#x %-5s) = %#010x\n", i, l->lvt_reg,
2020 l->lvt_desc, lapic_read32(l->lvt_reg));
2021 }
2022
2023 #define dump_field(prefix, regn, index) \
2024 dump_mask(__XSTRING(prefix ## index), \
2025 lapic_read32(LAPIC_ ## regn ## index), \
2026 index * 32)
2027
2028 db_printf("In-service Interrupts:\n");
2029 dump_field(isr, ISR, 0);
2030 dump_field(isr, ISR, 1);
2031 dump_field(isr, ISR, 2);
2032 dump_field(isr, ISR, 3);
2033 dump_field(isr, ISR, 4);
2034 dump_field(isr, ISR, 5);
2035 dump_field(isr, ISR, 6);
2036 dump_field(isr, ISR, 7);
2037
2038 db_printf("TMR Interrupts:\n");
2039 dump_field(tmr, TMR, 0);
2040 dump_field(tmr, TMR, 1);
2041 dump_field(tmr, TMR, 2);
2042 dump_field(tmr, TMR, 3);
2043 dump_field(tmr, TMR, 4);
2044 dump_field(tmr, TMR, 5);
2045 dump_field(tmr, TMR, 6);
2046 dump_field(tmr, TMR, 7);
2047
2048 db_printf("IRR Interrupts:\n");
2049 dump_field(irr, IRR, 0);
2050 dump_field(irr, IRR, 1);
2051 dump_field(irr, IRR, 2);
2052 dump_field(irr, IRR, 3);
2053 dump_field(irr, IRR, 4);
2054 dump_field(irr, IRR, 5);
2055 dump_field(irr, IRR, 6);
2056 dump_field(irr, IRR, 7);
2057
2058 #undef dump_field
2059 }
2060 #endif
2061
2062 /*
2063 * APIC probing support code. This includes code to manage enumerators.
2064 */
2065
2066 static SLIST_HEAD(, apic_enumerator) enumerators =
2067 SLIST_HEAD_INITIALIZER(enumerators);
2068 static struct apic_enumerator *best_enum;
2069
2070 void
apic_register_enumerator(struct apic_enumerator * enumerator)2071 apic_register_enumerator(struct apic_enumerator *enumerator)
2072 {
2073 #ifdef INVARIANTS
2074 struct apic_enumerator *apic_enum;
2075
2076 SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
2077 if (apic_enum == enumerator)
2078 panic("%s: Duplicate register of %s", __func__,
2079 enumerator->apic_name);
2080 }
2081 #endif
2082 SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
2083 }
2084
2085 /*
2086 * We have to look for CPU's very, very early because certain subsystems
2087 * want to know how many CPU's we have extremely early on in the boot
2088 * process.
2089 */
2090 static void
apic_init(void * dummy __unused)2091 apic_init(void *dummy __unused)
2092 {
2093 struct apic_enumerator *enumerator;
2094 int retval, best;
2095
2096 /* We only support built in local APICs. */
2097 if (!(cpu_feature & CPUID_APIC))
2098 return;
2099
2100 /* Don't probe if APIC mode is disabled. */
2101 if (resource_disabled("apic", 0))
2102 return;
2103
2104 /* Probe all the enumerators to find the best match. */
2105 best_enum = NULL;
2106 best = 0;
2107 SLIST_FOREACH(enumerator, &enumerators, apic_next) {
2108 retval = enumerator->apic_probe();
2109 if (retval > 0)
2110 continue;
2111 if (best_enum == NULL || best < retval) {
2112 best_enum = enumerator;
2113 best = retval;
2114 }
2115 }
2116 if (best_enum == NULL) {
2117 if (bootverbose)
2118 printf("APIC: Could not find any APICs.\n");
2119 #ifndef DEV_ATPIC
2120 panic("running without device atpic requires a local APIC");
2121 #endif
2122 return;
2123 }
2124
2125 if (bootverbose)
2126 printf("APIC: Using the %s enumerator.\n",
2127 best_enum->apic_name);
2128
2129 #ifdef I686_CPU
2130 /*
2131 * To work around an errata, we disable the local APIC on some
2132 * CPUs during early startup. We need to turn the local APIC back
2133 * on on such CPUs now.
2134 */
2135 ppro_reenable_apic();
2136 #endif
2137
2138 /* Probe the CPU's in the system. */
2139 retval = best_enum->apic_probe_cpus();
2140 if (retval != 0)
2141 printf("%s: Failed to probe CPUs: returned %d\n",
2142 best_enum->apic_name, retval);
2143
2144 }
2145 SYSINIT(apic_init, SI_SUB_FIRST, SI_ORDER_SECOND, apic_init, NULL);
2146
2147 /*
2148 * Setup the local APIC. We have to do this prior to starting up the APs
2149 * in the SMP case.
2150 */
2151 static void
apic_setup_local(void * dummy __unused)2152 apic_setup_local(void *dummy __unused)
2153 {
2154 int retval;
2155
2156 if (best_enum == NULL)
2157 return;
2158
2159 lapics = malloc(sizeof(*lapics) * (max_apic_id + 1), M_LAPIC,
2160 M_WAITOK | M_ZERO);
2161
2162 /* Initialize the local APIC. */
2163 retval = best_enum->apic_setup_local();
2164 if (retval != 0)
2165 printf("%s: Failed to setup the local APIC: returned %d\n",
2166 best_enum->apic_name, retval);
2167 }
2168 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL);
2169
2170 /* Are we in a VM which supports the Extended Destination ID standard? */
2171 int apic_ext_dest_id = -1;
2172 SYSCTL_INT(_machdep, OID_AUTO, apic_ext_dest_id, CTLFLAG_RDTUN, &apic_ext_dest_id, 0,
2173 "Use APIC Extended Destination IDs");
2174
2175 /* Detect support for Extended Destination IDs. */
2176 static void
detect_extended_dest_id(void)2177 detect_extended_dest_id(void)
2178 {
2179 u_int regs[4];
2180
2181 /* Check if we support extended destination IDs. */
2182 switch (vm_guest) {
2183 case VM_GUEST_XEN:
2184 cpuid_count(hv_base + 4, 0, regs);
2185 if (regs[0] & XEN_HVM_CPUID_EXT_DEST_ID)
2186 apic_ext_dest_id = 1;
2187 break;
2188 case VM_GUEST_HV:
2189 cpuid_count(CPUID_LEAF_HV_STACK_INTERFACE, 0, regs);
2190 if (regs[0] != HYPERV_STACK_INTERFACE_EAX_SIG)
2191 break;
2192 cpuid_count(CPUID_LEAF_HV_STACK_PROPERTIES, 0, regs);
2193 if (regs[0] & HYPERV_PROPERTIES_EXT_DEST_ID)
2194 apic_ext_dest_id = 1;
2195 break;
2196 case VM_GUEST_KVM:
2197 kvm_cpuid_get_features(regs);
2198 if (regs[0] & KVM_FEATURE_MSI_EXT_DEST_ID)
2199 apic_ext_dest_id = 1;
2200 break;
2201 case VM_GUEST_BHYVE:
2202 if (hv_high < CPUID_BHYVE_FEATURES)
2203 break;
2204 cpuid_count(CPUID_BHYVE_FEATURES, 0, regs);
2205 if (regs[0] & CPUID_BHYVE_FEAT_EXT_DEST_ID)
2206 apic_ext_dest_id = 1;
2207 break;
2208 }
2209 }
2210
2211 /*
2212 * Setup the I/O APICs.
2213 */
2214 static void
apic_setup_io(void * dummy __unused)2215 apic_setup_io(void *dummy __unused)
2216 {
2217 int retval;
2218
2219 if (best_enum == NULL)
2220 return;
2221
2222 /* Check hypervisor support for extended destination IDs. */
2223 if (apic_ext_dest_id == -1)
2224 detect_extended_dest_id();
2225
2226 /*
2227 * Local APIC must be registered before other PICs and pseudo PICs
2228 * for proper suspend/resume order.
2229 */
2230 intr_register_pic(&lapic_pic);
2231
2232 retval = best_enum->apic_setup_io();
2233 if (retval != 0)
2234 printf("%s: Failed to setup I/O APICs: returned %d\n",
2235 best_enum->apic_name, retval);
2236
2237 /*
2238 * Finish setting up the local APIC on the BSP once we know
2239 * how to properly program the LINT pins. In particular, this
2240 * enables the EOI suppression mode, if LAPIC supports it and
2241 * user did not disable the mode.
2242 */
2243 lapic_setup(1);
2244 if (bootverbose)
2245 lapic_dump("BSP");
2246
2247 /* Enable the MSI "pic". */
2248 msi_init();
2249
2250 #ifdef XENHVM
2251 xen_intr_alloc_irqs();
2252 #endif
2253 }
2254 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL);
2255
2256 #ifdef SMP
2257 /*
2258 * Inter Processor Interrupt functions. The lapic_ipi_*() functions are
2259 * private to the MD code. The public interface for the rest of the
2260 * kernel is defined in mp_machdep.c.
2261 */
2262
2263 /*
2264 * Wait delay microseconds for IPI to be sent. If delay is -1, we
2265 * wait forever.
2266 */
2267 int
lapic_ipi_wait(int delay)2268 lapic_ipi_wait(int delay)
2269 {
2270 uint64_t rx;
2271
2272 /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */
2273 if (x2apic_mode)
2274 return (1);
2275
2276 for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) {
2277 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) ==
2278 APIC_DELSTAT_IDLE)
2279 return (1);
2280 ia32_pause();
2281 }
2282 return (0);
2283 }
2284
2285 void
lapic_ipi_raw(register_t icrlo,u_int dest)2286 lapic_ipi_raw(register_t icrlo, u_int dest)
2287 {
2288 uint32_t icrhi;
2289
2290 /* XXX: Need more sanity checking of icrlo? */
2291 KASSERT(x2apic_mode || lapic_map != NULL,
2292 ("%s called too early", __func__));
2293 KASSERT(x2apic_mode ||
2294 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
2295 ("%s: invalid dest field", __func__));
2296 KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
2297 ("%s: reserved bits set in ICR LO register", __func__));
2298
2299 if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
2300 if (x2apic_mode)
2301 icrhi = dest;
2302 else
2303 icrhi = dest << APIC_ID_SHIFT;
2304 lapic_write_icr(icrhi, icrlo);
2305 } else {
2306 lapic_write_icr_lo(icrlo);
2307 }
2308 }
2309
2310 #ifdef DETECT_DEADLOCK
2311 #define AFTER_SPIN 50
2312 #endif
2313
2314 static void
native_lapic_ipi_vectored(u_int vector,int dest)2315 native_lapic_ipi_vectored(u_int vector, int dest)
2316 {
2317 register_t icrlo, destfield;
2318
2319 KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
2320 ("%s: invalid vector %d", __func__, vector));
2321
2322 destfield = 0;
2323 switch (dest) {
2324 case APIC_IPI_DEST_SELF:
2325 if (x2apic_mode && vector < IPI_NMI_FIRST) {
2326 lapic_write_self_ipi(vector);
2327 return;
2328 }
2329 icrlo = APIC_DEST_SELF;
2330 break;
2331 case APIC_IPI_DEST_ALL:
2332 icrlo = APIC_DEST_ALLISELF;
2333 break;
2334 case APIC_IPI_DEST_OTHERS:
2335 icrlo = APIC_DEST_ALLESELF;
2336 break;
2337 default:
2338 icrlo = 0;
2339 KASSERT(x2apic_mode ||
2340 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
2341 ("%s: invalid destination 0x%x", __func__, dest));
2342 destfield = dest;
2343 }
2344
2345 /*
2346 * NMI IPIs are just fake vectors used to send a NMI. Use special rules
2347 * regarding NMIs if passed, otherwise specify the vector.
2348 */
2349 if (vector >= IPI_NMI_FIRST)
2350 icrlo |= APIC_DELMODE_NMI;
2351 else
2352 icrlo |= vector | APIC_DELMODE_FIXED;
2353 icrlo |= APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
2354
2355 /* Wait for an earlier IPI to finish. */
2356 if (!lapic_ipi_wait(lapic_ds_idle_timeout)) {
2357 if (KERNEL_PANICKED())
2358 return;
2359 else
2360 panic("APIC: Previous IPI is stuck");
2361 }
2362
2363 lapic_ipi_raw(icrlo, destfield);
2364
2365 #ifdef DETECT_DEADLOCK
2366 /* Wait for IPI to be delivered. */
2367 if (!lapic_ipi_wait(AFTER_SPIN)) {
2368 #ifdef needsattention
2369 /*
2370 * XXX FIXME:
2371 *
2372 * The above function waits for the message to actually be
2373 * delivered. It breaks out after an arbitrary timeout
2374 * since the message should eventually be delivered (at
2375 * least in theory) and that if it wasn't we would catch
2376 * the failure with the check above when the next IPI is
2377 * sent.
2378 *
2379 * We could skip this wait entirely, EXCEPT it probably
2380 * protects us from other routines that assume that the
2381 * message was delivered and acted upon when this function
2382 * returns.
2383 */
2384 printf("APIC: IPI might be stuck\n");
2385 #else /* !needsattention */
2386 /* Wait until mesage is sent without a timeout. */
2387 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND)
2388 ia32_pause();
2389 #endif /* needsattention */
2390 }
2391 #endif /* DETECT_DEADLOCK */
2392 }
2393
2394 void (*ipi_vectored)(u_int, int) = &native_lapic_ipi_vectored;
2395 #endif /* SMP */
2396
2397 void (*fred_ipi_handlers[IPI_DYN_LAST - IPI_DYN_FIRST + 1])(struct trapframe *);
2398
2399 /*
2400 * Since the IDT is shared by all CPUs the IPI slot update needs to be globally
2401 * visible.
2402 *
2403 * Consider the case where an IPI is generated immediately after allocation:
2404 * vector = lapic_ipi_alloc(ipifunc);
2405 * ipi_selected(other_cpus, vector);
2406 *
2407 * In xAPIC mode a write to ICR_LO has serializing semantics because the
2408 * APIC page is mapped as an uncached region. In x2APIC mode there is an
2409 * explicit 'mfence' before the ICR MSR is written. Therefore in both cases
2410 * the IDT slot update is globally visible before the IPI is delivered.
2411 */
2412 int
lapic_ipi_alloc(inthand_t * ipifunc,void (* fred_ipifunc)(struct trapframe *))2413 lapic_ipi_alloc(inthand_t *ipifunc, void (*fred_ipifunc)(struct trapframe *))
2414 {
2415 void (*fip)(struct trapframe *);
2416 int idx, vector;
2417
2418 KASSERT(ipifunc != &IDTVEC(rsvd) && ipifunc != &IDTVEC(rsvd_pti),
2419 ("invalid ipifunc %p", ipifunc));
2420
2421 vector = -1;
2422 mtx_lock_spin(&icu_lock);
2423 for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
2424 fip = fred_ipi_handlers[idx - IPI_DYN_FIRST];
2425 if (fip == NULL) {
2426 vector = idx;
2427 fred_ipi_handlers[idx - IPI_DYN_FIRST] = fred_ipifunc;
2428 setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
2429 break;
2430 }
2431 }
2432 mtx_unlock_spin(&icu_lock);
2433 return (vector);
2434 }
2435
2436 void
lapic_ipi_free(int vector)2437 lapic_ipi_free(int vector)
2438 {
2439 struct gate_descriptor *ip;
2440 long func __diagused;
2441
2442 KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST,
2443 ("%s: invalid vector %d", __func__, vector));
2444
2445 mtx_lock_spin(&icu_lock);
2446 KASSERT(fred_ipi_handlers[vector - IPI_DYN_FIRST] != NULL,
2447 ("NULL fred func %d", vector));
2448 fred_ipi_handlers[vector - IPI_DYN_FIRST] = NULL;
2449 if (!fred) {
2450 ip = &idt[vector];
2451 func = (ip->gd_hioffset << 16) | ip->gd_looffset;
2452 #ifdef __i386__
2453 func -= setidt_disp;
2454 #endif
2455 KASSERT(func != (uintptr_t)&IDTVEC(rsvd) &&
2456 func != (uintptr_t)&IDTVEC(rsvd_pti),
2457 ("invalid idtfunc %d %#lx", vector, func));
2458 setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd),
2459 SDT_APIC, SEL_KPL, GSEL_APIC);
2460 }
2461 mtx_unlock_spin(&icu_lock);
2462 }
2463