1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Local APIC handling, local APIC timers
4 *
5 * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 *
7 * Fixes
8 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
9 * thanks to Eric Gilmore
10 * and Rolf G. Tews
11 * for testing these extensively.
12 * Maciej W. Rozycki : Various updates and fixes.
13 * Mikael Pettersson : Power Management for UP-APIC.
14 * Pavel Machek and
15 * Mikael Pettersson : PM converted to driver model.
16 */
17
18 #include <linux/perf_event.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/mc146818rtc.h>
21 #include <linux/acpi_pmtmr.h>
22 #include <linux/bitmap.h>
23 #include <linux/clockchips.h>
24 #include <linux/interrupt.h>
25 #include <linux/memblock.h>
26 #include <linux/ftrace.h>
27 #include <linux/ioport.h>
28 #include <linux/export.h>
29 #include <linux/syscore_ops.h>
30 #include <linux/delay.h>
31 #include <linux/timex.h>
32 #include <linux/i8253.h>
33 #include <linux/dmar.h>
34 #include <linux/init.h>
35 #include <linux/cpu.h>
36 #include <linux/dmi.h>
37 #include <linux/smp.h>
38 #include <linux/mm.h>
39 #include <linux/kvm_types.h>
40
41 #include <xen/xen.h>
42
43 #include <asm/trace/irq_vectors.h>
44 #include <asm/irq_remapping.h>
45 #include <asm/pc-conf-reg.h>
46 #include <asm/perf_event.h>
47 #include <asm/x86_init.h>
48 #include <linux/atomic.h>
49 #include <asm/barrier.h>
50 #include <asm/mpspec.h>
51 #include <asm/i8259.h>
52 #include <asm/proto.h>
53 #include <asm/traps.h>
54 #include <asm/apic.h>
55 #include <asm/acpi.h>
56 #include <asm/io_apic.h>
57 #include <asm/desc.h>
58 #include <asm/hpet.h>
59 #include <asm/mtrr.h>
60 #include <asm/time.h>
61 #include <asm/smp.h>
62 #include <asm/mce.h>
63 #include <asm/msr.h>
64 #include <asm/tsc.h>
65 #include <asm/hypervisor.h>
66 #include <asm/cpu_device_id.h>
67 #include <asm/intel-family.h>
68 #include <asm/irq_regs.h>
69 #include <asm/cpu.h>
70
71 #include "local.h"
72
73 /* Processor that is doing the boot up */
74 u32 boot_cpu_physical_apicid __ro_after_init = BAD_APICID;
75 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
76
77 u8 boot_cpu_apic_version __ro_after_init;
78
79 /*
80 * This variable controls which CPUs receive external NMIs. By default,
81 * external NMIs are delivered only to the BSP.
82 */
83 static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP;
84
85 /*
86 * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID
87 */
88 static bool virt_ext_dest_id __ro_after_init;
89
90 /* For parallel bootup. */
91 unsigned long apic_mmio_base __ro_after_init;
92
apic_accessible(void)93 static inline bool apic_accessible(void)
94 {
95 return x2apic_mode || apic_mmio_base;
96 }
97
98 #ifdef CONFIG_X86_32
99 /* Local APIC was disabled by the BIOS and enabled by the kernel */
100 static int enabled_via_apicbase __ro_after_init;
101
102 /*
103 * Handle interrupt mode configuration register (IMCR).
104 * This register controls whether the interrupt signals
105 * that reach the BSP come from the master PIC or from the
106 * local APIC. Before entering Symmetric I/O Mode, either
107 * the BIOS or the operating system must switch out of
108 * PIC Mode by changing the IMCR.
109 */
imcr_pic_to_apic(void)110 static inline void imcr_pic_to_apic(void)
111 {
112 /* NMI and 8259 INTR go through APIC */
113 pc_conf_set(PC_CONF_MPS_IMCR, 0x01);
114 }
115
imcr_apic_to_pic(void)116 static inline void imcr_apic_to_pic(void)
117 {
118 /* NMI and 8259 INTR go directly to BSP */
119 pc_conf_set(PC_CONF_MPS_IMCR, 0x00);
120 }
121 #endif
122
123 /*
124 * Knob to control our willingness to enable the local APIC.
125 *
126 * +1=force-enable
127 */
128 static int force_enable_local_apic __initdata;
129
130 /*
131 * APIC command line parameters
132 */
parse_lapic(char * arg)133 static int __init parse_lapic(char *arg)
134 {
135 if (IS_ENABLED(CONFIG_X86_32) && !arg)
136 force_enable_local_apic = 1;
137 else if (arg && !strncmp(arg, "notscdeadline", 13))
138 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
139 return 0;
140 }
141 early_param("lapic", parse_lapic);
142
143 #ifdef CONFIG_X86_64
144 static int apic_calibrate_pmtmr __initdata;
setup_apicpmtimer(char * s)145 static __init int setup_apicpmtimer(char *s)
146 {
147 apic_calibrate_pmtmr = 1;
148 notsc_setup(NULL);
149 return 1;
150 }
151 __setup("apicpmtimer", setup_apicpmtimer);
152 #endif
153
154 static unsigned long mp_lapic_addr __ro_after_init;
155 bool apic_is_disabled __ro_after_init;
156 /* Disable local APIC timer from the kernel commandline or via dmi quirk */
157 static int disable_apic_timer __initdata;
158 /* Local APIC timer works in C2 */
159 int local_apic_timer_c2_ok __ro_after_init;
160 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
161
162 /*
163 * Debug level, exported for io_apic.c
164 */
165 int apic_verbosity __ro_after_init;
166
167 int pic_mode __ro_after_init;
168
169 /* Have we found an MP table */
170 int smp_found_config __ro_after_init;
171
172 static struct resource lapic_resource = {
173 .name = "Local APIC",
174 .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
175 };
176
177 /* Measured in ticks per HZ. */
178 unsigned int lapic_timer_period = 0;
179
180 static void apic_pm_activate(void);
181
182 /*
183 * Get the LAPIC version
184 */
lapic_get_version(void)185 static inline int lapic_get_version(void)
186 {
187 return GET_APIC_VERSION(apic_read(APIC_LVR));
188 }
189
190 /*
191 * Check, if the APIC is integrated or a separate chip
192 */
lapic_is_integrated(void)193 static inline int lapic_is_integrated(void)
194 {
195 return APIC_INTEGRATED(lapic_get_version());
196 }
197
198 /*
199 * Check, whether this is a modern or a first generation APIC
200 */
modern_apic(void)201 static int modern_apic(void)
202 {
203 /* AMD systems use old APIC versions, so check the CPU */
204 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
205 boot_cpu_data.x86 >= 0xf)
206 return 1;
207
208 /* Hygon systems use modern APIC */
209 if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
210 return 1;
211
212 return lapic_get_version() >= 0x14;
213 }
214
215 /*
216 * right after this call apic become NOOP driven
217 * so apic->write/read doesn't do anything
218 */
apic_disable(void)219 static void __init apic_disable(void)
220 {
221 apic_install_driver(&apic_noop);
222 }
223
native_apic_icr_write(u32 low,u32 id)224 void native_apic_icr_write(u32 low, u32 id)
225 {
226 unsigned long flags;
227
228 local_irq_save(flags);
229 apic_write(APIC_ICR2, SET_XAPIC_DEST_FIELD(id));
230 apic_write(APIC_ICR, low);
231 local_irq_restore(flags);
232 }
233
native_apic_icr_read(void)234 u64 native_apic_icr_read(void)
235 {
236 u32 icr1, icr2;
237
238 icr2 = apic_read(APIC_ICR2);
239 icr1 = apic_read(APIC_ICR);
240
241 return icr1 | ((u64)icr2 << 32);
242 }
243
244 /**
245 * lapic_get_maxlvt - get the maximum number of local vector table entries
246 */
lapic_get_maxlvt(void)247 int lapic_get_maxlvt(void)
248 {
249 /*
250 * - we always have APIC integrated on 64bit mode
251 * - 82489DXs do not report # of LVT entries
252 */
253 return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2;
254 }
255
256 /*
257 * Local APIC timer
258 */
259
260 /* Clock divisor */
261 #define APIC_DIVISOR 16
262 #define TSC_DIVISOR 8
263
264 /* i82489DX specific */
265 #define I82489DX_BASE_DIVIDER (((0x2) << 18))
266
267 /*
268 * This function sets up the local APIC timer, with a timeout of
269 * 'clocks' APIC bus clock. During calibration we actually call
270 * this function twice on the boot CPU, once with a bogus timeout
271 * value, second time for real. The other (noncalibrating) CPUs
272 * call this function only once, with the real, calibrated value.
273 *
274 * We do reads before writes even if unnecessary, to get around the
275 * P5 APIC double write bug.
276 */
__setup_APIC_LVTT(unsigned int clocks,int oneshot,int irqen)277 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
278 {
279 unsigned int lvtt_value, tmp_value;
280
281 lvtt_value = LOCAL_TIMER_VECTOR;
282 if (!oneshot)
283 lvtt_value |= APIC_LVT_TIMER_PERIODIC;
284 else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
285 lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
286
287 /*
288 * The i82489DX APIC uses bit 18 and 19 for the base divider. This
289 * overlaps with bit 18 on integrated APICs, but is not documented
290 * in the SDM. No problem though. i82489DX equipped systems do not
291 * have TSC deadline timer.
292 */
293 if (!lapic_is_integrated())
294 lvtt_value |= I82489DX_BASE_DIVIDER;
295
296 if (!irqen)
297 lvtt_value |= APIC_LVT_MASKED;
298
299 apic_write(APIC_LVTT, lvtt_value);
300
301 if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
302 /*
303 * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
304 * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
305 * According to Intel, MFENCE can do the serialization here.
306 */
307 asm volatile("mfence" : : : "memory");
308 return;
309 }
310
311 /*
312 * Divide PICLK by 16
313 */
314 tmp_value = apic_read(APIC_TDCR);
315 apic_write(APIC_TDCR,
316 (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
317 APIC_TDR_DIV_16);
318
319 if (!oneshot)
320 apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
321 }
322
323 /*
324 * Setup extended LVT, AMD specific
325 *
326 * Software should use the LVT offsets the BIOS provides. The offsets
327 * are determined by the subsystems using it like those for MCE
328 * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
329 * are supported. Beginning with family 10h at least 4 offsets are
330 * available.
331 *
332 * Since the offsets must be consistent for all cores, we keep track
333 * of the LVT offsets in software and reserve the offset for the same
334 * vector also to be used on other cores. An offset is freed by
335 * setting the entry to APIC_EILVT_MASKED.
336 *
337 * If the BIOS is right, there should be no conflicts. Otherwise a
338 * "[Firmware Bug]: ..." error message is generated. However, if
339 * software does not properly determines the offsets, it is not
340 * necessarily a BIOS bug.
341 */
342
343 static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
344
eilvt_entry_is_changeable(unsigned int old,unsigned int new)345 static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
346 {
347 return (old & APIC_EILVT_MASKED)
348 || (new == APIC_EILVT_MASKED)
349 || ((new & ~APIC_EILVT_MASKED) == old);
350 }
351
reserve_eilvt_offset(int offset,unsigned int new)352 static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
353 {
354 unsigned int rsvd, vector;
355
356 if (offset >= APIC_EILVT_NR_MAX)
357 return ~0;
358
359 rsvd = atomic_read(&eilvt_offsets[offset]);
360 do {
361 vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */
362 if (vector && !eilvt_entry_is_changeable(vector, new))
363 /* may not change if vectors are different */
364 return rsvd;
365 } while (!atomic_try_cmpxchg(&eilvt_offsets[offset], &rsvd, new));
366
367 rsvd = new & ~APIC_EILVT_MASKED;
368 if (rsvd && rsvd != vector)
369 pr_info("LVT offset %d assigned for vector 0x%02x\n",
370 offset, rsvd);
371
372 return new;
373 }
374
375 /*
376 * If mask=1, the LVT entry does not generate interrupts while mask=0
377 * enables the vector. See also the BKDGs. Must be called with
378 * preemption disabled.
379 */
380
setup_APIC_eilvt(u8 offset,u8 vector,u8 msg_type,u8 mask)381 int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
382 {
383 unsigned long reg = APIC_EILVTn(offset);
384 unsigned int new, old, reserved;
385
386 new = (mask << 16) | (msg_type << 8) | vector;
387 old = apic_read(reg);
388 reserved = reserve_eilvt_offset(offset, new);
389
390 if (reserved != new) {
391 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
392 "vector 0x%x, but the register is already in use for "
393 "vector 0x%x on another cpu\n",
394 smp_processor_id(), reg, offset, new, reserved);
395 return -EINVAL;
396 }
397
398 if (!eilvt_entry_is_changeable(old, new)) {
399 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
400 "vector 0x%x, but the register is already in use for "
401 "vector 0x%x on this cpu\n",
402 smp_processor_id(), reg, offset, new, old);
403 return -EBUSY;
404 }
405
406 apic_write(reg, new);
407
408 return 0;
409 }
410 EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
411
412 /*
413 * Program the next event, relative to now
414 */
lapic_next_event(unsigned long delta,struct clock_event_device * evt)415 static int lapic_next_event(unsigned long delta,
416 struct clock_event_device *evt)
417 {
418 apic_write(APIC_TMICT, delta);
419 return 0;
420 }
421
lapic_next_deadline(unsigned long delta,struct clock_event_device * evt)422 static int lapic_next_deadline(unsigned long delta,
423 struct clock_event_device *evt)
424 {
425 u64 tsc;
426
427 /* This MSR is special and need a special fence: */
428 weak_wrmsr_fence();
429
430 tsc = rdtsc();
431 wrmsrq(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
432 return 0;
433 }
434
lapic_timer_shutdown(struct clock_event_device * evt)435 static int lapic_timer_shutdown(struct clock_event_device *evt)
436 {
437 unsigned int v;
438
439 /* Lapic used as dummy for broadcast ? */
440 if (evt->features & CLOCK_EVT_FEAT_DUMMY)
441 return 0;
442
443 v = apic_read(APIC_LVTT);
444 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
445 apic_write(APIC_LVTT, v);
446
447 /*
448 * Setting APIC_LVT_MASKED (above) should be enough to tell
449 * the hardware that this timer will never fire. But AMD
450 * erratum 411 and some Intel CPU behavior circa 2024 say
451 * otherwise. Time for belt and suspenders programming: mask
452 * the timer _and_ zero the counter registers:
453 */
454 if (v & APIC_LVT_TIMER_TSCDEADLINE)
455 wrmsrq(MSR_IA32_TSC_DEADLINE, 0);
456 else
457 apic_write(APIC_TMICT, 0);
458
459 return 0;
460 }
461
462 static inline int
lapic_timer_set_periodic_oneshot(struct clock_event_device * evt,bool oneshot)463 lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot)
464 {
465 /* Lapic used as dummy for broadcast ? */
466 if (evt->features & CLOCK_EVT_FEAT_DUMMY)
467 return 0;
468
469 __setup_APIC_LVTT(lapic_timer_period, oneshot, 1);
470 return 0;
471 }
472
lapic_timer_set_periodic(struct clock_event_device * evt)473 static int lapic_timer_set_periodic(struct clock_event_device *evt)
474 {
475 return lapic_timer_set_periodic_oneshot(evt, false);
476 }
477
lapic_timer_set_oneshot(struct clock_event_device * evt)478 static int lapic_timer_set_oneshot(struct clock_event_device *evt)
479 {
480 return lapic_timer_set_periodic_oneshot(evt, true);
481 }
482
483 /*
484 * Local APIC timer broadcast function
485 */
lapic_timer_broadcast(const struct cpumask * mask)486 static void lapic_timer_broadcast(const struct cpumask *mask)
487 {
488 #ifdef CONFIG_SMP
489 __apic_send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
490 #endif
491 }
492
493
494 /*
495 * The local apic timer can be used for any function which is CPU local.
496 */
497 static struct clock_event_device lapic_clockevent = {
498 .name = "lapic",
499 .features = CLOCK_EVT_FEAT_PERIODIC |
500 CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
501 | CLOCK_EVT_FEAT_DUMMY,
502 .shift = 32,
503 .set_state_shutdown = lapic_timer_shutdown,
504 .set_state_periodic = lapic_timer_set_periodic,
505 .set_state_oneshot = lapic_timer_set_oneshot,
506 .set_state_oneshot_stopped = lapic_timer_shutdown,
507 .set_next_event = lapic_next_event,
508 .broadcast = lapic_timer_broadcast,
509 .rating = 100,
510 .irq = -1,
511 };
512 static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
513
514 static const struct x86_cpu_id deadline_match[] __initconst = {
515 X86_MATCH_VFM_STEPS(INTEL_HASWELL_X, 0x2, 0x2, 0x3a), /* EP */
516 X86_MATCH_VFM_STEPS(INTEL_HASWELL_X, 0x4, 0x4, 0x0f), /* EX */
517
518 X86_MATCH_VFM(INTEL_BROADWELL_X, 0x0b000020),
519
520 X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x2, 0x2, 0x00000011),
521 X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x3, 0x3, 0x0700000e),
522 X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x4, 0x4, 0x0f00000c),
523 X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x5, 0x5, 0x0e000003),
524
525 X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X, 0x3, 0x3, 0x01000136),
526 X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X, 0x4, 0x4, 0x02000014),
527 X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X, 0x5, 0xf, 0),
528
529 X86_MATCH_VFM(INTEL_HASWELL, 0x22),
530 X86_MATCH_VFM(INTEL_HASWELL_L, 0x20),
531 X86_MATCH_VFM(INTEL_HASWELL_G, 0x17),
532
533 X86_MATCH_VFM(INTEL_BROADWELL, 0x25),
534 X86_MATCH_VFM(INTEL_BROADWELL_G, 0x17),
535
536 X86_MATCH_VFM(INTEL_SKYLAKE_L, 0xb2),
537 X86_MATCH_VFM(INTEL_SKYLAKE, 0xb2),
538
539 X86_MATCH_VFM(INTEL_KABYLAKE_L, 0x52),
540 X86_MATCH_VFM(INTEL_KABYLAKE, 0x52),
541
542 {},
543 };
544
apic_validate_deadline_timer(void)545 static __init bool apic_validate_deadline_timer(void)
546 {
547 const struct x86_cpu_id *m;
548 u32 rev;
549
550 if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
551 return false;
552 if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
553 return true;
554
555 m = x86_match_cpu(deadline_match);
556 if (!m)
557 return true;
558
559 rev = (u32)m->driver_data;
560
561 if (boot_cpu_data.microcode >= rev)
562 return true;
563
564 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
565 pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; "
566 "please update microcode to version: 0x%x (or later)\n", rev);
567 return false;
568 }
569
570 /*
571 * Setup the local APIC timer for this CPU. Copy the initialized values
572 * of the boot CPU and register the clock event in the framework.
573 */
setup_APIC_timer(void)574 static void setup_APIC_timer(void)
575 {
576 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
577
578 if (this_cpu_has(X86_FEATURE_ARAT)) {
579 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
580 /* Make LAPIC timer preferable over percpu HPET */
581 lapic_clockevent.rating = 150;
582 }
583
584 memcpy(levt, &lapic_clockevent, sizeof(*levt));
585 levt->cpumask = cpumask_of(smp_processor_id());
586
587 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
588 levt->name = "lapic-deadline";
589 levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
590 CLOCK_EVT_FEAT_DUMMY);
591 levt->set_next_event = lapic_next_deadline;
592 clockevents_config_and_register(levt,
593 tsc_khz * (1000 / TSC_DIVISOR),
594 0xF, ~0UL);
595 } else
596 clockevents_register_device(levt);
597
598 apic_update_vector(smp_processor_id(), LOCAL_TIMER_VECTOR, true);
599 }
600
601 /*
602 * Install the updated TSC frequency from recalibration at the TSC
603 * deadline clockevent devices.
604 */
__lapic_update_tsc_freq(void * info)605 static void __lapic_update_tsc_freq(void *info)
606 {
607 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
608
609 if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
610 return;
611
612 clockevents_update_freq(levt, tsc_khz * (1000 / TSC_DIVISOR));
613 }
614
lapic_update_tsc_freq(void)615 void lapic_update_tsc_freq(void)
616 {
617 /*
618 * The clockevent device's ->mult and ->shift can both be
619 * changed. In order to avoid races, schedule the frequency
620 * update code on each CPU.
621 */
622 on_each_cpu(__lapic_update_tsc_freq, NULL, 0);
623 }
624
625 /*
626 * In this functions we calibrate APIC bus clocks to the external timer.
627 *
628 * We want to do the calibration only once since we want to have local timer
629 * irqs synchronous. CPUs connected by the same APIC bus have the very same bus
630 * frequency.
631 *
632 * This was previously done by reading the PIT/HPET and waiting for a wrap
633 * around to find out, that a tick has elapsed. I have a box, where the PIT
634 * readout is broken, so it never gets out of the wait loop again. This was
635 * also reported by others.
636 *
637 * Monitoring the jiffies value is inaccurate and the clockevents
638 * infrastructure allows us to do a simple substitution of the interrupt
639 * handler.
640 *
641 * The calibration routine also uses the pm_timer when possible, as the PIT
642 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
643 * back to normal later in the boot process).
644 */
645
646 #define LAPIC_CAL_LOOPS (HZ/10)
647
648 static __initdata int lapic_cal_loops = -1;
649 static __initdata long lapic_cal_t1, lapic_cal_t2;
650 static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
651 static __initdata u32 lapic_cal_pm1, lapic_cal_pm2;
652 static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
653
654 /*
655 * Temporary interrupt handler and polled calibration function.
656 */
lapic_cal_handler(struct clock_event_device * dev)657 static void __init lapic_cal_handler(struct clock_event_device *dev)
658 {
659 unsigned long long tsc = 0;
660 long tapic = apic_read(APIC_TMCCT);
661 u32 pm = acpi_pm_read_early();
662
663 if (boot_cpu_has(X86_FEATURE_TSC))
664 tsc = rdtsc();
665
666 switch (lapic_cal_loops++) {
667 case 0:
668 lapic_cal_t1 = tapic;
669 lapic_cal_tsc1 = tsc;
670 lapic_cal_pm1 = pm;
671 lapic_cal_j1 = jiffies;
672 break;
673
674 case LAPIC_CAL_LOOPS:
675 lapic_cal_t2 = tapic;
676 lapic_cal_tsc2 = tsc;
677 if (pm < lapic_cal_pm1)
678 pm += ACPI_PM_OVRRUN;
679 lapic_cal_pm2 = pm;
680 lapic_cal_j2 = jiffies;
681 break;
682 }
683 }
684
685 static int __init
calibrate_by_pmtimer(u32 deltapm,long * delta,long * deltatsc)686 calibrate_by_pmtimer(u32 deltapm, long *delta, long *deltatsc)
687 {
688 const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
689 const long pm_thresh = pm_100ms / 100;
690 unsigned long mult;
691 u64 res;
692
693 #ifndef CONFIG_X86_PM_TIMER
694 return -1;
695 #endif
696
697 apic_pr_verbose("... PM-Timer delta = %u\n", deltapm);
698
699 /* Check, if the PM timer is available */
700 if (!deltapm)
701 return -1;
702
703 mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
704
705 if (deltapm > (pm_100ms - pm_thresh) &&
706 deltapm < (pm_100ms + pm_thresh)) {
707 apic_pr_verbose("... PM-Timer result ok\n");
708 return 0;
709 }
710
711 res = (((u64)deltapm) * mult) >> 22;
712 do_div(res, 1000000);
713 pr_warn("APIC calibration not consistent with PM-Timer: %ldms instead of 100ms\n",
714 (long)res);
715
716 /* Correct the lapic counter value */
717 res = (((u64)(*delta)) * pm_100ms);
718 do_div(res, deltapm);
719 pr_info("APIC delta adjusted to PM-Timer: "
720 "%lu (%ld)\n", (unsigned long)res, *delta);
721 *delta = (long)res;
722
723 /* Correct the tsc counter value */
724 if (boot_cpu_has(X86_FEATURE_TSC)) {
725 res = (((u64)(*deltatsc)) * pm_100ms);
726 do_div(res, deltapm);
727 apic_pr_verbose("TSC delta adjusted to PM-Timer: %lu (%ld)\n",
728 (unsigned long)res, *deltatsc);
729 *deltatsc = (long)res;
730 }
731
732 return 0;
733 }
734
lapic_init_clockevent(void)735 static int __init lapic_init_clockevent(void)
736 {
737 if (!lapic_timer_period)
738 return -1;
739
740 /* Calculate the scaled math multiplication factor */
741 lapic_clockevent.mult = div_sc(lapic_timer_period/APIC_DIVISOR,
742 TICK_NSEC, lapic_clockevent.shift);
743 lapic_clockevent.max_delta_ns =
744 clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
745 lapic_clockevent.max_delta_ticks = 0x7FFFFFFF;
746 lapic_clockevent.min_delta_ns =
747 clockevent_delta2ns(0xF, &lapic_clockevent);
748 lapic_clockevent.min_delta_ticks = 0xF;
749
750 return 0;
751 }
752
apic_needs_pit(void)753 bool __init apic_needs_pit(void)
754 {
755 /*
756 * If the frequencies are not known, PIT is required for both TSC
757 * and apic timer calibration.
758 */
759 if (!tsc_khz || !cpu_khz)
760 return true;
761
762 /* Is there an APIC at all or is it disabled? */
763 if (!boot_cpu_has(X86_FEATURE_APIC) || apic_is_disabled)
764 return true;
765
766 /*
767 * If interrupt delivery mode is legacy PIC or virtual wire without
768 * configuration, the local APIC timer won't be set up. Make sure
769 * that the PIT is initialized.
770 */
771 if (apic_intr_mode == APIC_PIC ||
772 apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG)
773 return true;
774
775 /* Virt guests may lack ARAT, but still have DEADLINE */
776 if (!boot_cpu_has(X86_FEATURE_ARAT))
777 return true;
778
779 /* Deadline timer is based on TSC so no further PIT action required */
780 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
781 return false;
782
783 /* APIC timer disabled? */
784 if (disable_apic_timer)
785 return true;
786 /*
787 * The APIC timer frequency is known already, no PIT calibration
788 * required. If unknown, let the PIT be initialized.
789 */
790 return lapic_timer_period == 0;
791 }
792
calibrate_APIC_clock(void)793 static int __init calibrate_APIC_clock(void)
794 {
795 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
796 u64 tsc_perj = 0, tsc_start = 0;
797 long delta_tsc_khz, bus_khz;
798 unsigned long jif_start;
799 unsigned long deltaj;
800 long delta, deltatsc;
801 int pm_referenced = 0;
802
803 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
804 return 0;
805
806 /*
807 * Check if lapic timer has already been calibrated by platform
808 * specific routine, such as tsc calibration code. If so just fill
809 * in the clockevent structure and return.
810 */
811 if (!lapic_init_clockevent()) {
812 apic_pr_verbose("lapic timer already calibrated %d\n", lapic_timer_period);
813 /*
814 * Direct calibration methods must have an always running
815 * local APIC timer, no need for broadcast timer.
816 */
817 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
818 return 0;
819 }
820
821 apic_pr_verbose("Using local APIC timer interrupts. Calibrating APIC timer ...\n");
822
823 /*
824 * There are platforms w/o global clockevent devices. Instead of
825 * making the calibration conditional on that, use a polling based
826 * approach everywhere.
827 */
828 local_irq_disable();
829
830 /*
831 * Setup the APIC counter to maximum. There is no way the lapic
832 * can underflow in the 100ms detection time frame
833 */
834 __setup_APIC_LVTT(0xffffffff, 0, 0);
835
836 /*
837 * Methods to terminate the calibration loop:
838 * 1) Global clockevent if available (jiffies)
839 * 2) TSC if available and frequency is known
840 */
841 jif_start = READ_ONCE(jiffies);
842
843 if (tsc_khz) {
844 tsc_start = rdtsc();
845 tsc_perj = div_u64((u64)tsc_khz * 1000, HZ);
846 }
847
848 /*
849 * Enable interrupts so the tick can fire, if a global
850 * clockevent device is available
851 */
852 local_irq_enable();
853
854 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) {
855 /* Wait for a tick to elapse */
856 while (1) {
857 if (tsc_khz) {
858 u64 tsc_now = rdtsc();
859 if ((tsc_now - tsc_start) >= tsc_perj) {
860 tsc_start += tsc_perj;
861 break;
862 }
863 } else {
864 unsigned long jif_now = READ_ONCE(jiffies);
865
866 if (time_after(jif_now, jif_start)) {
867 jif_start = jif_now;
868 break;
869 }
870 }
871 cpu_relax();
872 }
873
874 /* Invoke the calibration routine */
875 local_irq_disable();
876 lapic_cal_handler(NULL);
877 local_irq_enable();
878 }
879
880 local_irq_disable();
881
882 /* Build delta t1-t2 as apic timer counts down */
883 delta = lapic_cal_t1 - lapic_cal_t2;
884 apic_pr_verbose("... lapic delta = %ld\n", delta);
885
886 deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
887
888 /* we trust the PM based calibration if possible */
889 pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
890 &delta, &deltatsc);
891
892 lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
893 lapic_init_clockevent();
894
895 apic_pr_verbose("..... delta %ld\n", delta);
896 apic_pr_verbose("..... mult: %u\n", lapic_clockevent.mult);
897 apic_pr_verbose("..... calibration result: %u\n", lapic_timer_period);
898
899 if (boot_cpu_has(X86_FEATURE_TSC)) {
900 delta_tsc_khz = (deltatsc * HZ) / (1000 * LAPIC_CAL_LOOPS);
901
902 apic_pr_verbose("..... CPU clock speed is %ld.%03ld MHz.\n",
903 delta_tsc_khz / 1000, delta_tsc_khz % 1000);
904 }
905
906 bus_khz = (long)lapic_timer_period * HZ / 1000;
907 apic_pr_verbose("..... host bus clock speed is %ld.%03ld MHz.\n",
908 bus_khz / 1000, bus_khz % 1000);
909
910 /*
911 * Do a sanity check on the APIC calibration result
912 */
913 if (lapic_timer_period < (1000000 / HZ)) {
914 local_irq_enable();
915 pr_warn("APIC frequency too slow, disabling apic timer\n");
916 return -1;
917 }
918
919 levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
920
921 /*
922 * PM timer calibration failed or not turned on so lets try APIC
923 * timer based calibration, if a global clockevent device is
924 * available.
925 */
926 if (!pm_referenced && global_clock_event) {
927 apic_pr_verbose("... verify APIC timer\n");
928
929 /*
930 * Setup the apic timer manually
931 */
932 levt->event_handler = lapic_cal_handler;
933 lapic_timer_set_periodic(levt);
934 lapic_cal_loops = -1;
935
936 /* Let the interrupts run */
937 local_irq_enable();
938
939 while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
940 cpu_relax();
941
942 /* Stop the lapic timer */
943 local_irq_disable();
944 lapic_timer_shutdown(levt);
945
946 /* Jiffies delta */
947 deltaj = lapic_cal_j2 - lapic_cal_j1;
948 apic_pr_verbose("... jiffies delta = %lu\n", deltaj);
949
950 /* Check, if the jiffies result is consistent */
951 if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
952 apic_pr_verbose("... jiffies result ok\n");
953 else
954 levt->features |= CLOCK_EVT_FEAT_DUMMY;
955 }
956 local_irq_enable();
957
958 if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
959 pr_warn("APIC timer disabled due to verification failure\n");
960 return -1;
961 }
962
963 return 0;
964 }
965
966 /*
967 * Setup the boot APIC
968 *
969 * Calibrate and verify the result.
970 */
setup_boot_APIC_clock(void)971 void __init setup_boot_APIC_clock(void)
972 {
973 /*
974 * The local apic timer can be disabled via the kernel
975 * commandline or from the CPU detection code. Register the lapic
976 * timer as a dummy clock event source on SMP systems, so the
977 * broadcast mechanism is used. On UP systems simply ignore it.
978 */
979 if (disable_apic_timer) {
980 pr_info("Disabling APIC timer\n");
981 /* No broadcast on UP ! */
982 if (num_possible_cpus() > 1) {
983 lapic_clockevent.mult = 1;
984 setup_APIC_timer();
985 }
986 return;
987 }
988
989 if (calibrate_APIC_clock()) {
990 /* No broadcast on UP ! */
991 if (num_possible_cpus() > 1)
992 setup_APIC_timer();
993 return;
994 }
995
996 /*
997 * If nmi_watchdog is set to IO_APIC, we need the
998 * PIT/HPET going. Otherwise register lapic as a dummy
999 * device.
1000 */
1001 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
1002
1003 /* Setup the lapic or request the broadcast */
1004 setup_APIC_timer();
1005 amd_e400_c1e_apic_setup();
1006 }
1007
setup_secondary_APIC_clock(void)1008 void setup_secondary_APIC_clock(void)
1009 {
1010 setup_APIC_timer();
1011 amd_e400_c1e_apic_setup();
1012 }
1013
1014 /*
1015 * The guts of the apic timer interrupt
1016 */
local_apic_timer_interrupt(void)1017 static void local_apic_timer_interrupt(void)
1018 {
1019 struct clock_event_device *evt = this_cpu_ptr(&lapic_events);
1020
1021 /*
1022 * Normally we should not be here till LAPIC has been initialized but
1023 * in some cases like kdump, its possible that there is a pending LAPIC
1024 * timer interrupt from previous kernel's context and is delivered in
1025 * new kernel the moment interrupts are enabled.
1026 *
1027 * Interrupts are enabled early and LAPIC is setup much later, hence
1028 * its possible that when we get here evt->event_handler is NULL.
1029 * Check for event_handler being NULL and discard the interrupt as
1030 * spurious.
1031 */
1032 if (!evt->event_handler) {
1033 pr_warn("Spurious LAPIC timer interrupt on cpu %d\n",
1034 smp_processor_id());
1035 /* Switch it off */
1036 lapic_timer_shutdown(evt);
1037 return;
1038 }
1039
1040 /*
1041 * the NMI deadlock-detector uses this.
1042 */
1043 inc_irq_stat(apic_timer_irqs);
1044
1045 evt->event_handler(evt);
1046 }
1047
1048 /*
1049 * Local APIC timer interrupt. This is the most natural way for doing
1050 * local interrupts, but local timer interrupts can be emulated by
1051 * broadcast interrupts too. [in case the hw doesn't support APIC timers]
1052 *
1053 * [ if a single-CPU system runs an SMP kernel then we call the local
1054 * interrupt as well. Thus we cannot inline the local irq ... ]
1055 */
DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)1056 DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
1057 {
1058 struct pt_regs *old_regs = set_irq_regs(regs);
1059
1060 apic_eoi();
1061 trace_local_timer_entry(LOCAL_TIMER_VECTOR);
1062 local_apic_timer_interrupt();
1063 trace_local_timer_exit(LOCAL_TIMER_VECTOR);
1064
1065 set_irq_regs(old_regs);
1066 }
1067
1068 /*
1069 * Local APIC start and shutdown
1070 */
1071
1072 /**
1073 * clear_local_APIC - shutdown the local APIC
1074 *
1075 * This is called, when a CPU is disabled and before rebooting, so the state of
1076 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
1077 * leftovers during boot.
1078 */
clear_local_APIC(void)1079 void clear_local_APIC(void)
1080 {
1081 int maxlvt;
1082 u32 v;
1083
1084 if (!apic_accessible())
1085 return;
1086
1087 maxlvt = lapic_get_maxlvt();
1088 /*
1089 * Masking an LVT entry can trigger a local APIC error
1090 * if the vector is zero. Mask LVTERR first to prevent this.
1091 */
1092 if (maxlvt >= 3) {
1093 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
1094 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
1095 }
1096 /*
1097 * Careful: we have to set masks only first to deassert
1098 * any level-triggered sources.
1099 */
1100 v = apic_read(APIC_LVTT);
1101 apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
1102 v = apic_read(APIC_LVT0);
1103 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
1104 v = apic_read(APIC_LVT1);
1105 apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
1106 if (maxlvt >= 4) {
1107 v = apic_read(APIC_LVTPC);
1108 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
1109 }
1110
1111 /* lets not touch this if we didn't frob it */
1112 #ifdef CONFIG_X86_THERMAL_VECTOR
1113 if (maxlvt >= 5) {
1114 v = apic_read(APIC_LVTTHMR);
1115 apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
1116 }
1117 #endif
1118 #ifdef CONFIG_X86_MCE_INTEL
1119 if (maxlvt >= 6) {
1120 v = apic_read(APIC_LVTCMCI);
1121 if (!(v & APIC_LVT_MASKED))
1122 apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
1123 }
1124 #endif
1125
1126 /*
1127 * Clean APIC state for other OSs:
1128 */
1129 apic_write(APIC_LVTT, APIC_LVT_MASKED);
1130 apic_write(APIC_LVT0, APIC_LVT_MASKED);
1131 apic_write(APIC_LVT1, APIC_LVT_MASKED);
1132 if (maxlvt >= 3)
1133 apic_write(APIC_LVTERR, APIC_LVT_MASKED);
1134 if (maxlvt >= 4)
1135 apic_write(APIC_LVTPC, APIC_LVT_MASKED);
1136
1137 /* Integrated APIC (!82489DX) ? */
1138 if (lapic_is_integrated()) {
1139 if (maxlvt > 3)
1140 /* Clear ESR due to Pentium errata 3AP and 11AP */
1141 apic_write(APIC_ESR, 0);
1142 apic_read(APIC_ESR);
1143 }
1144 }
1145
1146 /**
1147 * apic_soft_disable - Clears and software disables the local APIC on hotplug
1148 *
1149 * Contrary to disable_local_APIC() this does not touch the enable bit in
1150 * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC
1151 * bus would require a hardware reset as the APIC would lose track of bus
1152 * arbitration. On systems with FSB delivery APICBASE could be disabled,
1153 * but it has to be guaranteed that no interrupt is sent to the APIC while
1154 * in that state and it's not clear from the SDM whether it still responds
1155 * to INIT/SIPI messages. Stay on the safe side and use software disable.
1156 */
apic_soft_disable(void)1157 void apic_soft_disable(void)
1158 {
1159 u32 value;
1160
1161 clear_local_APIC();
1162
1163 /* Soft disable APIC (implies clearing of registers for 82489DX!). */
1164 value = apic_read(APIC_SPIV);
1165 value &= ~APIC_SPIV_APIC_ENABLED;
1166 apic_write(APIC_SPIV, value);
1167 }
1168
1169 /**
1170 * disable_local_APIC - clear and disable the local APIC
1171 */
disable_local_APIC(void)1172 void disable_local_APIC(void)
1173 {
1174 if (!apic_accessible())
1175 return;
1176
1177 if (apic->teardown)
1178 apic->teardown();
1179
1180 apic_soft_disable();
1181
1182 #ifdef CONFIG_X86_32
1183 /*
1184 * When LAPIC was disabled by the BIOS and enabled by the kernel,
1185 * restore the disabled state.
1186 */
1187 if (enabled_via_apicbase) {
1188 unsigned int l, h;
1189
1190 rdmsr(MSR_IA32_APICBASE, l, h);
1191 l &= ~MSR_IA32_APICBASE_ENABLE;
1192 wrmsr(MSR_IA32_APICBASE, l, h);
1193 }
1194 #endif
1195 }
1196
1197 /*
1198 * If Linux enabled the LAPIC against the BIOS default disable it down before
1199 * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1200 * not power-off. Additionally clear all LVT entries before disable_local_APIC
1201 * for the case where Linux didn't enable the LAPIC.
1202 */
lapic_shutdown(void)1203 void lapic_shutdown(void)
1204 {
1205 unsigned long flags;
1206
1207 if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1208 return;
1209
1210 local_irq_save(flags);
1211
1212 #ifdef CONFIG_X86_32
1213 if (!enabled_via_apicbase)
1214 clear_local_APIC();
1215 else
1216 #endif
1217 disable_local_APIC();
1218
1219
1220 local_irq_restore(flags);
1221 }
1222
1223 /**
1224 * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1225 */
sync_Arb_IDs(void)1226 void __init sync_Arb_IDs(void)
1227 {
1228 /*
1229 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1230 * needed on AMD.
1231 */
1232 if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1233 return;
1234
1235 /*
1236 * Wait for idle.
1237 */
1238 apic_wait_icr_idle();
1239
1240 apic_pr_debug("Synchronizing Arb IDs.\n");
1241 apic_write(APIC_ICR, APIC_DEST_ALLINC | APIC_INT_LEVELTRIG | APIC_DM_INIT);
1242 }
1243
1244 enum apic_intr_mode_id apic_intr_mode __ro_after_init;
1245
__apic_intr_mode_select(void)1246 static int __init __apic_intr_mode_select(void)
1247 {
1248 /* Check kernel option */
1249 if (apic_is_disabled) {
1250 pr_info("APIC disabled via kernel command line\n");
1251 return APIC_PIC;
1252 }
1253
1254 /* Check BIOS */
1255 #ifdef CONFIG_X86_64
1256 /* On 64-bit, the APIC must be integrated, Check local APIC only */
1257 if (!boot_cpu_has(X86_FEATURE_APIC)) {
1258 apic_is_disabled = true;
1259 pr_info("APIC disabled by BIOS\n");
1260 return APIC_PIC;
1261 }
1262 #else
1263 /* On 32-bit, the APIC may be integrated APIC or 82489DX */
1264
1265 /* Neither 82489DX nor integrated APIC ? */
1266 if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) {
1267 apic_is_disabled = true;
1268 return APIC_PIC;
1269 }
1270
1271 /* If the BIOS pretends there is an integrated APIC ? */
1272 if (!boot_cpu_has(X86_FEATURE_APIC) &&
1273 APIC_INTEGRATED(boot_cpu_apic_version)) {
1274 apic_is_disabled = true;
1275 pr_err(FW_BUG "Local APIC not detected, force emulation\n");
1276 return APIC_PIC;
1277 }
1278 #endif
1279
1280 /* Check MP table or ACPI MADT configuration */
1281 if (!smp_found_config) {
1282 disable_ioapic_support();
1283 if (!acpi_lapic) {
1284 pr_info("APIC: ACPI MADT or MP tables are not detected\n");
1285 return APIC_VIRTUAL_WIRE_NO_CONFIG;
1286 }
1287 return APIC_VIRTUAL_WIRE;
1288 }
1289
1290 #ifdef CONFIG_SMP
1291 /* If SMP should be disabled, then really disable it! */
1292 if (!setup_max_cpus) {
1293 pr_info("APIC: SMP mode deactivated\n");
1294 return APIC_SYMMETRIC_IO_NO_ROUTING;
1295 }
1296 #endif
1297
1298 return APIC_SYMMETRIC_IO;
1299 }
1300
1301 /* Select the interrupt delivery mode for the BSP */
apic_intr_mode_select(void)1302 void __init apic_intr_mode_select(void)
1303 {
1304 apic_intr_mode = __apic_intr_mode_select();
1305 }
1306
1307 /*
1308 * An initial setup of the virtual wire mode.
1309 */
init_bsp_APIC(void)1310 void __init init_bsp_APIC(void)
1311 {
1312 unsigned int value;
1313
1314 /*
1315 * Don't do the setup now if we have a SMP BIOS as the
1316 * through-I/O-APIC virtual wire mode might be active.
1317 */
1318 if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC))
1319 return;
1320
1321 /*
1322 * Do not trust the local APIC being empty at bootup.
1323 */
1324 clear_local_APIC();
1325
1326 /*
1327 * Enable APIC.
1328 */
1329 value = apic_read(APIC_SPIV);
1330 value &= ~APIC_VECTOR_MASK;
1331 value |= APIC_SPIV_APIC_ENABLED;
1332
1333 #ifdef CONFIG_X86_32
1334 /* This bit is reserved on P4/Xeon and should be cleared */
1335 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1336 (boot_cpu_data.x86 == 15))
1337 value &= ~APIC_SPIV_FOCUS_DISABLED;
1338 else
1339 #endif
1340 value |= APIC_SPIV_FOCUS_DISABLED;
1341 value |= SPURIOUS_APIC_VECTOR;
1342 apic_write(APIC_SPIV, value);
1343
1344 /*
1345 * Set up the virtual wire mode.
1346 */
1347 apic_write(APIC_LVT0, APIC_DM_EXTINT);
1348 value = APIC_DM_NMI;
1349 if (!lapic_is_integrated()) /* 82489DX */
1350 value |= APIC_LVT_LEVEL_TRIGGER;
1351 if (apic_extnmi == APIC_EXTNMI_NONE)
1352 value |= APIC_LVT_MASKED;
1353 apic_write(APIC_LVT1, value);
1354 }
1355
1356 static void __init apic_bsp_setup(bool upmode);
1357
1358 /* Init the interrupt delivery mode for the BSP */
apic_intr_mode_init(void)1359 void __init apic_intr_mode_init(void)
1360 {
1361 bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT);
1362
1363 switch (apic_intr_mode) {
1364 case APIC_PIC:
1365 pr_info("APIC: Keep in PIC mode(8259)\n");
1366 return;
1367 case APIC_VIRTUAL_WIRE:
1368 pr_info("APIC: Switch to virtual wire mode setup\n");
1369 break;
1370 case APIC_VIRTUAL_WIRE_NO_CONFIG:
1371 pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");
1372 upmode = true;
1373 break;
1374 case APIC_SYMMETRIC_IO:
1375 pr_info("APIC: Switch to symmetric I/O mode setup\n");
1376 break;
1377 case APIC_SYMMETRIC_IO_NO_ROUTING:
1378 pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");
1379 break;
1380 }
1381
1382 x86_64_probe_apic();
1383
1384 if (x86_platform.apic_post_init)
1385 x86_platform.apic_post_init();
1386
1387 apic_bsp_setup(upmode);
1388 }
1389
lapic_setup_esr(void)1390 static void lapic_setup_esr(void)
1391 {
1392 unsigned int oldvalue, value, maxlvt;
1393
1394 if (!lapic_is_integrated()) {
1395 pr_info("No ESR for 82489DX.\n");
1396 return;
1397 }
1398
1399 if (apic->disable_esr) {
1400 /*
1401 * Something untraceable is creating bad interrupts on
1402 * secondary quads ... for the moment, just leave the
1403 * ESR disabled - we can't do anything useful with the
1404 * errors anyway - mbligh
1405 */
1406 pr_info("Leaving ESR disabled.\n");
1407 return;
1408 }
1409
1410 maxlvt = lapic_get_maxlvt();
1411 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1412 apic_write(APIC_ESR, 0);
1413 oldvalue = apic_read(APIC_ESR);
1414
1415 /* enables sending errors */
1416 value = ERROR_APIC_VECTOR;
1417 apic_write(APIC_LVTERR, value);
1418
1419 /*
1420 * spec says clear errors after enabling vector.
1421 */
1422 if (maxlvt > 3)
1423 apic_write(APIC_ESR, 0);
1424 value = apic_read(APIC_ESR);
1425 if (value != oldvalue) {
1426 apic_pr_verbose("ESR value before enabling vector: 0x%08x after: 0x%08x\n",
1427 oldvalue, value);
1428 }
1429 }
1430
1431 #define APIC_IR_REGS APIC_ISR_NR
1432 #define APIC_IR_BITS (APIC_IR_REGS * 32)
1433 #define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG)
1434
1435 union apic_ir {
1436 unsigned long map[APIC_IR_MAPSIZE];
1437 u32 regs[APIC_IR_REGS];
1438 };
1439
apic_check_and_eoi_isr(union apic_ir * isr)1440 static bool apic_check_and_eoi_isr(union apic_ir *isr)
1441 {
1442 int i, bit;
1443
1444 /* Read the ISRs */
1445 for (i = 0; i < APIC_IR_REGS; i++)
1446 isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
1447
1448 /* If the ISR map empty, nothing to do here. */
1449 if (bitmap_empty(isr->map, APIC_IR_BITS))
1450 return true;
1451
1452 /*
1453 * There can be multiple ISR bits set when a high priority
1454 * interrupt preempted a lower priority one. Issue an EOI for each
1455 * set bit. The priority traversal order does not matter as there
1456 * can't be new ISR bits raised at this point. What matters is that
1457 * an EOI is issued for each ISR bit.
1458 */
1459 for_each_set_bit(bit, isr->map, APIC_IR_BITS)
1460 apic_eoi();
1461
1462 /* Reread the ISRs, they should be empty now */
1463 for (i = 0; i < APIC_IR_REGS; i++)
1464 isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
1465
1466 return bitmap_empty(isr->map, APIC_IR_BITS);
1467 }
1468
1469 /*
1470 * If a CPU services an interrupt and crashes before issuing EOI to the
1471 * local APIC, the corresponding ISR bit is still set when the crashing CPU
1472 * jumps into a crash kernel. Read the ISR and issue an EOI for each set
1473 * bit to acknowledge it as otherwise these slots would be locked forever
1474 * waiting for an EOI.
1475 *
1476 * If there are pending bits in the IRR, then they won't be converted into
1477 * ISR bits as the CPU has interrupts disabled. They will be delivered once
1478 * the CPU enables interrupts and there is nothing which can prevent that.
1479 *
1480 * In the worst case this results in spurious interrupt warnings.
1481 */
apic_clear_isr(void)1482 static void apic_clear_isr(void)
1483 {
1484 union apic_ir ir;
1485 unsigned int i;
1486
1487 if (!apic_check_and_eoi_isr(&ir))
1488 pr_warn("APIC: Stale ISR: %256pb\n", ir.map);
1489
1490 for (i = 0; i < APIC_IR_REGS; i++)
1491 ir.regs[i] = apic_read(APIC_IRR + i * 0x10);
1492
1493 if (!bitmap_empty(ir.map, APIC_IR_BITS))
1494 pr_warn("APIC: Stale IRR: %256pb\n", ir.map);
1495 }
1496
1497 /**
1498 * setup_local_APIC - setup the local APIC
1499 *
1500 * Used to setup local APIC while initializing BSP or bringing up APs.
1501 * Always called with preemption disabled.
1502 */
setup_local_APIC(void)1503 static void setup_local_APIC(void)
1504 {
1505 int cpu = smp_processor_id();
1506 unsigned int value;
1507
1508 if (apic_is_disabled) {
1509 disable_ioapic_support();
1510 return;
1511 }
1512
1513 if (apic->setup)
1514 apic->setup();
1515
1516 /*
1517 * If this comes from kexec/kcrash the APIC might be enabled in
1518 * SPIV. Soft disable it before doing further initialization.
1519 */
1520 value = apic_read(APIC_SPIV);
1521 value &= ~APIC_SPIV_APIC_ENABLED;
1522 apic_write(APIC_SPIV, value);
1523
1524 #ifdef CONFIG_X86_32
1525 /* Pound the ESR really hard over the head with a big hammer - mbligh */
1526 if (lapic_is_integrated() && apic->disable_esr) {
1527 apic_write(APIC_ESR, 0);
1528 apic_write(APIC_ESR, 0);
1529 apic_write(APIC_ESR, 0);
1530 apic_write(APIC_ESR, 0);
1531 }
1532 #endif
1533 /*
1534 * Intel recommends to set DFR, LDR and TPR before enabling
1535 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
1536 * document number 292116).
1537 *
1538 * Except for APICs which operate in physical destination mode.
1539 */
1540 if (apic->init_apic_ldr)
1541 apic->init_apic_ldr();
1542
1543 /*
1544 * Set Task Priority to 'accept all except vectors 0-31'. An APIC
1545 * vector in the 16-31 range could be delivered if TPR == 0, but we
1546 * would think it's an exception and terrible things will happen. We
1547 * never change this later on.
1548 */
1549 value = apic_read(APIC_TASKPRI);
1550 value &= ~APIC_TPRI_MASK;
1551 value |= 0x10;
1552 apic_write(APIC_TASKPRI, value);
1553
1554 apic_clear_isr();
1555
1556 /*
1557 * Now that we are all set up, enable the APIC
1558 */
1559 value = apic_read(APIC_SPIV);
1560 value &= ~APIC_VECTOR_MASK;
1561 /*
1562 * Enable APIC
1563 */
1564 value |= APIC_SPIV_APIC_ENABLED;
1565
1566 #ifdef CONFIG_X86_32
1567 /*
1568 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
1569 * certain networking cards. If high frequency interrupts are
1570 * happening on a particular IOAPIC pin, plus the IOAPIC routing
1571 * entry is masked/unmasked at a high rate as well then sooner or
1572 * later IOAPIC line gets 'stuck', no more interrupts are received
1573 * from the device. If focus CPU is disabled then the hang goes
1574 * away, oh well :-(
1575 *
1576 * [ This bug can be reproduced easily with a level-triggered
1577 * PCI Ne2000 networking cards and PII/PIII processors, dual
1578 * BX chipset. ]
1579 */
1580 /*
1581 * Actually disabling the focus CPU check just makes the hang less
1582 * frequent as it makes the interrupt distribution model be more
1583 * like LRU than MRU (the short-term load is more even across CPUs).
1584 */
1585
1586 /*
1587 * - enable focus processor (bit==0)
1588 * - 64bit mode always use processor focus
1589 * so no need to set it
1590 */
1591 value &= ~APIC_SPIV_FOCUS_DISABLED;
1592 #endif
1593
1594 /*
1595 * Set spurious IRQ vector
1596 */
1597 value |= SPURIOUS_APIC_VECTOR;
1598 apic_write(APIC_SPIV, value);
1599
1600 perf_events_lapic_init();
1601
1602 /*
1603 * Set up LVT0, LVT1:
1604 *
1605 * set up through-local-APIC on the boot CPU's LINT0. This is not
1606 * strictly necessary in pure symmetric-IO mode, but sometimes
1607 * we delegate interrupts to the 8259A.
1608 */
1609 /*
1610 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
1611 */
1612 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1613 if (!cpu && (pic_mode || !value || ioapic_is_disabled)) {
1614 value = APIC_DM_EXTINT;
1615 apic_pr_verbose("Enabled ExtINT on CPU#%d\n", cpu);
1616 } else {
1617 value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1618 apic_pr_verbose("Masked ExtINT on CPU#%d\n", cpu);
1619 }
1620 apic_write(APIC_LVT0, value);
1621
1622 /*
1623 * Only the BSP sees the LINT1 NMI signal by default. This can be
1624 * modified by apic_extnmi= boot option.
1625 */
1626 if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) ||
1627 apic_extnmi == APIC_EXTNMI_ALL)
1628 value = APIC_DM_NMI;
1629 else
1630 value = APIC_DM_NMI | APIC_LVT_MASKED;
1631
1632 /* Is 82489DX ? */
1633 if (!lapic_is_integrated())
1634 value |= APIC_LVT_LEVEL_TRIGGER;
1635 apic_write(APIC_LVT1, value);
1636
1637 #ifdef CONFIG_X86_MCE_INTEL
1638 /* Recheck CMCI information after local APIC is up on CPU #0 */
1639 if (!cpu)
1640 cmci_recheck();
1641 #endif
1642 }
1643
end_local_APIC_setup(void)1644 static void end_local_APIC_setup(void)
1645 {
1646 lapic_setup_esr();
1647
1648 #ifdef CONFIG_X86_32
1649 {
1650 unsigned int value;
1651 /* Disable the local apic timer */
1652 value = apic_read(APIC_LVTT);
1653 value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1654 apic_write(APIC_LVTT, value);
1655 }
1656 #endif
1657
1658 apic_pm_activate();
1659 }
1660
1661 /*
1662 * APIC setup function for application processors. Called from smpboot.c
1663 */
apic_ap_setup(void)1664 void apic_ap_setup(void)
1665 {
1666 setup_local_APIC();
1667 end_local_APIC_setup();
1668 }
1669
apic_read_boot_cpu_id(bool x2apic)1670 static __init void apic_read_boot_cpu_id(bool x2apic)
1671 {
1672 /*
1673 * This can be invoked from check_x2apic() before the APIC has been
1674 * selected. But that code knows for sure that the BIOS enabled
1675 * X2APIC.
1676 */
1677 if (x2apic) {
1678 boot_cpu_physical_apicid = native_apic_msr_read(APIC_ID);
1679 boot_cpu_apic_version = GET_APIC_VERSION(native_apic_msr_read(APIC_LVR));
1680 } else {
1681 boot_cpu_physical_apicid = read_apic_id();
1682 boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
1683 }
1684 topology_register_boot_apic(boot_cpu_physical_apicid);
1685 }
1686
1687 #ifdef CONFIG_X86_X2APIC
1688 int x2apic_mode;
1689 EXPORT_SYMBOL_GPL(x2apic_mode);
1690
1691 enum {
1692 X2APIC_OFF,
1693 X2APIC_DISABLED,
1694 /* All states below here have X2APIC enabled */
1695 X2APIC_ON,
1696 X2APIC_ON_LOCKED
1697 };
1698 static int x2apic_state;
1699
x2apic_hw_locked(void)1700 static bool x2apic_hw_locked(void)
1701 {
1702 u64 x86_arch_cap_msr;
1703 u64 msr;
1704
1705 x86_arch_cap_msr = x86_read_arch_cap_msr();
1706 if (x86_arch_cap_msr & ARCH_CAP_XAPIC_DISABLE) {
1707 rdmsrq(MSR_IA32_XAPIC_DISABLE_STATUS, msr);
1708 return (msr & LEGACY_XAPIC_DISABLED);
1709 }
1710 return false;
1711 }
1712
__x2apic_disable(void)1713 static void __x2apic_disable(void)
1714 {
1715 u64 msr;
1716
1717 if (!boot_cpu_has(X86_FEATURE_APIC))
1718 return;
1719
1720 rdmsrq(MSR_IA32_APICBASE, msr);
1721 if (!(msr & X2APIC_ENABLE))
1722 return;
1723 /* Disable xapic and x2apic first and then reenable xapic mode */
1724 wrmsrq(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1725 wrmsrq(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
1726 printk_once(KERN_INFO "x2apic disabled\n");
1727 }
1728
__x2apic_enable(void)1729 static void __x2apic_enable(void)
1730 {
1731 u64 msr;
1732
1733 rdmsrq(MSR_IA32_APICBASE, msr);
1734 if (msr & X2APIC_ENABLE)
1735 return;
1736 wrmsrq(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
1737 printk_once(KERN_INFO "x2apic enabled\n");
1738 }
1739
setup_nox2apic(char * str)1740 static int __init setup_nox2apic(char *str)
1741 {
1742 if (x2apic_enabled()) {
1743 u32 apicid = native_apic_msr_read(APIC_ID);
1744
1745 if (apicid >= 255) {
1746 pr_warn("Apicid: %08x, cannot enforce nox2apic\n",
1747 apicid);
1748 return 0;
1749 }
1750 if (x2apic_hw_locked()) {
1751 pr_warn("APIC locked in x2apic mode, can't disable\n");
1752 return 0;
1753 }
1754 pr_warn("x2apic already enabled.\n");
1755 __x2apic_disable();
1756 }
1757 setup_clear_cpu_cap(X86_FEATURE_X2APIC);
1758 x2apic_state = X2APIC_DISABLED;
1759 x2apic_mode = 0;
1760 return 0;
1761 }
1762 early_param("nox2apic", setup_nox2apic);
1763
1764 /* Called from cpu_init() to enable x2apic on (secondary) cpus */
x2apic_setup(void)1765 void x2apic_setup(void)
1766 {
1767 /*
1768 * Try to make the AP's APIC state match that of the BSP, but if the
1769 * BSP is unlocked and the AP is locked then there is a state mismatch.
1770 * Warn about the mismatch in case a GP fault occurs due to a locked AP
1771 * trying to be turned off.
1772 */
1773 if (x2apic_state != X2APIC_ON_LOCKED && x2apic_hw_locked())
1774 pr_warn("x2apic lock mismatch between BSP and AP.\n");
1775 /*
1776 * If x2apic is not in ON or LOCKED state, disable it if already enabled
1777 * from BIOS.
1778 */
1779 if (x2apic_state < X2APIC_ON) {
1780 __x2apic_disable();
1781 return;
1782 }
1783 __x2apic_enable();
1784 }
1785
1786 static __init void apic_set_fixmap(bool read_apic);
1787
x2apic_disable(void)1788 static __init void x2apic_disable(void)
1789 {
1790 u32 x2apic_id;
1791
1792 if (x2apic_state < X2APIC_ON)
1793 return;
1794
1795 x2apic_id = read_apic_id();
1796 if (x2apic_id >= 255)
1797 panic("Cannot disable x2apic, id: %08x\n", x2apic_id);
1798
1799 if (x2apic_hw_locked()) {
1800 pr_warn("Cannot disable locked x2apic, id: %08x\n", x2apic_id);
1801 return;
1802 }
1803
1804 __x2apic_disable();
1805
1806 x2apic_mode = 0;
1807 x2apic_state = X2APIC_DISABLED;
1808
1809 /*
1810 * Don't reread the APIC ID as it was already done from
1811 * check_x2apic() and the APIC driver still is a x2APIC variant,
1812 * which fails to do the read after x2APIC was disabled.
1813 */
1814 apic_set_fixmap(false);
1815 }
1816
x2apic_enable(void)1817 static __init void x2apic_enable(void)
1818 {
1819 if (x2apic_state != X2APIC_OFF)
1820 return;
1821
1822 x2apic_mode = 1;
1823 x2apic_state = X2APIC_ON;
1824 __x2apic_enable();
1825 }
1826
try_to_enable_x2apic(int remap_mode)1827 static __init void try_to_enable_x2apic(int remap_mode)
1828 {
1829 if (x2apic_state == X2APIC_DISABLED)
1830 return;
1831
1832 if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
1833 u32 apic_limit = 255;
1834
1835 /*
1836 * Using X2APIC without IR is not architecturally supported
1837 * on bare metal but may be supported in guests.
1838 */
1839 if (!x86_init.hyper.x2apic_available()) {
1840 pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
1841 x2apic_disable();
1842 return;
1843 }
1844
1845 /*
1846 * If the hypervisor supports extended destination ID in
1847 * MSI, that increases the maximum APIC ID that can be
1848 * used for non-remapped IRQ domains.
1849 */
1850 if (x86_init.hyper.msi_ext_dest_id()) {
1851 virt_ext_dest_id = 1;
1852 apic_limit = 32767;
1853 }
1854
1855 /*
1856 * Without IR, all CPUs can be addressed by IOAPIC/MSI only
1857 * in physical mode, and CPUs with an APIC ID that cannot
1858 * be addressed must not be brought online.
1859 */
1860 x2apic_set_max_apicid(apic_limit);
1861 x2apic_phys = 1;
1862 }
1863 x2apic_enable();
1864 }
1865
check_x2apic(void)1866 void __init check_x2apic(void)
1867 {
1868 if (x2apic_enabled()) {
1869 pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1870 x2apic_mode = 1;
1871 if (x2apic_hw_locked())
1872 x2apic_state = X2APIC_ON_LOCKED;
1873 else
1874 x2apic_state = X2APIC_ON;
1875 apic_read_boot_cpu_id(true);
1876 } else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
1877 x2apic_state = X2APIC_DISABLED;
1878 }
1879 }
1880 #else /* CONFIG_X86_X2APIC */
check_x2apic(void)1881 void __init check_x2apic(void)
1882 {
1883 if (!apic_is_x2apic_enabled())
1884 return;
1885 /*
1886 * Checkme: Can we simply turn off x2APIC here instead of disabling the APIC?
1887 */
1888 pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n");
1889 pr_err("Disabling APIC, expect reduced performance and functionality.\n");
1890
1891 apic_is_disabled = true;
1892 setup_clear_cpu_cap(X86_FEATURE_APIC);
1893 }
1894
try_to_enable_x2apic(int remap_mode)1895 static inline void try_to_enable_x2apic(int remap_mode) { }
__x2apic_enable(void)1896 static inline void __x2apic_enable(void) { }
1897 #endif /* !CONFIG_X86_X2APIC */
1898
enable_IR_x2apic(void)1899 void __init enable_IR_x2apic(void)
1900 {
1901 unsigned long flags;
1902 int ret, ir_stat;
1903
1904 if (ioapic_is_disabled) {
1905 pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1906 return;
1907 }
1908
1909 ir_stat = irq_remapping_prepare();
1910 if (ir_stat < 0 && !x2apic_supported())
1911 return;
1912
1913 ret = save_ioapic_entries();
1914 if (ret) {
1915 pr_info("Saving IO-APIC state failed: %d\n", ret);
1916 return;
1917 }
1918
1919 local_irq_save(flags);
1920 legacy_pic->mask_all();
1921 mask_ioapic_entries();
1922
1923 /* If irq_remapping_prepare() succeeded, try to enable it */
1924 if (ir_stat >= 0)
1925 ir_stat = irq_remapping_enable();
1926 /* ir_stat contains the remap mode or an error code */
1927 try_to_enable_x2apic(ir_stat);
1928
1929 if (ir_stat < 0)
1930 restore_ioapic_entries();
1931 legacy_pic->restore_mask();
1932 local_irq_restore(flags);
1933 }
1934
1935 #ifdef CONFIG_X86_64
1936 /*
1937 * Detect and enable local APICs on non-SMP boards.
1938 * Original code written by Keir Fraser.
1939 * On AMD64 we trust the BIOS - if it says no APIC it is likely
1940 * not correctly set up (usually the APIC timer won't work etc.)
1941 */
detect_init_APIC(void)1942 static bool __init detect_init_APIC(void)
1943 {
1944 if (!boot_cpu_has(X86_FEATURE_APIC)) {
1945 pr_info("No local APIC present\n");
1946 return false;
1947 }
1948
1949 register_lapic_address(APIC_DEFAULT_PHYS_BASE);
1950 return true;
1951 }
1952 #else
1953
apic_verify(unsigned long addr)1954 static bool __init apic_verify(unsigned long addr)
1955 {
1956 u32 features, h, l;
1957
1958 /*
1959 * The APIC feature bit should now be enabled
1960 * in `cpuid'
1961 */
1962 features = cpuid_edx(1);
1963 if (!(features & (1 << X86_FEATURE_APIC))) {
1964 pr_warn("Could not enable APIC!\n");
1965 return false;
1966 }
1967 set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1968
1969 /* The BIOS may have set up the APIC at some other address */
1970 if (boot_cpu_data.x86 >= 6) {
1971 rdmsr(MSR_IA32_APICBASE, l, h);
1972 if (l & MSR_IA32_APICBASE_ENABLE)
1973 addr = l & MSR_IA32_APICBASE_BASE;
1974 }
1975
1976 register_lapic_address(addr);
1977 pr_info("Found and enabled local APIC!\n");
1978 return true;
1979 }
1980
apic_force_enable(unsigned long addr)1981 bool __init apic_force_enable(unsigned long addr)
1982 {
1983 u32 h, l;
1984
1985 if (apic_is_disabled)
1986 return false;
1987
1988 /*
1989 * Some BIOSes disable the local APIC in the APIC_BASE
1990 * MSR. This can only be done in software for Intel P6 or later
1991 * and AMD K7 (Model > 1) or later.
1992 */
1993 if (boot_cpu_data.x86 >= 6) {
1994 rdmsr(MSR_IA32_APICBASE, l, h);
1995 if (!(l & MSR_IA32_APICBASE_ENABLE)) {
1996 pr_info("Local APIC disabled by BIOS -- reenabling.\n");
1997 l &= ~MSR_IA32_APICBASE_BASE;
1998 l |= MSR_IA32_APICBASE_ENABLE | addr;
1999 wrmsr(MSR_IA32_APICBASE, l, h);
2000 enabled_via_apicbase = 1;
2001 }
2002 }
2003 return apic_verify(addr);
2004 }
2005
2006 /*
2007 * Detect and initialize APIC
2008 */
detect_init_APIC(void)2009 static bool __init detect_init_APIC(void)
2010 {
2011 /* Disabled by kernel option? */
2012 if (apic_is_disabled)
2013 return false;
2014
2015 switch (boot_cpu_data.x86_vendor) {
2016 case X86_VENDOR_AMD:
2017 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
2018 (boot_cpu_data.x86 >= 15))
2019 break;
2020 goto no_apic;
2021 case X86_VENDOR_HYGON:
2022 break;
2023 case X86_VENDOR_INTEL:
2024 if ((boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)) ||
2025 boot_cpu_data.x86_vfm >= INTEL_PENTIUM_PRO)
2026 break;
2027 goto no_apic;
2028 default:
2029 goto no_apic;
2030 }
2031
2032 if (!boot_cpu_has(X86_FEATURE_APIC)) {
2033 /*
2034 * Over-ride BIOS and try to enable the local APIC only if
2035 * "lapic" specified.
2036 */
2037 if (!force_enable_local_apic) {
2038 pr_info("Local APIC disabled by BIOS -- "
2039 "you can enable it with \"lapic\"\n");
2040 return false;
2041 }
2042 if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE))
2043 return false;
2044 } else {
2045 if (!apic_verify(APIC_DEFAULT_PHYS_BASE))
2046 return false;
2047 }
2048
2049 apic_pm_activate();
2050
2051 return true;
2052
2053 no_apic:
2054 pr_info("No local APIC present or hardware disabled\n");
2055 return false;
2056 }
2057 #endif
2058
2059 /**
2060 * init_apic_mappings - initialize APIC mappings
2061 */
init_apic_mappings(void)2062 void __init init_apic_mappings(void)
2063 {
2064 if (apic_validate_deadline_timer())
2065 pr_info("TSC deadline timer available\n");
2066
2067 if (x2apic_mode)
2068 return;
2069
2070 if (!smp_found_config) {
2071 if (!detect_init_APIC()) {
2072 pr_info("APIC: disable apic facility\n");
2073 apic_disable();
2074 }
2075 }
2076 }
2077
apic_set_fixmap(bool read_apic)2078 static __init void apic_set_fixmap(bool read_apic)
2079 {
2080 set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
2081 apic_mmio_base = APIC_BASE;
2082 apic_pr_verbose("Mapped APIC to %16lx (%16lx)\n", apic_mmio_base, mp_lapic_addr);
2083 if (read_apic)
2084 apic_read_boot_cpu_id(false);
2085 }
2086
register_lapic_address(unsigned long address)2087 void __init register_lapic_address(unsigned long address)
2088 {
2089 /* This should only happen once */
2090 WARN_ON_ONCE(mp_lapic_addr);
2091 mp_lapic_addr = address;
2092
2093 if (!x2apic_mode)
2094 apic_set_fixmap(true);
2095 }
2096
2097 /*
2098 * Local APIC interrupts
2099 */
2100
2101 /*
2102 * Common handling code for spurious_interrupt and spurious_vector entry
2103 * points below. No point in allowing the compiler to inline it twice.
2104 */
handle_spurious_interrupt(u8 vector)2105 static noinline void handle_spurious_interrupt(u8 vector)
2106 {
2107 u32 v;
2108
2109 trace_spurious_apic_entry(vector);
2110
2111 inc_irq_stat(irq_spurious_count);
2112
2113 /*
2114 * If this is a spurious interrupt then do not acknowledge
2115 */
2116 if (vector == SPURIOUS_APIC_VECTOR) {
2117 /* See SDM vol 3 */
2118 pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n",
2119 smp_processor_id());
2120 goto out;
2121 }
2122
2123 /*
2124 * If it is a vectored one, verify it's set in the ISR. If set,
2125 * acknowledge it.
2126 */
2127 v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
2128 if (v & (1 << (vector & 0x1f))) {
2129 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
2130 vector, smp_processor_id());
2131 apic_eoi();
2132 } else {
2133 pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
2134 vector, smp_processor_id());
2135 }
2136 out:
2137 trace_spurious_apic_exit(vector);
2138 }
2139
2140 /**
2141 * spurious_interrupt - Catch all for interrupts raised on unused vectors
2142 * @regs: Pointer to pt_regs on stack
2143 * @vector: The vector number
2144 *
2145 * This is invoked from ASM entry code to catch all interrupts which
2146 * trigger on an entry which is routed to the common_spurious idtentry
2147 * point.
2148 */
DEFINE_IDTENTRY_IRQ(spurious_interrupt)2149 DEFINE_IDTENTRY_IRQ(spurious_interrupt)
2150 {
2151 handle_spurious_interrupt(vector);
2152 }
2153
DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)2154 DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
2155 {
2156 handle_spurious_interrupt(SPURIOUS_APIC_VECTOR);
2157 }
2158
2159 /*
2160 * This interrupt should never happen with our APIC/SMP architecture
2161 */
DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)2162 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
2163 {
2164 static const char * const error_interrupt_reason[] = {
2165 "Send CS error", /* APIC Error Bit 0 */
2166 "Receive CS error", /* APIC Error Bit 1 */
2167 "Send accept error", /* APIC Error Bit 2 */
2168 "Receive accept error", /* APIC Error Bit 3 */
2169 "Redirectable IPI", /* APIC Error Bit 4 */
2170 "Send illegal vector", /* APIC Error Bit 5 */
2171 "Received illegal vector", /* APIC Error Bit 6 */
2172 "Illegal register address", /* APIC Error Bit 7 */
2173 };
2174 u32 v, i = 0;
2175
2176 trace_error_apic_entry(ERROR_APIC_VECTOR);
2177
2178 /* First tickle the hardware, only then report what went on. -- REW */
2179 if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
2180 apic_write(APIC_ESR, 0);
2181 v = apic_read(APIC_ESR);
2182 apic_eoi();
2183 atomic_inc(&irq_err_count);
2184
2185 apic_pr_debug("APIC error on CPU%d: %02x", smp_processor_id(), v);
2186
2187 v &= 0xff;
2188 while (v) {
2189 if (v & 0x1)
2190 apic_pr_debug_cont(" : %s", error_interrupt_reason[i]);
2191 i++;
2192 v >>= 1;
2193 }
2194
2195 apic_pr_debug_cont("\n");
2196
2197 trace_error_apic_exit(ERROR_APIC_VECTOR);
2198 }
2199
2200 /**
2201 * connect_bsp_APIC - attach the APIC to the interrupt system
2202 */
connect_bsp_APIC(void)2203 static void __init connect_bsp_APIC(void)
2204 {
2205 #ifdef CONFIG_X86_32
2206 if (pic_mode) {
2207 /*
2208 * Do not trust the local APIC being empty at bootup.
2209 */
2210 clear_local_APIC();
2211 /*
2212 * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
2213 * local APIC to INT and NMI lines.
2214 */
2215 apic_pr_verbose("Leaving PIC mode, enabling APIC mode.\n");
2216 imcr_pic_to_apic();
2217 }
2218 #endif
2219 }
2220
2221 /**
2222 * disconnect_bsp_APIC - detach the APIC from the interrupt system
2223 * @virt_wire_setup: indicates, whether virtual wire mode is selected
2224 *
2225 * Virtual wire mode is necessary to deliver legacy interrupts even when the
2226 * APIC is disabled.
2227 */
disconnect_bsp_APIC(int virt_wire_setup)2228 void disconnect_bsp_APIC(int virt_wire_setup)
2229 {
2230 unsigned int value;
2231
2232 #ifdef CONFIG_X86_32
2233 if (pic_mode) {
2234 /*
2235 * Put the board back into PIC mode (has an effect only on
2236 * certain older boards). Note that APIC interrupts, including
2237 * IPIs, won't work beyond this point! The only exception are
2238 * INIT IPIs.
2239 */
2240 apic_pr_verbose("Disabling APIC mode, entering PIC mode.\n");
2241 imcr_apic_to_pic();
2242 return;
2243 }
2244 #endif
2245
2246 /* Go back to Virtual Wire compatibility mode */
2247
2248 /* For the spurious interrupt use vector F, and enable it */
2249 value = apic_read(APIC_SPIV);
2250 value &= ~APIC_VECTOR_MASK;
2251 value |= APIC_SPIV_APIC_ENABLED;
2252 value |= 0xf;
2253 apic_write(APIC_SPIV, value);
2254
2255 if (!virt_wire_setup) {
2256 /*
2257 * For LVT0 make it edge triggered, active high,
2258 * external and enabled
2259 */
2260 value = apic_read(APIC_LVT0);
2261 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2262 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2263 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2264 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2265 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2266 apic_write(APIC_LVT0, value);
2267 } else {
2268 /* Disable LVT0 */
2269 apic_write(APIC_LVT0, APIC_LVT_MASKED);
2270 }
2271
2272 /*
2273 * For LVT1 make it edge triggered, active high,
2274 * nmi and enabled
2275 */
2276 value = apic_read(APIC_LVT1);
2277 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2278 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2279 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2280 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2281 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2282 apic_write(APIC_LVT1, value);
2283 }
2284
__irq_msi_compose_msg(struct irq_cfg * cfg,struct msi_msg * msg,bool dmar)2285 void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg,
2286 bool dmar)
2287 {
2288 memset(msg, 0, sizeof(*msg));
2289
2290 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
2291 msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical;
2292 msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF;
2293
2294 msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED;
2295 msg->arch_data.vector = cfg->vector;
2296
2297 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
2298 /*
2299 * Only the IOMMU itself can use the trick of putting destination
2300 * APIC ID into the high bits of the address. Anything else would
2301 * just be writing to memory if it tried that, and needs IR to
2302 * address APICs which can't be addressed in the normal 32-bit
2303 * address range at 0xFFExxxxx. That is typically just 8 bits, but
2304 * some hypervisors allow the extended destination ID field in bits
2305 * 5-11 to be used, giving support for 15 bits of APIC IDs in total.
2306 */
2307 if (dmar)
2308 msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8;
2309 else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000)
2310 msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8;
2311 else
2312 WARN_ON_ONCE(cfg->dest_apicid > 0xFF);
2313 }
2314
x86_msi_msg_get_destid(struct msi_msg * msg,bool extid)2315 u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid)
2316 {
2317 u32 dest = msg->arch_addr_lo.destid_0_7;
2318
2319 if (extid)
2320 dest |= msg->arch_addr_hi.destid_8_31 << 8;
2321 return dest;
2322 }
2323 EXPORT_SYMBOL_FOR_KVM(x86_msi_msg_get_destid);
2324
apic_bsp_up_setup(void)2325 static void __init apic_bsp_up_setup(void)
2326 {
2327 reset_phys_cpu_present_map(boot_cpu_physical_apicid);
2328 }
2329
2330 /**
2331 * apic_bsp_setup - Setup function for local apic and io-apic
2332 * @upmode: Force UP mode (for APIC_init_uniprocessor)
2333 */
apic_bsp_setup(bool upmode)2334 static void __init apic_bsp_setup(bool upmode)
2335 {
2336 connect_bsp_APIC();
2337 if (upmode)
2338 apic_bsp_up_setup();
2339 setup_local_APIC();
2340
2341 enable_IO_APIC();
2342 end_local_APIC_setup();
2343 irq_remap_enable_fault_handling();
2344 setup_IO_APIC();
2345 lapic_update_legacy_vectors();
2346 }
2347
2348 #ifdef CONFIG_UP_LATE_INIT
up_late_init(void)2349 void __init up_late_init(void)
2350 {
2351 if (apic_intr_mode == APIC_PIC)
2352 return;
2353
2354 /* Setup local timer */
2355 x86_init.timers.setup_percpu_clockev();
2356 }
2357 #endif
2358
2359 /*
2360 * Power management
2361 */
2362 #ifdef CONFIG_PM
2363
2364 static struct {
2365 /*
2366 * 'active' is true if the local APIC was enabled by us and
2367 * not the BIOS; this signifies that we are also responsible
2368 * for disabling it before entering apm/acpi suspend
2369 */
2370 int active;
2371 /* r/w apic fields */
2372 u32 apic_id;
2373 unsigned int apic_taskpri;
2374 unsigned int apic_ldr;
2375 unsigned int apic_dfr;
2376 unsigned int apic_spiv;
2377 unsigned int apic_lvtt;
2378 unsigned int apic_lvtpc;
2379 unsigned int apic_lvt0;
2380 unsigned int apic_lvt1;
2381 unsigned int apic_lvterr;
2382 unsigned int apic_tmict;
2383 unsigned int apic_tdcr;
2384 unsigned int apic_thmr;
2385 unsigned int apic_cmci;
2386 } apic_pm_state;
2387
lapic_suspend(void * data)2388 static int lapic_suspend(void *data)
2389 {
2390 unsigned long flags;
2391 int maxlvt;
2392
2393 if (!apic_pm_state.active)
2394 return 0;
2395
2396 maxlvt = lapic_get_maxlvt();
2397
2398 apic_pm_state.apic_id = apic_read(APIC_ID);
2399 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2400 apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2401 apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2402 apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2403 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2404 if (maxlvt >= 4)
2405 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2406 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2407 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2408 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2409 apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2410 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
2411 #ifdef CONFIG_X86_THERMAL_VECTOR
2412 if (maxlvt >= 5)
2413 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2414 #endif
2415 #ifdef CONFIG_X86_MCE_INTEL
2416 if (maxlvt >= 6)
2417 apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI);
2418 #endif
2419
2420 local_irq_save(flags);
2421
2422 /*
2423 * Mask IOAPIC before disabling the local APIC to prevent stale IRR
2424 * entries on some implementations.
2425 */
2426 mask_ioapic_entries();
2427
2428 disable_local_APIC();
2429
2430 irq_remapping_disable();
2431
2432 local_irq_restore(flags);
2433 return 0;
2434 }
2435
lapic_resume(void * data)2436 static void lapic_resume(void *data)
2437 {
2438 unsigned int l, h;
2439 unsigned long flags;
2440 int maxlvt;
2441
2442 if (!apic_pm_state.active)
2443 return;
2444
2445 local_irq_save(flags);
2446
2447 /*
2448 * IO-APIC and PIC have their own resume routines.
2449 * We just mask them here to make sure the interrupt
2450 * subsystem is completely quiet while we enable x2apic
2451 * and interrupt-remapping.
2452 */
2453 mask_ioapic_entries();
2454 legacy_pic->mask_all();
2455
2456 if (x2apic_mode) {
2457 __x2apic_enable();
2458 } else {
2459 /*
2460 * Make sure the APICBASE points to the right address
2461 *
2462 * FIXME! This will be wrong if we ever support suspend on
2463 * SMP! We'll need to do this as part of the CPU restore!
2464 */
2465 if (boot_cpu_data.x86 >= 6) {
2466 rdmsr(MSR_IA32_APICBASE, l, h);
2467 l &= ~MSR_IA32_APICBASE_BASE;
2468 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2469 wrmsr(MSR_IA32_APICBASE, l, h);
2470 }
2471 }
2472
2473 maxlvt = lapic_get_maxlvt();
2474 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2475 apic_write(APIC_ID, apic_pm_state.apic_id);
2476 apic_write(APIC_DFR, apic_pm_state.apic_dfr);
2477 apic_write(APIC_LDR, apic_pm_state.apic_ldr);
2478 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
2479 apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
2480 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
2481 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
2482 #ifdef CONFIG_X86_THERMAL_VECTOR
2483 if (maxlvt >= 5)
2484 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
2485 #endif
2486 #ifdef CONFIG_X86_MCE_INTEL
2487 if (maxlvt >= 6)
2488 apic_write(APIC_LVTCMCI, apic_pm_state.apic_cmci);
2489 #endif
2490 if (maxlvt >= 4)
2491 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
2492 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
2493 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
2494 apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
2495 apic_write(APIC_ESR, 0);
2496 apic_read(APIC_ESR);
2497 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
2498 apic_write(APIC_ESR, 0);
2499 apic_read(APIC_ESR);
2500
2501 irq_remapping_reenable(x2apic_mode);
2502
2503 local_irq_restore(flags);
2504 }
2505
2506 /*
2507 * This device has no shutdown method - fully functioning local APICs
2508 * are needed on every CPU up until machine_halt/restart/poweroff.
2509 */
2510
2511 static const struct syscore_ops lapic_syscore_ops = {
2512 .resume = lapic_resume,
2513 .suspend = lapic_suspend,
2514 };
2515
2516 static struct syscore lapic_syscore = {
2517 .ops = &lapic_syscore_ops,
2518 };
2519
apic_pm_activate(void)2520 static void apic_pm_activate(void)
2521 {
2522 apic_pm_state.active = 1;
2523 }
2524
init_lapic_sysfs(void)2525 static int __init init_lapic_sysfs(void)
2526 {
2527 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2528 if (boot_cpu_has(X86_FEATURE_APIC))
2529 register_syscore(&lapic_syscore);
2530
2531 return 0;
2532 }
2533
2534 /* local apic needs to resume before other devices access its registers. */
2535 core_initcall(init_lapic_sysfs);
2536
2537 #else /* CONFIG_PM */
2538
apic_pm_activate(void)2539 static void apic_pm_activate(void) { }
2540
2541 #endif /* CONFIG_PM */
2542
2543 #ifdef CONFIG_X86_64
2544
2545 static int multi_checked;
2546 static int multi;
2547
set_multi(const struct dmi_system_id * d)2548 static int set_multi(const struct dmi_system_id *d)
2549 {
2550 if (multi)
2551 return 0;
2552 pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2553 multi = 1;
2554 return 0;
2555 }
2556
2557 static const struct dmi_system_id multi_dmi_table[] = {
2558 {
2559 .callback = set_multi,
2560 .ident = "IBM System Summit2",
2561 .matches = {
2562 DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2563 DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2564 },
2565 },
2566 {}
2567 };
2568
dmi_check_multi(void)2569 static void dmi_check_multi(void)
2570 {
2571 if (multi_checked)
2572 return;
2573
2574 dmi_check_system(multi_dmi_table);
2575 multi_checked = 1;
2576 }
2577
2578 /*
2579 * apic_is_clustered_box() -- Check if we can expect good TSC
2580 *
2581 * Thus far, the major user of this is IBM's Summit2 series:
2582 * Clustered boxes may have unsynced TSC problems if they are
2583 * multi-chassis.
2584 * Use DMI to check them
2585 */
apic_is_clustered_box(void)2586 int apic_is_clustered_box(void)
2587 {
2588 dmi_check_multi();
2589 return multi;
2590 }
2591 #endif
2592
2593 /*
2594 * APIC command line parameters
2595 */
setup_nolapic(char * arg)2596 static int __init setup_nolapic(char *arg)
2597 {
2598 apic_is_disabled = true;
2599 setup_clear_cpu_cap(X86_FEATURE_APIC);
2600 return 0;
2601 }
2602 early_param("nolapic", setup_nolapic);
2603
parse_lapic_timer_c2_ok(char * arg)2604 static int __init parse_lapic_timer_c2_ok(char *arg)
2605 {
2606 local_apic_timer_c2_ok = 1;
2607 return 0;
2608 }
2609 early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2610
parse_disable_apic_timer(char * arg)2611 static int __init parse_disable_apic_timer(char *arg)
2612 {
2613 disable_apic_timer = 1;
2614 return 0;
2615 }
2616 early_param("noapictimer", parse_disable_apic_timer);
2617
parse_nolapic_timer(char * arg)2618 static int __init parse_nolapic_timer(char *arg)
2619 {
2620 disable_apic_timer = 1;
2621 return 0;
2622 }
2623 early_param("nolapic_timer", parse_nolapic_timer);
2624
apic_set_verbosity(char * arg)2625 static int __init apic_set_verbosity(char *arg)
2626 {
2627 if (!arg) {
2628 if (IS_ENABLED(CONFIG_X86_32))
2629 return -EINVAL;
2630
2631 ioapic_is_disabled = false;
2632 return 0;
2633 }
2634
2635 if (strcmp("debug", arg) == 0)
2636 apic_verbosity = APIC_DEBUG;
2637 else if (strcmp("verbose", arg) == 0)
2638 apic_verbosity = APIC_VERBOSE;
2639 #ifdef CONFIG_X86_64
2640 else {
2641 pr_warn("APIC Verbosity level %s not recognised"
2642 " use apic=verbose or apic=debug\n", arg);
2643 return -EINVAL;
2644 }
2645 #endif
2646
2647 return 0;
2648 }
2649 early_param("apic", apic_set_verbosity);
2650
lapic_insert_resource(void)2651 static int __init lapic_insert_resource(void)
2652 {
2653 if (!apic_mmio_base)
2654 return -1;
2655
2656 /* Put local APIC into the resource map. */
2657 lapic_resource.start = apic_mmio_base;
2658 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2659 insert_resource(&iomem_resource, &lapic_resource);
2660
2661 return 0;
2662 }
2663
2664 /*
2665 * need call insert after e820__reserve_resources()
2666 * that is using request_resource
2667 */
2668 late_initcall(lapic_insert_resource);
2669
apic_set_extnmi(char * arg)2670 static int __init apic_set_extnmi(char *arg)
2671 {
2672 if (!arg)
2673 return -EINVAL;
2674
2675 if (!strncmp("all", arg, 3))
2676 apic_extnmi = APIC_EXTNMI_ALL;
2677 else if (!strncmp("none", arg, 4))
2678 apic_extnmi = APIC_EXTNMI_NONE;
2679 else if (!strncmp("bsp", arg, 3))
2680 apic_extnmi = APIC_EXTNMI_BSP;
2681 else {
2682 pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg);
2683 return -EINVAL;
2684 }
2685
2686 return 0;
2687 }
2688 early_param("apic_extnmi", apic_set_extnmi);
2689