1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 #ifndef _ASM_X86_APIC_H 3 #define _ASM_X86_APIC_H 4 5 #include <linux/cpumask.h> 6 #include <linux/static_call.h> 7 8 #include <asm/alternative.h> 9 #include <asm/cpufeature.h> 10 #include <asm/apicdef.h> 11 #include <linux/atomic.h> 12 #include <asm/fixmap.h> 13 #include <asm/mpspec.h> 14 #include <asm/msr.h> 15 #include <asm/hardirq.h> 16 17 #define ARCH_APICTIMER_STOPS_ON_C3 1 18 19 /* 20 * Debugging macros 21 */ 22 #define APIC_QUIET 0 23 #define APIC_VERBOSE 1 24 #define APIC_DEBUG 2 25 26 /* Macros for apic_extnmi which controls external NMI masking */ 27 #define APIC_EXTNMI_BSP 0 /* Default */ 28 #define APIC_EXTNMI_ALL 1 29 #define APIC_EXTNMI_NONE 2 30 31 /* 32 * Define the default level of output to be very little 33 * This can be turned up by using apic=verbose for more 34 * information and apic=debug for _lots_ of information. 35 * apic_verbosity is defined in apic.c 36 */ 37 #define apic_printk(v, s, a...) do { \ 38 if ((v) <= apic_verbosity) \ 39 printk(s, ##a); \ 40 } while (0) 41 42 43 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32) 44 extern void x86_32_probe_apic(void); 45 #else 46 static inline void x86_32_probe_apic(void) { } 47 #endif 48 49 extern u32 cpuid_to_apicid[]; 50 51 #define CPU_ACPIID_INVALID U32_MAX 52 53 #ifdef CONFIG_X86_LOCAL_APIC 54 55 extern int apic_verbosity; 56 extern int local_apic_timer_c2_ok; 57 58 extern bool apic_is_disabled; 59 extern unsigned int lapic_timer_period; 60 61 extern enum apic_intr_mode_id apic_intr_mode; 62 enum apic_intr_mode_id { 63 APIC_PIC, 64 APIC_VIRTUAL_WIRE, 65 APIC_VIRTUAL_WIRE_NO_CONFIG, 66 APIC_SYMMETRIC_IO, 67 APIC_SYMMETRIC_IO_NO_ROUTING 68 }; 69 70 /* 71 * With 82489DX we can't rely on apic feature bit 72 * retrieved via cpuid but still have to deal with 73 * such an apic chip so we assume that SMP configuration 74 * is found from MP table (64bit case uses ACPI mostly 75 * which set smp presence flag as well so we are safe 76 * to use this helper too). 77 */ 78 static inline bool apic_from_smp_config(void) 79 { 80 return smp_found_config && !apic_is_disabled; 81 } 82 83 /* 84 * Basic functions accessing APICs. 85 */ 86 #ifdef CONFIG_PARAVIRT 87 #include <asm/paravirt.h> 88 #endif 89 90 static inline void native_apic_mem_write(u32 reg, u32 v) 91 { 92 volatile u32 *addr = (volatile u32 *)(APIC_BASE + reg); 93 94 alternative_io("movl %0, %P1", "xchgl %0, %P1", X86_BUG_11AP, 95 ASM_OUTPUT2("=r" (v), "=m" (*addr)), 96 ASM_OUTPUT2("0" (v), "m" (*addr))); 97 } 98 99 static inline u32 native_apic_mem_read(u32 reg) 100 { 101 return *((volatile u32 *)(APIC_BASE + reg)); 102 } 103 104 static inline void native_apic_mem_eoi(void) 105 { 106 native_apic_mem_write(APIC_EOI, APIC_EOI_ACK); 107 } 108 109 extern void native_apic_icr_write(u32 low, u32 id); 110 extern u64 native_apic_icr_read(void); 111 112 static inline bool apic_is_x2apic_enabled(void) 113 { 114 u64 msr; 115 116 if (rdmsrl_safe(MSR_IA32_APICBASE, &msr)) 117 return false; 118 return msr & X2APIC_ENABLE; 119 } 120 121 extern void enable_IR_x2apic(void); 122 123 extern int get_physical_broadcast(void); 124 125 extern int lapic_get_maxlvt(void); 126 extern void clear_local_APIC(void); 127 extern void disconnect_bsp_APIC(int virt_wire_setup); 128 extern void disable_local_APIC(void); 129 extern void apic_soft_disable(void); 130 extern void lapic_shutdown(void); 131 extern void sync_Arb_IDs(void); 132 extern void init_bsp_APIC(void); 133 extern void apic_intr_mode_select(void); 134 extern void apic_intr_mode_init(void); 135 extern void init_apic_mappings(void); 136 void register_lapic_address(unsigned long address); 137 extern void setup_boot_APIC_clock(void); 138 extern void setup_secondary_APIC_clock(void); 139 extern void lapic_update_tsc_freq(void); 140 141 #ifdef CONFIG_X86_64 142 static inline bool apic_force_enable(unsigned long addr) 143 { 144 return false; 145 } 146 #else 147 extern bool apic_force_enable(unsigned long addr); 148 #endif 149 150 extern void apic_ap_setup(void); 151 152 /* 153 * On 32bit this is mach-xxx local 154 */ 155 #ifdef CONFIG_X86_64 156 extern int apic_is_clustered_box(void); 157 #else 158 static inline int apic_is_clustered_box(void) 159 { 160 return 0; 161 } 162 #endif 163 164 extern int setup_APIC_eilvt(u8 lvt_off, u8 vector, u8 msg_type, u8 mask); 165 extern void lapic_assign_system_vectors(void); 166 extern void lapic_assign_legacy_vector(unsigned int isairq, bool replace); 167 extern void lapic_update_legacy_vectors(void); 168 extern void lapic_online(void); 169 extern void lapic_offline(void); 170 extern bool apic_needs_pit(void); 171 172 extern void apic_send_IPI_allbutself(unsigned int vector); 173 174 extern void topology_register_apic(u32 apic_id, u32 acpi_id, bool present); 175 extern void topology_register_boot_apic(u32 apic_id); 176 extern int topology_hotplug_apic(u32 apic_id, u32 acpi_id); 177 extern void topology_hotunplug_apic(unsigned int cpu); 178 extern void topology_apply_cmdline_limits_early(void); 179 extern void topology_init_possible_cpus(void); 180 extern void topology_reset_possible_cpus_up(void); 181 182 #else /* !CONFIG_X86_LOCAL_APIC */ 183 static inline void lapic_shutdown(void) { } 184 #define local_apic_timer_c2_ok 1 185 static inline void init_apic_mappings(void) { } 186 static inline void disable_local_APIC(void) { } 187 # define setup_boot_APIC_clock x86_init_noop 188 # define setup_secondary_APIC_clock x86_init_noop 189 static inline void lapic_update_tsc_freq(void) { } 190 static inline void init_bsp_APIC(void) { } 191 static inline void apic_intr_mode_select(void) { } 192 static inline void apic_intr_mode_init(void) { } 193 static inline void lapic_assign_system_vectors(void) { } 194 static inline void lapic_assign_legacy_vector(unsigned int i, bool r) { } 195 static inline bool apic_needs_pit(void) { return true; } 196 static inline void topology_apply_cmdline_limits_early(void) { } 197 static inline void topology_init_possible_cpus(void) { } 198 #endif /* !CONFIG_X86_LOCAL_APIC */ 199 200 #ifdef CONFIG_X86_X2APIC 201 static inline void native_apic_msr_write(u32 reg, u32 v) 202 { 203 if (reg == APIC_DFR || reg == APIC_ID || reg == APIC_LDR || 204 reg == APIC_LVR) 205 return; 206 207 wrmsr(APIC_BASE_MSR + (reg >> 4), v, 0); 208 } 209 210 static inline void native_apic_msr_eoi(void) 211 { 212 __wrmsr(APIC_BASE_MSR + (APIC_EOI >> 4), APIC_EOI_ACK, 0); 213 } 214 215 static inline u32 native_apic_msr_read(u32 reg) 216 { 217 u64 msr; 218 219 if (reg == APIC_DFR) 220 return -1; 221 222 rdmsrl(APIC_BASE_MSR + (reg >> 4), msr); 223 return (u32)msr; 224 } 225 226 static inline void native_x2apic_icr_write(u32 low, u32 id) 227 { 228 wrmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low); 229 } 230 231 static inline u64 native_x2apic_icr_read(void) 232 { 233 unsigned long val; 234 235 rdmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), val); 236 return val; 237 } 238 239 extern int x2apic_mode; 240 extern int x2apic_phys; 241 extern void __init x2apic_set_max_apicid(u32 apicid); 242 extern void x2apic_setup(void); 243 static inline int x2apic_enabled(void) 244 { 245 return boot_cpu_has(X86_FEATURE_X2APIC) && apic_is_x2apic_enabled(); 246 } 247 248 #define x2apic_supported() (boot_cpu_has(X86_FEATURE_X2APIC)) 249 #else /* !CONFIG_X86_X2APIC */ 250 static inline void x2apic_setup(void) { } 251 static inline int x2apic_enabled(void) { return 0; } 252 static inline u32 native_apic_msr_read(u32 reg) { BUG(); } 253 #define x2apic_mode (0) 254 #define x2apic_supported() (0) 255 #endif /* !CONFIG_X86_X2APIC */ 256 extern void __init check_x2apic(void); 257 258 struct irq_data; 259 260 /* 261 * Copyright 2004 James Cleverdon, IBM. 262 * 263 * Generic APIC sub-arch data struct. 264 * 265 * Hacked for x86-64 by James Cleverdon from i386 architecture code by 266 * Martin Bligh, Andi Kleen, James Bottomley, John Stultz, and 267 * James Cleverdon. 268 */ 269 struct apic { 270 /* Hotpath functions first */ 271 void (*eoi)(void); 272 void (*native_eoi)(void); 273 void (*write)(u32 reg, u32 v); 274 u32 (*read)(u32 reg); 275 276 /* IPI related functions */ 277 void (*wait_icr_idle)(void); 278 u32 (*safe_wait_icr_idle)(void); 279 280 void (*send_IPI)(int cpu, int vector); 281 void (*send_IPI_mask)(const struct cpumask *mask, int vector); 282 void (*send_IPI_mask_allbutself)(const struct cpumask *msk, int vec); 283 void (*send_IPI_allbutself)(int vector); 284 void (*send_IPI_all)(int vector); 285 void (*send_IPI_self)(int vector); 286 287 u32 disable_esr : 1, 288 dest_mode_logical : 1, 289 x2apic_set_max_apicid : 1, 290 nmi_to_offline_cpu : 1; 291 292 u32 (*calc_dest_apicid)(unsigned int cpu); 293 294 /* ICR related functions */ 295 u64 (*icr_read)(void); 296 void (*icr_write)(u32 low, u32 high); 297 298 /* The limit of the APIC ID space. */ 299 u32 max_apic_id; 300 301 /* Probe, setup and smpboot functions */ 302 int (*probe)(void); 303 int (*acpi_madt_oem_check)(char *oem_id, char *oem_table_id); 304 305 void (*init_apic_ldr)(void); 306 u32 (*cpu_present_to_apicid)(int mps_cpu); 307 308 u32 (*get_apic_id)(u32 id); 309 310 /* wakeup_secondary_cpu */ 311 int (*wakeup_secondary_cpu)(u32 apicid, unsigned long start_eip); 312 /* wakeup secondary CPU using 64-bit wakeup point */ 313 int (*wakeup_secondary_cpu_64)(u32 apicid, unsigned long start_eip); 314 315 char *name; 316 }; 317 318 struct apic_override { 319 void (*eoi)(void); 320 void (*native_eoi)(void); 321 void (*write)(u32 reg, u32 v); 322 u32 (*read)(u32 reg); 323 void (*send_IPI)(int cpu, int vector); 324 void (*send_IPI_mask)(const struct cpumask *mask, int vector); 325 void (*send_IPI_mask_allbutself)(const struct cpumask *msk, int vec); 326 void (*send_IPI_allbutself)(int vector); 327 void (*send_IPI_all)(int vector); 328 void (*send_IPI_self)(int vector); 329 u64 (*icr_read)(void); 330 void (*icr_write)(u32 low, u32 high); 331 int (*wakeup_secondary_cpu)(u32 apicid, unsigned long start_eip); 332 int (*wakeup_secondary_cpu_64)(u32 apicid, unsigned long start_eip); 333 }; 334 335 /* 336 * Pointer to the local APIC driver in use on this system (there's 337 * always just one such driver in use - the kernel decides via an 338 * early probing process which one it picks - and then sticks to it): 339 */ 340 extern struct apic *apic; 341 342 /* 343 * APIC drivers are probed based on how they are listed in the .apicdrivers 344 * section. So the order is important and enforced by the ordering 345 * of different apic driver files in the Makefile. 346 * 347 * For the files having two apic drivers, we use apic_drivers() 348 * to enforce the order with in them. 349 */ 350 #define apic_driver(sym) \ 351 static const struct apic *__apicdrivers_##sym __used \ 352 __aligned(sizeof(struct apic *)) \ 353 __section(".apicdrivers") = { &sym } 354 355 #define apic_drivers(sym1, sym2) \ 356 static struct apic *__apicdrivers_##sym1##sym2[2] __used \ 357 __aligned(sizeof(struct apic *)) \ 358 __section(".apicdrivers") = { &sym1, &sym2 } 359 360 extern struct apic *__apicdrivers[], *__apicdrivers_end[]; 361 362 /* 363 * APIC functionality to boot other CPUs - only used on SMP: 364 */ 365 #ifdef CONFIG_SMP 366 extern int lapic_can_unplug_cpu(void); 367 #endif 368 369 #ifdef CONFIG_X86_LOCAL_APIC 370 extern struct apic_override __x86_apic_override; 371 372 void __init apic_setup_apic_calls(void); 373 void __init apic_install_driver(struct apic *driver); 374 375 #define apic_update_callback(_callback, _fn) { \ 376 __x86_apic_override._callback = _fn; \ 377 apic->_callback = _fn; \ 378 static_call_update(apic_call_##_callback, _fn); \ 379 pr_info("APIC: %s() replaced with %ps()\n", #_callback, _fn); \ 380 } 381 382 #define DECLARE_APIC_CALL(__cb) \ 383 DECLARE_STATIC_CALL(apic_call_##__cb, *apic->__cb) 384 385 DECLARE_APIC_CALL(eoi); 386 DECLARE_APIC_CALL(native_eoi); 387 DECLARE_APIC_CALL(icr_read); 388 DECLARE_APIC_CALL(icr_write); 389 DECLARE_APIC_CALL(read); 390 DECLARE_APIC_CALL(send_IPI); 391 DECLARE_APIC_CALL(send_IPI_mask); 392 DECLARE_APIC_CALL(send_IPI_mask_allbutself); 393 DECLARE_APIC_CALL(send_IPI_allbutself); 394 DECLARE_APIC_CALL(send_IPI_all); 395 DECLARE_APIC_CALL(send_IPI_self); 396 DECLARE_APIC_CALL(wait_icr_idle); 397 DECLARE_APIC_CALL(wakeup_secondary_cpu); 398 DECLARE_APIC_CALL(wakeup_secondary_cpu_64); 399 DECLARE_APIC_CALL(write); 400 401 static __always_inline u32 apic_read(u32 reg) 402 { 403 return static_call(apic_call_read)(reg); 404 } 405 406 static __always_inline void apic_write(u32 reg, u32 val) 407 { 408 static_call(apic_call_write)(reg, val); 409 } 410 411 static __always_inline void apic_eoi(void) 412 { 413 static_call(apic_call_eoi)(); 414 } 415 416 static __always_inline void apic_native_eoi(void) 417 { 418 static_call(apic_call_native_eoi)(); 419 } 420 421 static __always_inline u64 apic_icr_read(void) 422 { 423 return static_call(apic_call_icr_read)(); 424 } 425 426 static __always_inline void apic_icr_write(u32 low, u32 high) 427 { 428 static_call(apic_call_icr_write)(low, high); 429 } 430 431 static __always_inline void __apic_send_IPI(int cpu, int vector) 432 { 433 static_call(apic_call_send_IPI)(cpu, vector); 434 } 435 436 static __always_inline void __apic_send_IPI_mask(const struct cpumask *mask, int vector) 437 { 438 static_call_mod(apic_call_send_IPI_mask)(mask, vector); 439 } 440 441 static __always_inline void __apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector) 442 { 443 static_call(apic_call_send_IPI_mask_allbutself)(mask, vector); 444 } 445 446 static __always_inline void __apic_send_IPI_allbutself(int vector) 447 { 448 static_call(apic_call_send_IPI_allbutself)(vector); 449 } 450 451 static __always_inline void __apic_send_IPI_all(int vector) 452 { 453 static_call(apic_call_send_IPI_all)(vector); 454 } 455 456 static __always_inline void __apic_send_IPI_self(int vector) 457 { 458 static_call_mod(apic_call_send_IPI_self)(vector); 459 } 460 461 static __always_inline void apic_wait_icr_idle(void) 462 { 463 static_call_cond(apic_call_wait_icr_idle)(); 464 } 465 466 static __always_inline u32 safe_apic_wait_icr_idle(void) 467 { 468 return apic->safe_wait_icr_idle ? apic->safe_wait_icr_idle() : 0; 469 } 470 471 static __always_inline bool apic_id_valid(u32 apic_id) 472 { 473 return apic_id <= apic->max_apic_id; 474 } 475 476 #else /* CONFIG_X86_LOCAL_APIC */ 477 478 static inline u32 apic_read(u32 reg) { return 0; } 479 static inline void apic_write(u32 reg, u32 val) { } 480 static inline void apic_eoi(void) { } 481 static inline u64 apic_icr_read(void) { return 0; } 482 static inline void apic_icr_write(u32 low, u32 high) { } 483 static inline void apic_wait_icr_idle(void) { } 484 static inline u32 safe_apic_wait_icr_idle(void) { return 0; } 485 static inline void apic_set_eoi_cb(void (*eoi)(void)) {} 486 static inline void apic_native_eoi(void) { WARN_ON_ONCE(1); } 487 static inline void apic_setup_apic_calls(void) { } 488 489 #define apic_update_callback(_callback, _fn) do { } while (0) 490 491 #endif /* CONFIG_X86_LOCAL_APIC */ 492 493 extern void apic_ack_irq(struct irq_data *data); 494 495 static inline bool lapic_vector_set_in_irr(unsigned int vector) 496 { 497 u32 irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); 498 499 return !!(irr & (1U << (vector % 32))); 500 } 501 502 /* 503 * Warm reset vector position: 504 */ 505 #define TRAMPOLINE_PHYS_LOW 0x467 506 #define TRAMPOLINE_PHYS_HIGH 0x469 507 508 extern void generic_bigsmp_probe(void); 509 510 #ifdef CONFIG_X86_LOCAL_APIC 511 512 #include <asm/smp.h> 513 514 extern struct apic apic_noop; 515 516 static inline u32 read_apic_id(void) 517 { 518 u32 reg = apic_read(APIC_ID); 519 520 return apic->get_apic_id(reg); 521 } 522 523 #ifdef CONFIG_X86_64 524 typedef int (*wakeup_cpu_handler)(int apicid, unsigned long start_eip); 525 extern int default_acpi_madt_oem_check(char *, char *); 526 extern void x86_64_probe_apic(void); 527 #else 528 static inline int default_acpi_madt_oem_check(char *a, char *b) { return 0; } 529 static inline void x86_64_probe_apic(void) { } 530 #endif 531 532 extern int default_apic_id_valid(u32 apicid); 533 534 extern u32 apic_default_calc_apicid(unsigned int cpu); 535 extern u32 apic_flat_calc_apicid(unsigned int cpu); 536 537 extern u32 default_cpu_present_to_apicid(int mps_cpu); 538 539 void apic_send_nmi_to_offline_cpu(unsigned int cpu); 540 541 #else /* CONFIG_X86_LOCAL_APIC */ 542 543 static inline u32 read_apic_id(void) { return 0; } 544 545 #endif /* !CONFIG_X86_LOCAL_APIC */ 546 547 #ifdef CONFIG_SMP 548 void apic_smt_update(void); 549 #else 550 static inline void apic_smt_update(void) { } 551 #endif 552 553 struct msi_msg; 554 struct irq_cfg; 555 556 extern void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg, 557 bool dmar); 558 559 extern void ioapic_zap_locks(void); 560 561 #endif /* _ASM_X86_APIC_H */ 562