1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/cpumask.h> 4 #include <linux/smp.h> 5 #include <asm/io_apic.h> 6 7 #include "local.h" 8 9 DEFINE_STATIC_KEY_FALSE(apic_use_ipi_shorthand); 10 11 #ifdef CONFIG_SMP 12 static int apic_ipi_shorthand_off __ro_after_init; 13 14 static __init int apic_ipi_shorthand(char *str) 15 { 16 get_option(&str, &apic_ipi_shorthand_off); 17 return 1; 18 } 19 __setup("no_ipi_broadcast=", apic_ipi_shorthand); 20 21 static int __init print_ipi_mode(void) 22 { 23 pr_info("IPI shorthand broadcast: %s\n", 24 apic_ipi_shorthand_off ? "disabled" : "enabled"); 25 return 0; 26 } 27 late_initcall(print_ipi_mode); 28 29 void apic_smt_update(void) 30 { 31 /* 32 * Do not switch to broadcast mode if: 33 * - Disabled on the command line 34 * - Only a single CPU is online 35 * - Not all present CPUs have been at least booted once 36 * 37 * The latter is important as the local APIC might be in some 38 * random state and a broadcast might cause havoc. That's 39 * especially true for NMI broadcasting. 40 */ 41 if (apic_ipi_shorthand_off || num_online_cpus() == 1 || 42 !cpumask_equal(cpu_present_mask, &cpus_booted_once_mask)) { 43 static_branch_disable(&apic_use_ipi_shorthand); 44 } else { 45 static_branch_enable(&apic_use_ipi_shorthand); 46 } 47 } 48 49 void apic_send_IPI_allbutself(unsigned int vector) 50 { 51 if (num_online_cpus() < 2) 52 return; 53 54 if (static_branch_likely(&apic_use_ipi_shorthand)) 55 apic->send_IPI_allbutself(vector); 56 else 57 apic->send_IPI_mask_allbutself(cpu_online_mask, vector); 58 } 59 60 /* 61 * Send a 'reschedule' IPI to another CPU. It goes straight through and 62 * wastes no time serializing anything. Worst case is that we lose a 63 * reschedule ... 64 */ 65 void native_smp_send_reschedule(int cpu) 66 { 67 if (unlikely(cpu_is_offline(cpu))) { 68 WARN(1, "sched: Unexpected reschedule of offline CPU#%d!\n", cpu); 69 return; 70 } 71 apic->send_IPI(cpu, RESCHEDULE_VECTOR); 72 } 73 74 void native_send_call_func_single_ipi(int cpu) 75 { 76 apic->send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR); 77 } 78 79 void native_send_call_func_ipi(const struct cpumask *mask) 80 { 81 if (static_branch_likely(&apic_use_ipi_shorthand)) { 82 unsigned int cpu = smp_processor_id(); 83 84 if (!cpumask_or_equal(mask, cpumask_of(cpu), cpu_online_mask)) 85 goto sendmask; 86 87 if (cpumask_test_cpu(cpu, mask)) 88 apic->send_IPI_all(CALL_FUNCTION_VECTOR); 89 else if (num_online_cpus() > 1) 90 apic->send_IPI_allbutself(CALL_FUNCTION_VECTOR); 91 return; 92 } 93 94 sendmask: 95 apic->send_IPI_mask(mask, CALL_FUNCTION_VECTOR); 96 } 97 98 #endif /* CONFIG_SMP */ 99 100 static inline int __prepare_ICR2(unsigned int mask) 101 { 102 return SET_APIC_DEST_FIELD(mask); 103 } 104 105 static inline void __xapic_wait_icr_idle(void) 106 { 107 while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY) 108 cpu_relax(); 109 } 110 111 void __default_send_IPI_shortcut(unsigned int shortcut, int vector) 112 { 113 /* 114 * Subtle. In the case of the 'never do double writes' workaround 115 * we have to lock out interrupts to be safe. As we don't care 116 * of the value read we use an atomic rmw access to avoid costly 117 * cli/sti. Otherwise we use an even cheaper single atomic write 118 * to the APIC. 119 */ 120 unsigned int cfg; 121 122 /* 123 * Wait for idle. 124 */ 125 if (unlikely(vector == NMI_VECTOR)) 126 safe_apic_wait_icr_idle(); 127 else 128 __xapic_wait_icr_idle(); 129 130 /* 131 * No need to touch the target chip field. Also the destination 132 * mode is ignored when a shorthand is used. 133 */ 134 cfg = __prepare_ICR(shortcut, vector, 0); 135 136 /* 137 * Send the IPI. The write to APIC_ICR fires this off. 138 */ 139 native_apic_mem_write(APIC_ICR, cfg); 140 } 141 142 /* 143 * This is used to send an IPI with no shorthand notation (the destination is 144 * specified in bits 56 to 63 of the ICR). 145 */ 146 void __default_send_IPI_dest_field(unsigned int mask, int vector, unsigned int dest) 147 { 148 unsigned long cfg; 149 150 /* 151 * Wait for idle. 152 */ 153 if (unlikely(vector == NMI_VECTOR)) 154 safe_apic_wait_icr_idle(); 155 else 156 __xapic_wait_icr_idle(); 157 158 /* 159 * prepare target chip field 160 */ 161 cfg = __prepare_ICR2(mask); 162 native_apic_mem_write(APIC_ICR2, cfg); 163 164 /* 165 * program the ICR 166 */ 167 cfg = __prepare_ICR(0, vector, dest); 168 169 /* 170 * Send the IPI. The write to APIC_ICR fires this off. 171 */ 172 native_apic_mem_write(APIC_ICR, cfg); 173 } 174 175 void default_send_IPI_single_phys(int cpu, int vector) 176 { 177 unsigned long flags; 178 179 local_irq_save(flags); 180 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, cpu), 181 vector, APIC_DEST_PHYSICAL); 182 local_irq_restore(flags); 183 } 184 185 void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector) 186 { 187 unsigned long query_cpu; 188 unsigned long flags; 189 190 /* 191 * Hack. The clustered APIC addressing mode doesn't allow us to send 192 * to an arbitrary mask, so I do a unicast to each CPU instead. 193 * - mbligh 194 */ 195 local_irq_save(flags); 196 for_each_cpu(query_cpu, mask) { 197 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, 198 query_cpu), vector, APIC_DEST_PHYSICAL); 199 } 200 local_irq_restore(flags); 201 } 202 203 void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask, 204 int vector) 205 { 206 unsigned int this_cpu = smp_processor_id(); 207 unsigned int query_cpu; 208 unsigned long flags; 209 210 /* See Hack comment above */ 211 212 local_irq_save(flags); 213 for_each_cpu(query_cpu, mask) { 214 if (query_cpu == this_cpu) 215 continue; 216 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, 217 query_cpu), vector, APIC_DEST_PHYSICAL); 218 } 219 local_irq_restore(flags); 220 } 221 222 /* 223 * Helper function for APICs which insist on cpumasks 224 */ 225 void default_send_IPI_single(int cpu, int vector) 226 { 227 apic->send_IPI_mask(cpumask_of(cpu), vector); 228 } 229 230 void default_send_IPI_allbutself(int vector) 231 { 232 __default_send_IPI_shortcut(APIC_DEST_ALLBUT, vector); 233 } 234 235 void default_send_IPI_all(int vector) 236 { 237 __default_send_IPI_shortcut(APIC_DEST_ALLINC, vector); 238 } 239 240 void default_send_IPI_self(int vector) 241 { 242 __default_send_IPI_shortcut(APIC_DEST_SELF, vector); 243 } 244 245 #ifdef CONFIG_X86_32 246 247 void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, 248 int vector) 249 { 250 unsigned long flags; 251 unsigned int query_cpu; 252 253 /* 254 * Hack. The clustered APIC addressing mode doesn't allow us to send 255 * to an arbitrary mask, so I do a unicasts to each CPU instead. This 256 * should be modified to do 1 message per cluster ID - mbligh 257 */ 258 259 local_irq_save(flags); 260 for_each_cpu(query_cpu, mask) 261 __default_send_IPI_dest_field( 262 early_per_cpu(x86_cpu_to_logical_apicid, query_cpu), 263 vector, APIC_DEST_LOGICAL); 264 local_irq_restore(flags); 265 } 266 267 void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask, 268 int vector) 269 { 270 unsigned long flags; 271 unsigned int query_cpu; 272 unsigned int this_cpu = smp_processor_id(); 273 274 /* See Hack comment above */ 275 276 local_irq_save(flags); 277 for_each_cpu(query_cpu, mask) { 278 if (query_cpu == this_cpu) 279 continue; 280 __default_send_IPI_dest_field( 281 early_per_cpu(x86_cpu_to_logical_apicid, query_cpu), 282 vector, APIC_DEST_LOGICAL); 283 } 284 local_irq_restore(flags); 285 } 286 287 /* 288 * This is only used on smaller machines. 289 */ 290 void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector) 291 { 292 unsigned long mask = cpumask_bits(cpumask)[0]; 293 unsigned long flags; 294 295 if (!mask) 296 return; 297 298 local_irq_save(flags); 299 WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]); 300 __default_send_IPI_dest_field(mask, vector, APIC_DEST_LOGICAL); 301 local_irq_restore(flags); 302 } 303 304 /* must come after the send_IPI functions above for inlining */ 305 static int convert_apicid_to_cpu(int apic_id) 306 { 307 int i; 308 309 for_each_possible_cpu(i) { 310 if (per_cpu(x86_cpu_to_apicid, i) == apic_id) 311 return i; 312 } 313 return -1; 314 } 315 316 int safe_smp_processor_id(void) 317 { 318 int apicid, cpuid; 319 320 if (!boot_cpu_has(X86_FEATURE_APIC)) 321 return 0; 322 323 apicid = hard_smp_processor_id(); 324 if (apicid == BAD_APICID) 325 return 0; 326 327 cpuid = convert_apicid_to_cpu(apicid); 328 329 return cpuid >= 0 ? cpuid : 0; 330 } 331 #endif 332