1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Generic helpers for smp ipi calls 4 * 5 * (C) Jens Axboe <jens.axboe@oracle.com> 2008 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/irq_work.h> 11 #include <linux/rcupdate.h> 12 #include <linux/rculist.h> 13 #include <linux/kernel.h> 14 #include <linux/export.h> 15 #include <linux/percpu.h> 16 #include <linux/init.h> 17 #include <linux/interrupt.h> 18 #include <linux/gfp.h> 19 #include <linux/slab.h> 20 #include <linux/smp.h> 21 #include <linux/cpu.h> 22 #include <linux/sched.h> 23 #include <linux/sched/idle.h> 24 #include <linux/hypervisor.h> 25 #include <linux/sched/clock.h> 26 #include <linux/nmi.h> 27 #include <linux/sched/debug.h> 28 #include <linux/jump_label.h> 29 #include <linux/string_choices.h> 30 31 #include <trace/events/ipi.h> 32 #define CREATE_TRACE_POINTS 33 #include <trace/events/csd.h> 34 #undef CREATE_TRACE_POINTS 35 36 #include "smpboot.h" 37 #include "sched/smp.h" 38 39 #define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK) 40 41 struct call_function_data { 42 call_single_data_t __percpu *csd; 43 cpumask_var_t cpumask; 44 cpumask_var_t cpumask_ipi; 45 }; 46 47 static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data); 48 49 static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue); 50 51 static DEFINE_PER_CPU(atomic_t, trigger_backtrace) = ATOMIC_INIT(1); 52 53 static void __flush_smp_call_function_queue(bool warn_cpu_offline); 54 55 int smpcfd_prepare_cpu(unsigned int cpu) 56 { 57 struct call_function_data *cfd = &per_cpu(cfd_data, cpu); 58 59 if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL, 60 cpu_to_node(cpu))) 61 return -ENOMEM; 62 if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL, 63 cpu_to_node(cpu))) { 64 free_cpumask_var(cfd->cpumask); 65 return -ENOMEM; 66 } 67 68 /* 69 * Allocate the per-CPU CSD the first time a CPU comes up. It is 70 * not freed when the CPU is offlined, so csd_lock_wait() can access 71 * it even when the CPU was offlined after preemption was re-enabled. 72 */ 73 if (!cfd->csd) 74 cfd->csd = alloc_percpu(call_single_data_t); 75 if (!cfd->csd) { 76 free_cpumask_var(cfd->cpumask); 77 free_cpumask_var(cfd->cpumask_ipi); 78 return -ENOMEM; 79 } 80 81 return 0; 82 } 83 84 int smpcfd_dead_cpu(unsigned int cpu) 85 { 86 struct call_function_data *cfd = &per_cpu(cfd_data, cpu); 87 88 free_cpumask_var(cfd->cpumask); 89 free_cpumask_var(cfd->cpumask_ipi); 90 irq_work_run_cpu(cpu); 91 return 0; 92 } 93 94 int smpcfd_dying_cpu(unsigned int cpu) 95 { 96 /* 97 * The IPIs for the smp-call-function callbacks queued by other CPUs 98 * might arrive late, either due to hardware latencies or because this 99 * CPU disabled interrupts (inside stop-machine) before the IPIs were 100 * sent. So flush out any pending callbacks explicitly (without waiting 101 * for the IPIs to arrive), to ensure that the outgoing CPU doesn't go 102 * offline with work still pending. 103 * 104 * This runs with interrupts disabled inside the stopper task invoked by 105 * stop_machine(), ensuring mutually exclusive CPU offlining and IPI flush. 106 */ 107 __flush_smp_call_function_queue(false); 108 irq_work_run(); 109 return 0; 110 } 111 112 void __init call_function_init(void) 113 { 114 int i; 115 116 for_each_possible_cpu(i) 117 init_llist_head(&per_cpu(call_single_queue, i)); 118 119 smpcfd_prepare_cpu(smp_processor_id()); 120 } 121 122 static __always_inline void 123 send_call_function_single_ipi(int cpu) 124 { 125 if (call_function_single_prep_ipi(cpu)) { 126 trace_ipi_send_cpu(cpu, _RET_IP_, 127 generic_smp_call_function_single_interrupt); 128 arch_send_call_function_single_ipi(cpu); 129 } 130 } 131 132 static __always_inline void 133 send_call_function_ipi_mask(struct cpumask *mask) 134 { 135 trace_ipi_send_cpumask(mask, _RET_IP_, 136 generic_smp_call_function_single_interrupt); 137 arch_send_call_function_ipi_mask(mask); 138 } 139 140 static __always_inline void 141 csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd) 142 { 143 trace_csd_function_entry(func, csd); 144 func(info); 145 trace_csd_function_exit(func, csd); 146 } 147 148 static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled); 149 150 #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG 151 152 /* 153 * Parse the csdlock_debug= kernel boot parameter. 154 * 155 * If you need to restore the old "ext" value that once provided 156 * additional debugging information, reapply the following commits: 157 * 158 * de7b09ef658d ("locking/csd_lock: Prepare more CSD lock debugging") 159 * a5aabace5fb8 ("locking/csd_lock: Add more data to CSD lock debugging") 160 */ 161 static int __init csdlock_debug(char *str) 162 { 163 int ret; 164 unsigned int val = 0; 165 166 ret = get_option(&str, &val); 167 if (ret) { 168 if (val) 169 static_branch_enable(&csdlock_debug_enabled); 170 else 171 static_branch_disable(&csdlock_debug_enabled); 172 } 173 174 return 1; 175 } 176 __setup("csdlock_debug=", csdlock_debug); 177 178 static DEFINE_PER_CPU(call_single_data_t *, cur_csd); 179 static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func); 180 static DEFINE_PER_CPU(void *, cur_csd_info); 181 182 static ulong csd_lock_timeout = 5000; /* CSD lock timeout in milliseconds. */ 183 module_param(csd_lock_timeout, ulong, 0644); 184 static int panic_on_ipistall; /* CSD panic timeout in milliseconds, 300000 for five minutes. */ 185 module_param(panic_on_ipistall, int, 0644); 186 187 static atomic_t csd_bug_count = ATOMIC_INIT(0); 188 189 /* Record current CSD work for current CPU, NULL to erase. */ 190 static void __csd_lock_record(call_single_data_t *csd) 191 { 192 if (!csd) { 193 /* 194 * Pairs with smp_load_acquire() of cur_csd in 195 * csd_lock_wait_toolong(): orders any preceding CSD 196 * callback/unlock before a remote reader observes NULL. 197 */ 198 smp_store_release(this_cpu_ptr(&cur_csd), NULL); 199 return; 200 } 201 __this_cpu_write(cur_csd_func, csd->func); 202 __this_cpu_write(cur_csd_info, csd->info); 203 /* 204 * Pairs with smp_load_acquire() of cur_csd in 205 * csd_lock_wait_toolong(): publishes cur_csd_func and 206 * cur_csd_info before the non-NULL pointer becomes visible. 207 */ 208 smp_store_release(this_cpu_ptr(&cur_csd), csd); 209 } 210 211 static __always_inline void csd_lock_record(call_single_data_t *csd) 212 { 213 if (static_branch_unlikely(&csdlock_debug_enabled)) 214 __csd_lock_record(csd); 215 } 216 217 static int csd_lock_wait_getcpu(call_single_data_t *csd) 218 { 219 unsigned int csd_type; 220 221 csd_type = CSD_TYPE(csd); 222 if (csd_type == CSD_TYPE_ASYNC || csd_type == CSD_TYPE_SYNC) 223 return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */ 224 return -1; 225 } 226 227 static atomic_t n_csd_lock_stuck; 228 229 /** 230 * csd_lock_is_stuck - Has a CSD-lock acquisition been stuck too long? 231 * 232 * Returns: @true if a CSD-lock acquisition is stuck and has been stuck 233 * long enough for a "non-responsive CSD lock" message to be printed. 234 */ 235 bool csd_lock_is_stuck(void) 236 { 237 return !!atomic_read(&n_csd_lock_stuck); 238 } 239 240 /* State that csd_lock_wait_toolong() carries across the __csd_lock_wait() loop. */ 241 struct csd_wait_state { 242 u64 ts_start; /* When the wait began. */ 243 u64 ts_report; /* When the last complaint was printed. */ 244 u64 ts_resend; /* When the last IPI was re-sent, 0 if never. */ 245 int bug_id; 246 unsigned long nmessages; 247 }; 248 249 /* 250 * Report a CSD lock that came back, @ts_unstuck being when the release was 251 * noticed. Only mention the re-send delta if an IPI was actually re-sent. 252 */ 253 static void csd_lock_print_unstuck(struct csd_wait_state *state, int cpu, u64 ts_unstuck) 254 { 255 if (state->ts_resend) 256 pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock after %lld ns, %lld ns after the last IPI re-send.\n", 257 state->bug_id, raw_smp_processor_id(), cpu, 258 (s64)(ts_unstuck - state->ts_start), 259 (s64)(ts_unstuck - state->ts_resend)); 260 else 261 pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock after %lld ns.\n", 262 state->bug_id, raw_smp_processor_id(), cpu, 263 (s64)(ts_unstuck - state->ts_start)); 264 } 265 266 /* 267 * Complain if too much time spent waiting. Note that only 268 * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, 269 * so waiting on other types gets much less information. 270 */ 271 static bool csd_lock_wait_toolong(call_single_data_t *csd, struct csd_wait_state *state) 272 { 273 int cpu = -1; 274 int cpux; 275 bool firsttime; 276 u64 ts_now, ts_delta; 277 call_single_data_t *cpu_cur_csd; 278 unsigned int flags = READ_ONCE(csd->node.u_flags); 279 unsigned long long csd_lock_timeout_ns = csd_lock_timeout * NSEC_PER_MSEC; 280 281 if (!(flags & CSD_FLAG_LOCK)) { 282 if (!unlikely(state->bug_id)) 283 return true; 284 ts_now = ktime_get_mono_fast_ns(); 285 cpu = csd_lock_wait_getcpu(csd); 286 csd_lock_print_unstuck(state, cpu, ts_now); 287 atomic_dec(&n_csd_lock_stuck); 288 return true; 289 } 290 291 ts_now = ktime_get_mono_fast_ns(); 292 /* How long since we last checked for a stuck CSD lock.*/ 293 ts_delta = ts_now - state->ts_report; 294 if (likely(ts_delta <= csd_lock_timeout_ns * (state->nmessages + 1) * 295 (!state->nmessages ? 1 : (ilog2(num_online_cpus()) / 2 + 1)) || 296 csd_lock_timeout_ns == 0)) 297 return false; 298 299 if (state->ts_start > ts_now) { 300 /* Our own sched_clock went backward; don't blame another CPU. */ 301 ts_delta = state->ts_start - ts_now; 302 pr_alert("sched_clock on CPU %d went backward by %llu ns\n", raw_smp_processor_id(), ts_delta); 303 state->ts_report = ts_now; 304 return false; 305 } 306 307 firsttime = !state->bug_id; 308 if (firsttime) 309 state->bug_id = atomic_inc_return(&csd_bug_count); 310 cpu = csd_lock_wait_getcpu(csd); 311 if (WARN_ONCE(cpu < 0 || cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu)) 312 cpux = 0; 313 else 314 cpux = cpu; 315 /* 316 * Pairs with smp_store_release() of cur_csd in __csd_lock_record(): 317 * a non-NULL cur_csd here implies cur_csd_func and cur_csd_info 318 * are the matching publication; a NULL value is ordered after any 319 * preceding CSD callback/unlock on the remote CPU. 320 */ 321 cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); 322 /* How long since this CSD lock was stuck. */ 323 ts_delta = ts_now - state->ts_start; 324 pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n", 325 firsttime ? "Detected" : "Continued", state->bug_id, raw_smp_processor_id(), (s64)ts_delta, 326 cpu, csd->func, csd->info); 327 state->nmessages++; 328 if (firsttime) 329 atomic_inc(&n_csd_lock_stuck); 330 /* 331 * If the CSD lock is still stuck after 5 minutes, it is unlikely 332 * to become unstuck. Use a signed comparison to avoid triggering 333 * on underflows when the TSC is out of sync between sockets. 334 */ 335 BUG_ON(panic_on_ipistall > 0 && (s64)ts_delta > ((s64)panic_on_ipistall * NSEC_PER_MSEC)); 336 if (cpu_cur_csd && csd != cpu_cur_csd) { 337 pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n", 338 state->bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)), 339 READ_ONCE(per_cpu(cur_csd_info, cpux))); 340 } else { 341 pr_alert("\tcsd: CSD lock (#%d) %s.\n", 342 state->bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request"); 343 } 344 if (cpu >= 0) { 345 if (atomic_cmpxchg_acquire(&per_cpu(trigger_backtrace, cpu), 1, 0)) 346 dump_cpu_task(cpu); 347 if (!cpu_cur_csd) { 348 pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", state->bug_id, raw_smp_processor_id(), cpu); 349 arch_send_call_function_single_ipi(cpu); 350 state->ts_resend = ktime_get_mono_fast_ns(); 351 } 352 } 353 if (firsttime) 354 dump_stack(); 355 state->ts_report = ts_now; 356 357 return false; 358 } 359 360 /* 361 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources 362 * 363 * For non-synchronous ipi calls the csd can still be in use by the 364 * previous function call. For multi-cpu calls its even more interesting 365 * as we'll have to ensure no other cpu is observing our csd. 366 */ 367 static void __csd_lock_wait(call_single_data_t *csd) 368 { 369 struct csd_wait_state state = {}; 370 371 guard(preempt)(); 372 373 state.ts_report = state.ts_start = ktime_get_mono_fast_ns(); 374 for (;;) { 375 if (csd_lock_wait_toolong(csd, &state)) 376 break; 377 cpu_relax(); 378 } 379 smp_acquire__after_ctrl_dep(); 380 } 381 382 static __always_inline void csd_lock_wait(call_single_data_t *csd) 383 { 384 if (static_branch_unlikely(&csdlock_debug_enabled)) { 385 __csd_lock_wait(csd); 386 return; 387 } 388 389 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); 390 } 391 #else 392 static __always_inline void __csd_lock_wait(call_single_data_t *csd) 393 { 394 } 395 396 static void csd_lock_record(call_single_data_t *csd) 397 { 398 } 399 400 static __always_inline void csd_lock_wait(call_single_data_t *csd) 401 { 402 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); 403 } 404 #endif 405 406 static __always_inline void csd_lock(call_single_data_t *csd) 407 { 408 if (IS_ENABLED(CONFIG_CSD_LOCK_WAIT_DEBUG) && 409 static_branch_unlikely(&csdlock_debug_enabled)) { 410 411 for (;;) { 412 unsigned int flags; 413 414 __csd_lock_wait(csd); 415 flags = READ_ONCE(csd->node.u_flags); 416 417 if (!(flags & CSD_FLAG_LOCK) && 418 try_cmpxchg_acquire(&csd->node.u_flags, &flags, flags | CSD_FLAG_LOCK)) 419 break; 420 } 421 } else { 422 csd_lock_wait(csd); 423 csd->node.u_flags |= CSD_FLAG_LOCK; 424 } 425 426 /* 427 * prevent CPU from reordering the above assignment 428 * to ->flags with any subsequent assignments to other 429 * fields of the specified call_single_data_t structure: 430 */ 431 smp_wmb(); 432 } 433 434 static __always_inline void csd_unlock(call_single_data_t *csd) 435 { 436 WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); 437 438 /* 439 * ensure we're all done before releasing data: 440 */ 441 smp_store_release(&csd->node.u_flags, 0); 442 } 443 444 static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); 445 446 #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG 447 static call_single_data_t *get_single_csd_data(int cpu) 448 { 449 if (static_branch_unlikely(&csdlock_debug_enabled) && 450 (unsigned int)cpu < nr_cpu_ids) 451 return per_cpu_ptr(&csd_data, cpu); 452 return this_cpu_ptr(&csd_data); 453 } 454 #else 455 static call_single_data_t *get_single_csd_data(int cpu) 456 { 457 return this_cpu_ptr(&csd_data); 458 } 459 #endif 460 461 void __smp_call_single_queue(int cpu, struct llist_node *node) 462 { 463 /* 464 * We have to check the type of the CSD before queueing it, because 465 * once queued it can have its flags cleared by 466 * flush_smp_call_function_queue() 467 * even if we haven't sent the smp_call IPI yet (e.g. the stopper 468 * executes migration_cpu_stop() on the remote CPU). 469 */ 470 if (trace_csd_queue_cpu_enabled()) { 471 call_single_data_t *csd; 472 smp_call_func_t func; 473 474 csd = container_of(node, call_single_data_t, node.llist); 475 func = CSD_TYPE(csd) == CSD_TYPE_TTWU ? 476 sched_ttwu_pending : csd->func; 477 478 trace_call__csd_queue_cpu(cpu, _RET_IP_, func, csd); 479 } 480 481 /* 482 * The list addition should be visible to the target CPU when it pops 483 * the head of the list to pull the entry off it in the IPI handler 484 * because of normal cache coherency rules implied by the underlying 485 * llist ops. 486 * 487 * If IPIs can go out of order to the cache coherency protocol 488 * in an architecture, sufficient synchronisation should be added 489 * to arch code to make it appear to obey cache coherency WRT 490 * locking and barrier primitives. Generic code isn't really 491 * equipped to do the right thing... 492 */ 493 if (llist_add(node, &per_cpu(call_single_queue, cpu))) 494 send_call_function_single_ipi(cpu); 495 } 496 497 /* 498 * Insert a previously allocated call_single_data_t element 499 * for execution on the given CPU. data must already have 500 * ->func, ->info, and ->flags set. 501 */ 502 static int generic_exec_single(int cpu, call_single_data_t *csd) 503 { 504 /* 505 * Preemption already disabled here so stopper cannot run on this CPU, 506 * ensuring mutually exclusive CPU offlining and last IPI flush. 507 */ 508 if (cpu == smp_processor_id()) { 509 smp_call_func_t func = csd->func; 510 void *info = csd->info; 511 unsigned long flags; 512 513 /* 514 * We can unlock early even for the synchronous on-stack case, 515 * since we're doing this from the same CPU.. 516 */ 517 csd_lock_record(csd); 518 csd_unlock(csd); 519 local_irq_save(flags); 520 csd_do_func(func, info, NULL); 521 csd_lock_record(NULL); 522 local_irq_restore(flags); 523 return 0; 524 } 525 526 if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) { 527 csd_unlock(csd); 528 return -ENXIO; 529 } 530 531 __smp_call_single_queue(cpu, &csd->node.llist); 532 533 return 0; 534 } 535 536 /** 537 * generic_smp_call_function_single_interrupt - Execute SMP IPI callbacks 538 * 539 * Invoked by arch to handle an IPI for call function single. 540 * Must be called with interrupts disabled. 541 */ 542 void generic_smp_call_function_single_interrupt(void) 543 { 544 __flush_smp_call_function_queue(true); 545 } 546 547 /** 548 * __flush_smp_call_function_queue - Flush pending smp-call-function callbacks 549 * 550 * @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an 551 * offline CPU. Skip this check if set to 'false'. 552 * 553 * Flush any pending smp-call-function callbacks queued on this CPU. This is 554 * invoked by the generic IPI handler, as well as by a CPU about to go offline, 555 * to ensure that all pending IPI callbacks are run before it goes completely 556 * offline. 557 * 558 * Loop through the call_single_queue and run all the queued callbacks. 559 * Must be called with interrupts disabled. 560 */ 561 static void __flush_smp_call_function_queue(bool warn_cpu_offline) 562 { 563 call_single_data_t *csd, *csd_next; 564 struct llist_node *entry, *prev; 565 struct llist_head *head; 566 static bool warned; 567 atomic_t *tbt; 568 569 lockdep_assert_irqs_disabled(); 570 571 /* Allow waiters to send backtrace NMI from here onwards */ 572 tbt = this_cpu_ptr(&trigger_backtrace); 573 atomic_set_release(tbt, 1); 574 575 head = this_cpu_ptr(&call_single_queue); 576 entry = llist_del_all(head); 577 entry = llist_reverse_order(entry); 578 579 /* There shouldn't be any pending callbacks on an offline CPU. */ 580 if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) && 581 !warned && entry != NULL)) { 582 warned = true; 583 WARN(1, "IPI on offline CPU %d\n", smp_processor_id()); 584 585 /* 586 * We don't have to use the _safe() variant here 587 * because we are not invoking the IPI handlers yet. 588 */ 589 llist_for_each_entry(csd, entry, node.llist) { 590 switch (CSD_TYPE(csd)) { 591 case CSD_TYPE_ASYNC: 592 case CSD_TYPE_SYNC: 593 case CSD_TYPE_IRQ_WORK: 594 pr_warn("IPI callback %pS sent to offline CPU\n", 595 csd->func); 596 break; 597 598 case CSD_TYPE_TTWU: 599 pr_warn("IPI task-wakeup sent to offline CPU\n"); 600 break; 601 602 default: 603 pr_warn("IPI callback, unknown type %d, sent to offline CPU\n", 604 CSD_TYPE(csd)); 605 break; 606 } 607 } 608 } 609 610 /* 611 * First; run all SYNC callbacks, people are waiting for us. 612 */ 613 prev = NULL; 614 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) { 615 /* Do we wait until *after* callback? */ 616 if (CSD_TYPE(csd) == CSD_TYPE_SYNC) { 617 smp_call_func_t func = csd->func; 618 void *info = csd->info; 619 620 if (prev) { 621 prev->next = &csd_next->node.llist; 622 } else { 623 entry = &csd_next->node.llist; 624 } 625 626 csd_lock_record(csd); 627 csd_do_func(func, info, csd); 628 csd_unlock(csd); 629 csd_lock_record(NULL); 630 } else { 631 prev = &csd->node.llist; 632 } 633 } 634 635 if (!entry) 636 return; 637 638 /* 639 * Second; run all !SYNC callbacks. 640 */ 641 prev = NULL; 642 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) { 643 int type = CSD_TYPE(csd); 644 645 if (type != CSD_TYPE_TTWU) { 646 if (prev) { 647 prev->next = &csd_next->node.llist; 648 } else { 649 entry = &csd_next->node.llist; 650 } 651 652 if (type == CSD_TYPE_ASYNC) { 653 smp_call_func_t func = csd->func; 654 void *info = csd->info; 655 656 csd_lock_record(csd); 657 csd_unlock(csd); 658 csd_do_func(func, info, csd); 659 csd_lock_record(NULL); 660 } else if (type == CSD_TYPE_IRQ_WORK) { 661 irq_work_single(csd); 662 } 663 664 } else { 665 prev = &csd->node.llist; 666 } 667 } 668 669 /* 670 * Third; only CSD_TYPE_TTWU is left, issue those. 671 */ 672 if (entry) { 673 csd = llist_entry(entry, typeof(*csd), node.llist); 674 csd_do_func(sched_ttwu_pending, entry, csd); 675 } 676 } 677 678 679 /** 680 * flush_smp_call_function_queue - Flush pending smp-call-function callbacks 681 * from task context (idle, migration thread) 682 * 683 * When TIF_POLLING_NRFLAG is supported and a CPU is in idle and has it 684 * set, then remote CPUs can avoid sending IPIs and wake the idle CPU by 685 * setting TIF_NEED_RESCHED. The idle task on the woken up CPU has to 686 * handle queued SMP function calls before scheduling. 687 * 688 * The migration thread has to ensure that an eventually pending wakeup has 689 * been handled before it migrates a task. 690 */ 691 void flush_smp_call_function_queue(void) 692 { 693 unsigned int was_pending; 694 unsigned long flags; 695 696 if (llist_empty(this_cpu_ptr(&call_single_queue))) 697 return; 698 699 local_irq_save(flags); 700 /* Get the already pending soft interrupts for RT enabled kernels */ 701 was_pending = local_softirq_pending(); 702 __flush_smp_call_function_queue(true); 703 if (local_softirq_pending()) 704 do_softirq_post_smp_call_flush(was_pending); 705 706 local_irq_restore(flags); 707 } 708 709 static int __smp_call_function_single(int cpu, smp_call_func_t func, 710 void *info, const struct cpumask *mask, 711 bool wait) 712 { 713 call_single_data_t *csd; 714 call_single_data_t csd_stack = { 715 .node = { .u_flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC, }, 716 }; 717 int this_cpu; 718 int err; 719 720 /* 721 * Prevent preemption and reschedule on another CPU, as well as CPU 722 * removal. This prevents stopper from running on this CPU, thus 723 * providing mutual exclusion of the below cpu_online() check and 724 * IPI sending ensuring IPI are not missed by CPU going offline. 725 */ 726 this_cpu = get_cpu(); 727 728 if (mask) { 729 /* Try for same CPU (cheapest) */ 730 if (!cpumask_test_cpu(this_cpu, mask)) 731 cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(this_cpu)); 732 else 733 cpu = this_cpu; 734 } 735 736 /* 737 * Can deadlock when called with interrupts disabled. 738 * We allow cpu's that are not yet online though, as no one else can 739 * send smp call function interrupt to this cpu and as such deadlocks 740 * can't happen. 741 */ 742 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled() 743 && !oops_in_progress); 744 745 /* 746 * When @wait we can deadlock when we interrupt between llist_add() and 747 * arch_send_call_function_ipi*(); when !@wait we can deadlock due to 748 * csd_lock() on because the interrupt context uses the same csd 749 * storage. 750 */ 751 WARN_ON_ONCE(!in_task()); 752 753 csd = &csd_stack; 754 if (!wait) { 755 csd = get_single_csd_data(cpu); 756 csd_lock(csd); 757 } 758 759 csd->func = func; 760 csd->info = info; 761 #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG 762 csd->node.src = this_cpu; 763 csd->node.dst = cpu; 764 #endif 765 766 err = generic_exec_single(cpu, csd); 767 768 /* 769 * @csd is stack-allocated when @wait is true. No concurrent access 770 * except from the IPI completion path, so we can re-enable preemption 771 * early to reduce latency. 772 */ 773 put_cpu(); 774 775 if (wait) 776 csd_lock_wait(csd); 777 778 return err; 779 } 780 781 /** 782 * smp_call_function_single - Run a function on a specific CPU 783 * @cpu: Specific target CPU for this function. 784 * @func: The function to run. This must be fast and non-blocking. 785 * @info: An arbitrary pointer to pass to the function. 786 * @wait: If true, wait until function has completed on other CPUs. 787 * 788 * Returns: %0 on success, else a negative status code. 789 */ 790 int smp_call_function_single(int cpu, smp_call_func_t func, void *info, bool wait) 791 { 792 return __smp_call_function_single(cpu, func, info, NULL, wait); 793 } 794 EXPORT_SYMBOL(smp_call_function_single); 795 796 /** 797 * smp_call_function_single_async() - Run an asynchronous function on a 798 * specific CPU. 799 * @cpu: The CPU to run on. 800 * @csd: Pre-allocated and setup data structure 801 * 802 * Like smp_call_function_single(), but the call is asynchonous and 803 * can thus be done from contexts with disabled interrupts. 804 * 805 * The caller passes his own pre-allocated data structure 806 * (ie: embedded in an object) and is responsible for synchronizing it 807 * such that the IPIs performed on the @csd are strictly serialized. 808 * 809 * If the function is called with one csd which has not yet been 810 * processed by previous call to smp_call_function_single_async(), the 811 * function will return immediately with -EBUSY showing that the csd 812 * object is still in progress. 813 * 814 * NOTE: Be careful, there is unfortunately no current debugging facility to 815 * validate the correctness of this serialization. 816 * 817 * Return: %0 on success or negative errno value on error 818 */ 819 int smp_call_function_single_async(int cpu, call_single_data_t *csd) 820 { 821 int err = 0; 822 823 preempt_disable(); 824 825 if (csd->node.u_flags & CSD_FLAG_LOCK) { 826 err = -EBUSY; 827 goto out; 828 } 829 830 csd->node.u_flags = CSD_FLAG_LOCK; 831 smp_wmb(); 832 833 err = generic_exec_single(cpu, csd); 834 835 out: 836 preempt_enable(); 837 838 return err; 839 } 840 EXPORT_SYMBOL_GPL(smp_call_function_single_async); 841 842 /** 843 * smp_call_function_any - Run a function on any of the given cpus 844 * @mask: The mask of cpus it can run on. 845 * @func: The function to run. This must be fast and non-blocking. 846 * @info: An arbitrary pointer to pass to the function. 847 * @wait: If true, wait until function has completed. 848 * 849 * Selection preference: 850 * 1) current cpu if in @mask 851 * 2) nearest cpu in @mask, based on NUMA topology 852 * 853 * Returns: %0 on success, else a negative status code (if no cpus were online). 854 */ 855 int smp_call_function_any(const struct cpumask *mask, 856 smp_call_func_t func, void *info, int wait) 857 { 858 return __smp_call_function_single(-1, func, info, mask, wait); 859 } 860 EXPORT_SYMBOL_GPL(smp_call_function_any); 861 862 static DEFINE_STATIC_KEY_FALSE(ipi_mask_inlined); 863 864 #ifdef CONFIG_PREEMPTION 865 866 int smp_task_ipi_mask_alloc(struct task_struct *task) 867 { 868 if (static_branch_unlikely(&ipi_mask_inlined)) 869 return 0; 870 871 ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr = 872 kmalloc(cpumask_size(), GFP_KERNEL); 873 if (!ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr) 874 return -ENOMEM; 875 876 return 0; 877 } 878 879 void smp_task_ipi_mask_free(struct task_struct *task) 880 { 881 if (static_branch_unlikely(&ipi_mask_inlined)) 882 return; 883 884 kfree(ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr); 885 } 886 887 static cpumask_t *smp_task_ipi_mask(struct task_struct *cur) 888 { 889 /* 890 * If cpumask_size() is smaller than or equal to the pointer 891 * size, it stashes the cpumask in the pointer itself to 892 * avoid extra memory allocations. 893 */ 894 if (static_branch_unlikely(&ipi_mask_inlined)) 895 return (cpumask_t *)&ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_val; 896 897 return ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_ptr; 898 } 899 #else 900 static cpumask_t *smp_task_ipi_mask(struct task_struct *cur) 901 { 902 return NULL; 903 } 904 #endif 905 906 /* 907 * Flags to be used as scf_flags argument of smp_call_function_many_cond(). 908 * 909 * %SCF_WAIT: Wait until function execution is completed 910 * %SCF_RUN_LOCAL: Run also locally if local cpu is set in cpumask 911 */ 912 #define SCF_WAIT (1U << 0) 913 #define SCF_RUN_LOCAL (1U << 1) 914 915 static void smp_call_function_many_cond(const struct cpumask *mask, 916 smp_call_func_t func, void *info, 917 unsigned int scf_flags, 918 smp_cond_func_t cond_func) 919 { 920 struct cpumask *cpumask, *task_mask; 921 bool wait = scf_flags & SCF_WAIT; 922 struct call_function_data *cfd; 923 int cpu, last_cpu, this_cpu; 924 bool run_remote = false; 925 int nr_cpus = 0; 926 927 this_cpu = get_cpu(); 928 cfd = this_cpu_ptr(&cfd_data); 929 task_mask = smp_task_ipi_mask(current); 930 if (task_mask) 931 cpumask = task_mask; 932 else 933 cpumask = cfd->cpumask; 934 935 /* 936 * Can deadlock when called with interrupts disabled. 937 * We allow cpu's that are not yet online though, as no one else can 938 * send smp call function interrupt to this cpu and as such deadlocks 939 * can't happen. 940 */ 941 if (cpu_online(this_cpu) && !oops_in_progress && 942 !early_boot_irqs_disabled) 943 lockdep_assert_irqs_enabled(); 944 945 /* 946 * When @wait we can deadlock when we interrupt between llist_add() and 947 * arch_send_call_function_ipi*(); when !@wait we can deadlock due to 948 * csd_lock() on because the interrupt context uses the same csd 949 * storage. 950 */ 951 WARN_ON_ONCE(!in_task()); 952 953 /* Check if we need remote execution, i.e., any CPU excluding this one. */ 954 if (cpumask_any_and_but(mask, cpu_online_mask, this_cpu) < nr_cpu_ids) { 955 cpumask_and(cpumask, mask, cpu_online_mask); 956 __cpumask_clear_cpu(this_cpu, cpumask); 957 958 cpumask_clear(cfd->cpumask_ipi); 959 for_each_cpu(cpu, cpumask) { 960 call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu); 961 962 if (cond_func && !cond_func(cpu, info)) { 963 __cpumask_clear_cpu(cpu, cpumask); 964 continue; 965 } 966 967 /* Work is enqueued on a remote CPU. */ 968 run_remote = true; 969 970 csd_lock(csd); 971 if (wait) 972 csd->node.u_flags |= CSD_TYPE_SYNC; 973 csd->func = func; 974 csd->info = info; 975 #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG 976 csd->node.src = this_cpu; 977 csd->node.dst = cpu; 978 #endif 979 trace_csd_queue_cpu(cpu, _RET_IP_, func, csd); 980 981 /* 982 * Kick the remote CPU if this is the first work 983 * item enqueued. 984 */ 985 if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) { 986 __cpumask_set_cpu(cpu, cfd->cpumask_ipi); 987 nr_cpus++; 988 last_cpu = cpu; 989 } 990 } 991 992 /* 993 * Choose the most efficient way to send an IPI. Note that the 994 * number of CPUs might be zero due to concurrent changes to the 995 * provided mask. 996 */ 997 if (nr_cpus == 1) 998 send_call_function_single_ipi(last_cpu); 999 else if (likely(nr_cpus > 1)) 1000 send_call_function_ipi_mask(cfd->cpumask_ipi); 1001 } 1002 1003 /* Check if we need local execution. */ 1004 if ((scf_flags & SCF_RUN_LOCAL) && cpumask_test_cpu(this_cpu, mask) && 1005 (!cond_func || cond_func(this_cpu, info))) { 1006 unsigned long flags; 1007 1008 local_irq_save(flags); 1009 csd_do_func(func, info, NULL); 1010 local_irq_restore(flags); 1011 } 1012 1013 /* 1014 * The IPI work has been queued and dispatched. On PREEMPT kernels, 1015 * tasks created through dup_task_struct() have task-local wait masks. 1016 * The boot init_task can fall back to cfd->cpumask when the mask is 1017 * not inlined, but other tasks still use task-local masks and cannot 1018 * overwrite it. On !PREEMPT kernels, preempt_enable() cannot schedule 1019 * another task, so the per-CPU mask remains protected. 1020 */ 1021 put_cpu(); 1022 1023 if (run_remote && wait) { 1024 for_each_cpu(cpu, cpumask) { 1025 call_single_data_t *csd; 1026 1027 csd = per_cpu_ptr(cfd->csd, cpu); 1028 csd_lock_wait(csd); 1029 } 1030 } 1031 } 1032 1033 /** 1034 * smp_call_function_many() - Run a function on a set of CPUs. 1035 * @mask: The set of cpus to run on (only runs on online subset). 1036 * @func: The function to run. This must be fast and non-blocking. 1037 * @info: An arbitrary pointer to pass to the function. 1038 * @wait: If true, wait (atomically) until function has completed 1039 * on other CPUs. 1040 * 1041 * You must not call this function with disabled interrupts or from a 1042 * hardware interrupt handler or from a bottom half handler. 1043 * 1044 * @func is not called on the local CPU even if @mask contains it. Consider 1045 * using on_each_cpu_cond_mask() instead if this is not desirable. 1046 */ 1047 void smp_call_function_many(const struct cpumask *mask, 1048 smp_call_func_t func, void *info, bool wait) 1049 { 1050 smp_call_function_many_cond(mask, func, info, wait * SCF_WAIT, NULL); 1051 } 1052 EXPORT_SYMBOL(smp_call_function_many); 1053 1054 /** 1055 * smp_call_function() - Run a function on all other CPUs. 1056 * @func: The function to run. This must be fast and non-blocking. 1057 * @info: An arbitrary pointer to pass to the function. 1058 * @wait: If true, wait (atomically) until function has completed 1059 * on other CPUs. 1060 * 1061 * If @wait is true, then returns once @func has returned; otherwise 1062 * it returns just before the target cpu calls @func. 1063 * 1064 * You must not call this function with disabled interrupts or from a 1065 * hardware interrupt handler or from a bottom half handler. 1066 */ 1067 void smp_call_function(smp_call_func_t func, void *info, int wait) 1068 { 1069 smp_call_function_many_cond(cpu_online_mask, func, info, 1070 wait ? SCF_WAIT : 0, NULL); 1071 } 1072 EXPORT_SYMBOL(smp_call_function); 1073 1074 /* Setup configured maximum number of CPUs to activate */ 1075 unsigned int setup_max_cpus = NR_CPUS; 1076 EXPORT_SYMBOL(setup_max_cpus); 1077 1078 1079 /* 1080 * Setup routine for controlling SMP activation 1081 * 1082 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP 1083 * activation entirely (the MPS table probe still happens, though). 1084 * 1085 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer 1086 * greater than 0, limits the maximum number of CPUs activated in 1087 * SMP mode to <NUM>. 1088 */ 1089 1090 void __weak __init arch_disable_smp_support(void) { } 1091 1092 static int __init nosmp(char *str) 1093 { 1094 setup_max_cpus = 0; 1095 arch_disable_smp_support(); 1096 1097 return 0; 1098 } 1099 1100 early_param("nosmp", nosmp); 1101 1102 /* this is hard limit */ 1103 static int __init nrcpus(char *str) 1104 { 1105 int nr_cpus; 1106 1107 if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids) 1108 set_nr_cpu_ids(nr_cpus); 1109 1110 return 0; 1111 } 1112 1113 early_param("nr_cpus", nrcpus); 1114 1115 static int __init maxcpus(char *str) 1116 { 1117 get_option(&str, &setup_max_cpus); 1118 if (setup_max_cpus == 0) 1119 arch_disable_smp_support(); 1120 1121 return 0; 1122 } 1123 1124 early_param("maxcpus", maxcpus); 1125 1126 #if (NR_CPUS > 1) && !defined(CONFIG_FORCE_NR_CPUS) 1127 /* Setup number of possible processor ids */ 1128 unsigned int nr_cpu_ids __read_mostly = NR_CPUS; 1129 EXPORT_SYMBOL(nr_cpu_ids); 1130 #endif 1131 1132 /* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */ 1133 void __init setup_nr_cpu_ids(void) 1134 { 1135 set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1); 1136 1137 if (IS_ENABLED(CONFIG_PREEMPTION) && cpumask_size() <= sizeof(unsigned long)) 1138 static_branch_enable(&ipi_mask_inlined); 1139 } 1140 1141 /* Called by boot processor to activate the rest. */ 1142 void __init smp_init(void) 1143 { 1144 int num_nodes, num_cpus; 1145 1146 idle_threads_init(); 1147 cpuhp_threads_init(); 1148 1149 pr_info("Bringing up secondary CPUs ...\n"); 1150 1151 bringup_nonboot_cpus(setup_max_cpus); 1152 1153 num_nodes = num_online_nodes(); 1154 num_cpus = num_online_cpus(); 1155 pr_info("Brought up %d node%s, %d CPU%s\n", 1156 num_nodes, str_plural(num_nodes), num_cpus, str_plural(num_cpus)); 1157 1158 /* Any cleanup work */ 1159 smp_cpus_done(setup_max_cpus); 1160 } 1161 1162 /** 1163 * on_each_cpu_cond_mask() - Call a function on each processor for which 1164 * the supplied function cond_func returns true, optionally waiting 1165 * for all the required CPUs to finish. This may include the local 1166 * processor. 1167 * @cond_func: A callback function that is passed a cpu id and 1168 * the info parameter. The function is called 1169 * with preemption disabled. The function should 1170 * return a boolean value indicating whether to IPI 1171 * the specified CPU. 1172 * @func: The function to run on all applicable CPUs. 1173 * This must be fast and non-blocking. 1174 * @info: An arbitrary pointer to pass to both functions. 1175 * @wait: If true, wait until function has completed on other CPUs. 1176 * @mask: The set of cpus to run on (only runs on online subset). 1177 * 1178 * Target CPU selection and work queueing are done with preemption 1179 * disabled. This protects against CPUs going offline, but not against 1180 * CPUs coming online concurrently; newly online CPUs are not guaranteed 1181 * to be seen or sent an IPI. If @wait is true, the final wait for remote 1182 * completion happens after that preemption-disabled section. 1183 * 1184 * You must not call this function with disabled interrupts or 1185 * from a hardware interrupt handler or from a bottom half handler. 1186 */ 1187 void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, 1188 void *info, bool wait, const struct cpumask *mask) 1189 { 1190 unsigned int scf_flags = SCF_RUN_LOCAL; 1191 1192 if (wait) 1193 scf_flags |= SCF_WAIT; 1194 1195 smp_call_function_many_cond(mask, func, info, scf_flags, cond_func); 1196 } 1197 EXPORT_SYMBOL(on_each_cpu_cond_mask); 1198 1199 static void do_nothing(void *unused) 1200 { 1201 } 1202 1203 /** 1204 * kick_all_cpus_sync - Force all cpus out of idle 1205 * 1206 * Used to synchronize the update of pm_idle function pointer. It's 1207 * called after the pointer is updated and returns after the dummy 1208 * callback function has been executed on all cpus. The execution of 1209 * the function can only happen on the remote cpus after they have 1210 * left the idle function which had been called via pm_idle function 1211 * pointer. So it's guaranteed that nothing uses the previous pointer 1212 * anymore. 1213 */ 1214 void kick_all_cpus_sync(void) 1215 { 1216 /* Make sure the change is visible before we kick the cpus */ 1217 smp_mb(); 1218 smp_call_function(do_nothing, NULL, 1); 1219 } 1220 EXPORT_SYMBOL_GPL(kick_all_cpus_sync); 1221 1222 /** 1223 * wake_up_all_idle_cpus - break all cpus out of idle 1224 * wake_up_all_idle_cpus try to break all cpus which is in idle state even 1225 * including idle polling cpus, for non-idle cpus, we will do nothing 1226 * for them. 1227 */ 1228 void wake_up_all_idle_cpus(void) 1229 { 1230 int cpu; 1231 1232 for_each_possible_cpu(cpu) { 1233 preempt_disable(); 1234 if (cpu != smp_processor_id() && cpu_online(cpu)) 1235 wake_up_if_idle(cpu); 1236 preempt_enable(); 1237 } 1238 } 1239 EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus); 1240 1241 /** 1242 * cpus_peek_for_pending_ipi - Check for pending IPI for CPUs 1243 * @mask: The CPU mask for the CPUs to check. 1244 * 1245 * This function walks through the @mask to check if there are any pending IPIs 1246 * scheduled, for any of the CPUs in the @mask. It does not guarantee 1247 * correctness as it only provides a racy snapshot. 1248 * 1249 * Returns: true if there is a pending IPI scheduled and false otherwise. 1250 */ 1251 bool cpus_peek_for_pending_ipi(const struct cpumask *mask) 1252 { 1253 unsigned int cpu; 1254 1255 for_each_cpu(cpu, mask) { 1256 if (!llist_empty(per_cpu_ptr(&call_single_queue, cpu))) 1257 return true; 1258 } 1259 1260 return false; 1261 } 1262 1263 /** 1264 * struct smp_call_on_cpu_struct - Call a function on a specific CPU 1265 * @work: &work_struct 1266 * @done: &completion to signal 1267 * @func: function to call 1268 * @data: function's data argument 1269 * @ret: return value from @func 1270 * @cpu: target CPU (%-1 for any CPU) 1271 * 1272 * Used to call a function on a specific cpu and wait for it to return. 1273 * Optionally make sure the call is done on a specified physical cpu via vcpu 1274 * pinning in order to support virtualized environments. 1275 */ 1276 struct smp_call_on_cpu_struct { 1277 struct work_struct work; 1278 struct completion done; 1279 int (*func)(void *); 1280 void *data; 1281 int ret; 1282 int cpu; 1283 }; 1284 1285 static void smp_call_on_cpu_callback(struct work_struct *work) 1286 { 1287 struct smp_call_on_cpu_struct *sscs; 1288 1289 sscs = container_of(work, struct smp_call_on_cpu_struct, work); 1290 if (sscs->cpu >= 0) 1291 hypervisor_pin_vcpu(sscs->cpu); 1292 sscs->ret = sscs->func(sscs->data); 1293 if (sscs->cpu >= 0) 1294 hypervisor_pin_vcpu(-1); 1295 1296 complete(&sscs->done); 1297 } 1298 1299 /** 1300 * smp_call_on_cpu() - Call a function on a specific CPU and wait 1301 * for it to return. 1302 * @cpu: The CPU to run on. 1303 * @func: The function to run 1304 * @par: An arbitrary pointer parameter for @func. 1305 * @phys: If @true, force to run on physical @cpu. See 1306 * &struct smp_call_on_cpu_struct for more info. 1307 * 1308 * Returns: %-ENXIO if the @cpu is invalid; otherwise the return value 1309 * from @func. 1310 */ 1311 int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par, bool phys) 1312 { 1313 struct smp_call_on_cpu_struct sscs = { 1314 .done = COMPLETION_INITIALIZER_ONSTACK(sscs.done), 1315 .func = func, 1316 .data = par, 1317 .cpu = phys ? cpu : -1, 1318 }; 1319 1320 INIT_WORK_ONSTACK(&sscs.work, smp_call_on_cpu_callback); 1321 1322 if (cpu >= nr_cpu_ids || !cpu_online(cpu)) 1323 return -ENXIO; 1324 1325 queue_work_on(cpu, system_percpu_wq, &sscs.work); 1326 wait_for_completion(&sscs.done); 1327 destroy_work_on_stack(&sscs.work); 1328 1329 return sscs.ret; 1330 } 1331 EXPORT_SYMBOL_GPL(smp_call_on_cpu); 1332