Lines Matching refs:csd
32 #include <trace/events/csd.h>
41 call_single_data_t __percpu *csd;
66 cfd->csd = alloc_percpu(call_single_data_t);
67 if (!cfd->csd) {
82 free_percpu(cfd->csd);
133 csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd)
135 trace_csd_function_entry(func, csd);
137 trace_csd_function_exit(func, csd);
182 static void __csd_lock_record(call_single_data_t *csd)
184 if (!csd) {
189 __this_cpu_write(cur_csd_func, csd->func);
190 __this_cpu_write(cur_csd_info, csd->info);
191 smp_wmb(); /* func and info before csd. */
192 __this_cpu_write(cur_csd, csd);
197 static __always_inline void csd_lock_record(call_single_data_t *csd)
200 __csd_lock_record(csd);
203 static int csd_lock_wait_getcpu(call_single_data_t *csd)
207 csd_type = CSD_TYPE(csd);
209 return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */
231 static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id, unsigned long *nmessages)
238 unsigned int flags = READ_ONCE(csd->node.u_flags);
244 cpu = csd_lock_wait_getcpu(csd);
245 pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n",
270 cpu = csd_lock_wait_getcpu(csd);
278 pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n",
280 cpu, csd->func, csd->info);
290 if (cpu_cur_csd && csd != cpu_cur_csd) {
302 pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu);
314 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources
316 * For non-synchronous ipi calls the csd can still be in use by the
318 * as we'll have to ensure no other cpu is observing our csd.
320 static void __csd_lock_wait(call_single_data_t *csd)
328 if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages))
335 static __always_inline void csd_lock_wait(call_single_data_t *csd)
338 __csd_lock_wait(csd);
342 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
345 static void csd_lock_record(call_single_data_t *csd)
349 static __always_inline void csd_lock_wait(call_single_data_t *csd)
351 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
355 static __always_inline void csd_lock(call_single_data_t *csd)
357 csd_lock_wait(csd);
358 csd->node.u_flags |= CSD_FLAG_LOCK;
368 static __always_inline void csd_unlock(call_single_data_t *csd)
370 WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));
375 smp_store_release(&csd->node.u_flags, 0);
390 call_single_data_t *csd;
393 csd = container_of(node, call_single_data_t, node.llist);
394 func = CSD_TYPE(csd) == CSD_TYPE_TTWU ?
395 sched_ttwu_pending : csd->func;
397 trace_csd_queue_cpu(cpu, _RET_IP_, func, csd);
421 static int generic_exec_single(int cpu, call_single_data_t *csd)
428 smp_call_func_t func = csd->func;
429 void *info = csd->info;
436 csd_lock_record(csd);
437 csd_unlock(csd);
446 csd_unlock(csd);
450 __smp_call_single_queue(cpu, &csd->node.llist);
482 call_single_data_t *csd, *csd_next;
508 llist_for_each_entry(csd, entry, node.llist) {
509 switch (CSD_TYPE(csd)) {
514 csd->func);
523 CSD_TYPE(csd));
533 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
535 if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {
536 smp_call_func_t func = csd->func;
537 void *info = csd->info;
545 csd_lock_record(csd);
546 csd_do_func(func, info, csd);
547 csd_unlock(csd);
550 prev = &csd->node.llist;
561 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
562 int type = CSD_TYPE(csd);
572 smp_call_func_t func = csd->func;
573 void *info = csd->info;
575 csd_lock_record(csd);
576 csd_unlock(csd);
577 csd_do_func(func, info, csd);
580 irq_work_single(csd);
584 prev = &csd->node.llist;
592 csd = llist_entry(entry, typeof(*csd), node.llist);
593 csd_do_func(sched_ttwu_pending, entry, csd);
639 call_single_data_t *csd;
666 * csd_lock() on because the interrupt context uses the same csd
671 csd = &csd_stack;
673 csd = this_cpu_ptr(&csd_data);
674 csd_lock(csd);
677 csd->func = func;
678 csd->info = info;
680 csd->node.src = smp_processor_id();
681 csd->node.dst = cpu;
684 err = generic_exec_single(cpu, csd);
687 csd_lock_wait(csd);
699 * @csd: Pre-allocated and setup data structure
706 * such that the IPIs performed on the @csd are strictly serialized.
708 * If the function is called with one csd which has not yet been
710 * function will return immediately with -EBUSY showing that the csd
718 int smp_call_function_single_async(int cpu, call_single_data_t *csd)
724 if (csd->node.u_flags & CSD_FLAG_LOCK) {
729 csd->node.u_flags = CSD_FLAG_LOCK;
732 err = generic_exec_single(cpu, csd);
806 * csd_lock() on because the interrupt context uses the same csd
819 call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
829 csd_lock(csd);
831 csd->node.u_flags |= CSD_TYPE_SYNC;
832 csd->func = func;
833 csd->info = info;
835 csd->node.src = smp_processor_id();
836 csd->node.dst = cpu;
838 trace_csd_queue_cpu(cpu, _RET_IP_, func, csd);
844 if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) {
874 call_single_data_t *csd;
876 csd = per_cpu_ptr(cfd->csd, cpu);
877 csd_lock_wait(csd);