Lines Matching +full:report +full:- +full:rate +full:- +full:hz
1 // SPDX-License-Identifier: GPL-2.0+
24 #define RCU_STALL_DELAY_DELTA (5 * HZ)
29 #define RCU_STALL_MIGHT_MIN (2 * HZ)
45 till_stall_check = clamp(msecs_to_jiffies(cpu_stall_timeout), 2UL, 300UL * HZ); in rcu_exp_jiffies_till_stall_check()
59 /* Limit-check stall timeouts specified at boottime and runtime. */
75 return till_stall_check * HZ + RCU_STALL_DELAY_DELTA; in rcu_jiffies_till_stall_check()
110 /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */
123 * rcu_cpu_stall_reset - restart stall-warning timeout for current grace period
150 smp_mb(); // ->gp_start before ->jiffies_stall and caller's ->gp_seq. in record_gp_stall_check_time()
157 /* Zero ->ticks_this_gp and snapshot the number of RCU softirq handlers. */
160 rdp->ticks_this_gp = 0; in zero_cpu_stall_ticks()
161 rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id()); in zero_cpu_stall_ticks()
162 WRITE_ONCE(rdp->last_fqs_resched, jiffies); in zero_cpu_stall_ticks()
178 WARN_ONCE(1, "Kicking %s grace-period kthread\n", in rcu_stall_kick_kthreads()
182 WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ); in rcu_stall_kick_kthreads()
197 rnp = rdp->mynode; in rcu_iw_handler()
199 if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) { in rcu_iw_handler()
200 rdp->rcu_iw_gp_seq = rnp->gp_seq; in rcu_iw_handler()
201 rdp->rcu_iw_pending = false; in rcu_iw_handler()
226 t = list_entry(rnp->gp_tasks->prev, in rcu_print_detail_task_stall_rnp()
228 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { in rcu_print_detail_task_stall_rnp()
247 * Report out the state of a not-running task that is stalling the
255 return -EBUSY; // It is running, so decline to inspect it. in check_slow_task()
256 rscrp->nesting = t->rcu_read_lock_nesting; in check_slow_task()
257 rscrp->rs = t->rcu_read_unlock_special; in check_slow_task()
258 rscrp->on_blkd_list = !list_empty(&t->rcu_node_entry); in check_slow_task()
263 * Scan the current list of tasks blocked within RCU read-side critical
267 __releases(rnp->lock) in rcu_print_task_stall()
280 pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):", in rcu_print_task_stall()
281 rnp->level, rnp->grplo, rnp->grphi); in rcu_print_task_stall()
282 t = list_entry(rnp->gp_tasks->prev, in rcu_print_task_stall()
284 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { in rcu_print_task_stall()
292 t = ts[--i]; in rcu_print_task_stall()
294 pr_cont(" P%d", t->pid); in rcu_print_task_stall()
297 t->pid, rscr.nesting, in rcu_print_task_stall()
314 * tasks blocked within RCU read-side critical sections.
322 * tasks blocked within RCU read-side critical sections.
325 __releases(rnp->lock) in rcu_print_task_stall()
335 * that don't support NMI-based stack dumps. The NMI-triggered stack
352 if (!(data_race(rnp->qsmask) & leaf_node_cpu_bit(rnp, cpu))) in rcu_dump_cpu_stacks()
355 if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { in rcu_dump_cpu_stacks()
380 * Convert a ->gp_state value to a character string.
389 /* Is the RCU grace-period kthread being starved of CPU time? */
392 unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity); in rcu_is_gp_kthread_starving()
396 return j > 2 * HZ; in rcu_is_gp_kthread_starving()
405 rcuc = rdp->rcu_cpu_kthread_task; in rcu_is_rcuc_kthread_starving()
413 j = jiffies - READ_ONCE(rdp->rcuc_activity); in rcu_is_rcuc_kthread_starving()
417 return j > 2 * HZ; in rcu_is_rcuc_kthread_starving()
429 rsrp = &rdp->snap_record; in print_cpu_stat_info()
430 if (rsrp->gp_seq != rdp->gp_seq) in print_cpu_stat_info()
439 kstat_cpu_irqs_sum(cpu) - rsrp->nr_hardirqs, in print_cpu_stat_info()
440 kstat_cpu_softirqs_sum(cpu) - rsrp->nr_softirqs, in print_cpu_stat_info()
441 nr_context_switches_cpu(cpu) - rsrp->nr_csw); in print_cpu_stat_info()
443 div_u64(rsr.cputime_irq - rsrp->cputime_irq, NSEC_PER_MSEC), in print_cpu_stat_info()
444 div_u64(rsr.cputime_softirq - rsrp->cputime_softirq, NSEC_PER_MSEC), in print_cpu_stat_info()
445 div_u64(rsr.cputime_system - rsrp->cputime_system, NSEC_PER_MSEC), in print_cpu_stat_info()
446 jiffies_to_msecs(jiffies - rsrp->jiffies)); in print_cpu_stat_info()
477 ticks_value = rcu_seq_ctr(rcu_state.gp_seq - rdp->gp_seq); in print_cpu_stall_info()
482 ticks_value = rdp->ticks_this_gp; in print_cpu_stall_info()
484 delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq); in print_cpu_stall_info()
491 pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%04x/%ld/%#lx softirq=%u/%u fqs=%ld%s%s\n", in print_cpu_stall_info()
494 "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)], in print_cpu_stall_info()
495 "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)], in print_cpu_stall_info()
497 rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' : in print_cpu_stall_info()
502 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), in print_cpu_stall_info()
503 data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, in print_cpu_stall_info()
510 /* Complain about starvation of grace-period kthread. */
518 cpu = gpk ? task_cpu(gpk) : -1; in rcu_check_gp_kthread_starvation()
519 pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x ->cpu=%d\n", in rcu_check_gp_kthread_starvation()
525 gpk ? data_race(READ_ONCE(gpk->__state)) : ~0, cpu); in rcu_check_gp_kthread_starvation()
530 pr_err("RCU grace-period kthread stack dump:\n"); in rcu_check_gp_kthread_starvation()
534 } else if (!(data_race(READ_ONCE(rdp->mynode->qsmask)) & rdp->grpmask)) { in rcu_check_gp_kthread_starvation()
560 gpk && !READ_ONCE(gpk->on_rq)) { in rcu_check_gp_kthread_expired_fqs_timer()
562 pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x\n", in rcu_check_gp_kthread_expired_fqs_timer()
563 rcu_state.name, (jiffies - jiffies_fqs), in rcu_check_gp_kthread_expired_fqs_timer()
567 data_race(READ_ONCE(gpk->__state))); in rcu_check_gp_kthread_expired_fqs_timer()
568 pr_err("\tPossible timer handling issue on cpu=%d timer-softirq=%u\n", in rcu_check_gp_kthread_expired_fqs_timer()
601 if (rnp->qsmask != 0) { in print_other_cpu_stall()
603 if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { in print_other_cpu_stall()
608 ndetected += rcu_print_task_stall(rnp, flags); // Releases rnp->lock. in print_other_cpu_stall()
615 smp_processor_id(), (long)(jiffies - gps), in print_other_cpu_stall()
630 …pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->q… in print_other_cpu_stall()
631 rcu_state.name, j - gpa, j, gpa, in print_other_cpu_stall()
633 data_race(READ_ONCE(rcu_get_root()->qsmask))); in print_other_cpu_stall()
674 pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name); in print_cpu_stall()
675 raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags); in print_cpu_stall()
677 raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags); in print_cpu_stall()
681 jiffies - gps, in print_cpu_stall()
734 * loop has to set jiffies to ensure a non-stale jiffies value. This in check_cpu_stall()
750 * grace-period initialization and cleanup. Now, a false positive in check_cpu_stall()
762 smp_rmb(); /* Pick up ->gp_seq first... */ in check_cpu_stall()
764 smp_rmb(); /* ...then ->jiffies_stall before the rest... */ in check_cpu_stall()
766 smp_rmb(); /* ...and finally ->gp_start before ->gp_seq again. */ in check_cpu_stall()
773 rnp = rdp->mynode; in check_cpu_stall()
775 self_detected = READ_ONCE(rnp->qsmask) & rdp->grpmask; in check_cpu_stall()
787 rcu_stall_notifier_call_chain(RCU_STALL_NOTIFY_NORM, (void *)j - gps); in check_cpu_stall()
789 …pr_err("INFO: %s detected stall, but suppressed full report due to a stuck CSD-lock.\n", rcu_state… in check_cpu_stall()
810 // RCU forward-progress mechanisms, including for callback invocation.
821 * in need of being boosted, *cpup is set to -1. This can happen in case
826 * for high-rate usage. On the other hand, if cpup is non-NULL, each
827 * rcu_node structure's ->lock is acquired, ruling out high-rate usage.
838 if (data_race(READ_ONCE(rnp->qsmask))) { in rcu_check_boost_fail()
841 if (READ_ONCE(rnp->gp_tasks)) in rcu_check_boost_fail()
846 *cpup = -1; in rcu_check_boost_fail()
848 if (rnp->gp_tasks) in rcu_check_boost_fail()
850 if (!rnp->qsmask) { in rcu_check_boost_fail()
857 if (rnp->qsmask & (1UL << (cpu - rnp->grplo))) { in rcu_check_boost_fail()
871 * Show the state of the grace-period kthreads.
887 ja = j - data_race(READ_ONCE(rcu_state.gp_activity)); in show_rcu_gp_kthreads()
888 jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity)); in show_rcu_gp_kthreads()
889 js = j - data_race(READ_ONCE(rcu_state.gp_start)); in show_rcu_gp_kthreads()
890 jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time)); in show_rcu_gp_kthreads()
891 …->state: %#x ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wa… in show_rcu_gp_kthreads()
894 t ? data_race(READ_ONCE(t->__state)) : 0x1ffff, t ? t->rt_priority : 0xffU, in show_rcu_gp_kthreads()
897 (long)data_race(READ_ONCE(rcu_get_root()->gp_seq_needed)), in show_rcu_gp_kthreads()
901 if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) && in show_rcu_gp_kthreads()
902 !data_race(READ_ONCE(rnp->qsmask)) && !data_race(READ_ONCE(rnp->boost_tasks)) && in show_rcu_gp_kthreads()
903 !data_race(READ_ONCE(rnp->exp_tasks)) && !data_race(READ_ONCE(rnp->gp_tasks))) in show_rcu_gp_kthreads()
905 …pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n… in show_rcu_gp_kthreads()
906 rnp->grplo, rnp->grphi, in show_rcu_gp_kthreads()
907 (long)data_race(READ_ONCE(rnp->gp_seq)), in show_rcu_gp_kthreads()
908 (long)data_race(READ_ONCE(rnp->gp_seq_needed)), in show_rcu_gp_kthreads()
909 data_race(READ_ONCE(rnp->qsmask)), in show_rcu_gp_kthreads()
910 ".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))], in show_rcu_gp_kthreads()
911 ".B"[!!data_race(READ_ONCE(rnp->boost_tasks))], in show_rcu_gp_kthreads()
912 ".E"[!!data_race(READ_ONCE(rnp->exp_tasks))], in show_rcu_gp_kthreads()
913 ".G"[!!data_race(READ_ONCE(rnp->gp_tasks))], in show_rcu_gp_kthreads()
914 data_race(READ_ONCE(rnp->n_boosts))); in show_rcu_gp_kthreads()
919 if (READ_ONCE(rdp->gpwrap) || in show_rcu_gp_kthreads()
921 READ_ONCE(rdp->gp_seq_needed))) in show_rcu_gp_kthreads()
923 pr_info("\tcpu %d ->gp_seq_needed %ld\n", in show_rcu_gp_kthreads()
924 cpu, (long)data_race(READ_ONCE(rdp->gp_seq_needed))); in show_rcu_gp_kthreads()
929 cbs += data_race(READ_ONCE(rdp->n_cbs_invoked)); in show_rcu_gp_kthreads()
930 if (rcu_segcblist_is_offloaded(&rdp->cblist)) in show_rcu_gp_kthreads()
939 * This function checks for grace-period requests that fail to motivate
951 ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq), in rcu_check_gp_start_stall()
952 READ_ONCE(rnp_root->gp_seq_needed)) || in rcu_check_gp_start_stall()
964 ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq), in rcu_check_gp_start_stall()
965 READ_ONCE(rnp_root->gp_seq_needed)) || in rcu_check_gp_start_stall()
978 ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq), in rcu_check_gp_start_stall()
979 READ_ONCE(rnp_root->gp_seq_needed)) || in rcu_check_gp_start_stall()
997 * Do a forward-progress check for rcutorture. This is normally invoked
1006 int max_cpu = -1; in rcu_fwd_progress_check()
1011 __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_start))); in rcu_fwd_progress_check()
1015 __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end))); in rcu_fwd_progress_check()
1018 rcu_check_gp_start_stall(rdp->mynode, rdp, j); in rcu_fwd_progress_check()
1042 /* Dump grace-period-request information due to commandeered sysrq. */
1050 .help_msg = "show-rcu(y)",
1067 // RCU CPU stall-warning notifiers
1072 * rcu_stall_chain_notifier_register - Add an RCU CPU stall notifier
1080 * Returns 0 on success, %-EEXIST on error.
1086 WARN(1, "Adding %pS() to RCU stall notifier list (%s).\n", n->notifier_call, in rcu_stall_chain_notifier_register()
1090 return -EEXIST; in rcu_stall_chain_notifier_register()
1095 * rcu_stall_chain_notifier_unregister - Remove an RCU CPU stall notifier
1100 * Returns zero on success, %-ENOENT on failure.
1109 * rcu_stall_notifier_call_chain - Call functions in an RCU CPU stall notifier chain
1118 * to indicate a non-kerneldoc format header comment.