Lines Matching defs:rfp
3128 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)
3135 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)
3136 if (rfp->n_launders_hist[i].n_launders > 0)
3139 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat);
3140 gps_old = rfp->rcu_launder_gp_seq_start;
3142 gps = rfp->n_launders_hist[j].launder_gp_seq;
3145 rfp->n_launders_hist[j].n_launders,
3159 struct rcu_fwd *rfp = rfcp->rfc_rfp;
3163 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);
3164 rfcpp = rfp->rcu_fwd_cb_tail;
3165 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next;
3167 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1);
3168 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV));
3169 if (i >= ARRAY_SIZE(rfp->n_launders_hist))
3170 i = ARRAY_SIZE(rfp->n_launders_hist) - 1;
3171 rfp->n_launders_hist[i].n_launders++;
3172 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();
3173 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);
3193 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp)
3200 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);
3201 rfcp = rfp->rcu_fwd_cb_head;
3203 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);
3206 rfp->rcu_fwd_cb_head = rfcp->rfc_next;
3207 if (!rfp->rcu_fwd_cb_head)
3208 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;
3209 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);
3223 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp,
3237 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
3257 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);
3258 stopat = rfp->rcu_fwd_startat + dur;
3277 rfp->rcu_fwd_id, dur, cver, gps);
3282 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
3295 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp)
3311 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
3320 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);
3321 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES;
3323 rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread
3327 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++)
3328 rfp->n_launders_hist[i].n_launders = 0;
3331 rfp->rcu_launder_gp_seq_start = gps;
3336 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head);
3344 rfp->rcu_fwd_cb_head = rfcpn;
3356 rfcp->rfc_rfp = rfp;
3370 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb);
3373 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
3375 (void)rcu_torture_fwd_prog_cbfree(rfp);
3383 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat,
3389 rcu_torture_fwd_cb_hist(rfp);
3407 struct rcu_fwd *rfp;
3410 rfp = rcu_fwds;
3411 if (!rfp) {
3418 rcu_torture_fwd_cb_hist(&rfp[i]);
3419 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2);
3425 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
3430 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
3435 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
3455 struct rcu_fwd *rfp = args;
3466 if (!rfp->rcu_fwd_id) {
3480 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
3481 if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id)
3482 rcu_torture_fwd_prog_cr(rfp);
3486 torture_num_online_cpus() > rfp->rcu_fwd_id)))
3487 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);
3494 if (!rfp->rcu_fwd_id) {
3507 struct rcu_fwd *rfp;
3536 rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL);
3538 if (!rfp || !fwd_prog_tasks) {
3539 kfree(rfp);
3546 spin_lock_init(&rfp[i].rcu_fwd_lock);
3547 rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head;
3548 rfp[i].rcu_fwd_id = i;
3551 rcu_fwds = rfp;
3567 struct rcu_fwd *rfp;
3575 rfp = rcu_fwds;
3578 kfree(rfp);