Lines Matching defs:cur_ops
440 static struct rcu_torture_ops *cur_ops;
472 started = cur_ops->get_gp_seq();
478 completed = cur_ops->get_gp_seq();
479 do_trace_rcu_torture_read(cur_ops->name, NULL, ts,
564 cur_ops->deferred_free(rp);
771 return !!(cur_ops->have_up_down & rf);
1152 if (!cur_ops->gp_diff)
1154 return cur_ops->gp_diff(new, old);
1204 if (cur_ops->poll_gp_state(gp_state))
1206 if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) {
1221 if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) {
1224 cur_ops->gp_kthread_dbg();
1226 gp_done = cur_ops->poll_gp_state(gp_state);
1233 } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) {
1285 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))
1288 if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) {
1289 gp_state = cur_ops->start_gp_poll();
1298 if (cur_ops->poll_gp_state(gp_state))
1306 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))
1366 cur_ops->fqs();
1415 if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {
1418 } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) {
1421 if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) {
1424 } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) {
1427 if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) {
1430 } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) {
1433 if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) {
1437 (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) {
1440 if (gp_exp1 && cur_ops->exp_sync) {
1443 } else if (gp_exp && !cur_ops->exp_sync) {
1446 if (gp_normal1 && cur_ops->deferred_free) {
1449 } else if (gp_normal && !cur_ops->deferred_free) {
1452 if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state &&
1453 cur_ops->start_gp_poll && cur_ops->poll_gp_state) {
1456 } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {
1459 if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full
1460 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {
1463 } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {
1466 if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {
1469 } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) {
1472 if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) {
1476 (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) {
1479 if (gp_sync1 && cur_ops->sync) {
1482 } else if (gp_sync && !cur_ops->sync) {
1502 dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300);
1503 dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00);
1507 cookie = cur_ops->get_gp_state();
1509 cur_ops->get_gp_state_full(&cookie_full);
1510 if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full))
1513 WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie),
1516 WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full),
1562 torture_type, cur_ops->name);
1574 if (cur_ops->poll_active > 0) {
1575 ulo = kcalloc(cur_ops->poll_active, sizeof(*ulo), GFP_KERNEL);
1577 ulo_size = cur_ops->poll_active;
1579 if (cur_ops->poll_active_full > 0) {
1580 rgo = kcalloc(cur_ops->poll_active_full, sizeof(*rgo), GFP_KERNEL);
1582 rgo_size = cur_ops->poll_active_full;
1621 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) {
1622 idx = cur_ops->readlock();
1623 cookie = cur_ops->get_gp_state();
1624 WARN_ONCE(cur_ops->poll_gp_state(cookie),
1629 cookie, cur_ops->get_gp_state());
1630 if (cur_ops->get_comp_state) {
1631 cookie = cur_ops->get_comp_state();
1632 WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie));
1634 cur_ops->readunlock(idx);
1636 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) {
1637 idx = cur_ops->readlock();
1638 cur_ops->get_gp_state_full(&cookie_full);
1639 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),
1645 if (cur_ops->get_comp_state_full) {
1646 cur_ops->get_comp_state_full(&cookie_full);
1647 WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full));
1649 cur_ops->readunlock(idx);
1654 cur_ops->deferred_free(old_rp);
1658 do_rtws_sync(&rand, cur_ops->exp_sync);
1663 gp_snap = cur_ops->get_gp_state();
1667 cur_ops->cond_sync(gp_snap);
1672 gp_snap = cur_ops->get_gp_state_exp();
1676 cur_ops->cond_sync_exp(gp_snap);
1681 cur_ops->get_gp_state_full(&gp_snap_full);
1685 cur_ops->cond_sync_full(&gp_snap_full);
1690 cur_ops->get_gp_state_full(&gp_snap_full);
1694 cur_ops->cond_sync_exp_full(&gp_snap_full);
1700 ulo[i] = cur_ops->get_comp_state();
1701 gp_snap = cur_ops->start_gp_poll();
1703 while (!cur_ops->poll_gp_state(gp_snap)) {
1704 gp_snap1 = cur_ops->get_gp_state();
1706 if (cur_ops->poll_gp_state(ulo[i]) ||
1707 cur_ops->same_gp_state(ulo[i], gp_snap1)) {
1720 cur_ops->get_comp_state_full(&rgo[i]);
1721 cur_ops->start_gp_poll_full(&gp_snap_full);
1723 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1724 cur_ops->get_gp_state_full(&gp_snap1_full);
1726 if (cur_ops->poll_gp_state_full(&rgo[i]) ||
1727 cur_ops->same_gp_state_full(&rgo[i],
1740 gp_snap = cur_ops->start_gp_poll_exp();
1742 while (!cur_ops->poll_gp_state_exp(gp_snap))
1749 cur_ops->start_gp_poll_exp_full(&gp_snap_full);
1751 while (!cur_ops->poll_gp_state_full(&gp_snap_full))
1758 do_rtws_sync(&rand, cur_ops->sync);
1786 !cur_ops->slow_gps &&
1793 if (cur_ops->gp_kthread_dbg)
1794 cur_ops->gp_kthread_dbg();
1847 if (cur_ops->cb_barrier != NULL &&
1849 cur_ops->cb_barrier();
1855 cur_ops->exp_sync();
1858 gp_snap = cur_ops->get_gp_state();
1860 cur_ops->cond_sync(gp_snap);
1863 gp_snap = cur_ops->get_gp_state_exp();
1865 cur_ops->cond_sync_exp(gp_snap);
1868 cur_ops->get_gp_state_full(&gp_snap_full);
1870 cur_ops->cond_sync_full(&gp_snap_full);
1873 cur_ops->get_gp_state_full(&gp_snap_full);
1875 cur_ops->cond_sync_exp_full(&gp_snap_full);
1878 if (cur_ops->start_poll_irqsoff)
1880 gp_snap = cur_ops->start_gp_poll();
1881 if (cur_ops->start_poll_irqsoff)
1883 while (!cur_ops->poll_gp_state(gp_snap)) {
1889 if (cur_ops->start_poll_irqsoff)
1891 cur_ops->start_gp_poll_full(&gp_snap_full);
1892 if (cur_ops->start_poll_irqsoff)
1894 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1900 gp_snap = cur_ops->start_gp_poll_exp();
1901 while (!cur_ops->poll_gp_state_exp(gp_snap)) {
1907 cur_ops->start_gp_poll_exp_full(&gp_snap_full);
1908 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1914 cur_ops->sync();
2009 WARN_ONCE(cur_ops->readlock_nesting &&
2011 cur_ops->readlock_nesting() == 0, ROEC_ARGS);
2018 WARN_ONCE(cur_ops->extendables &&
2030 WARN_ONCE(cur_ops->extendables && !(curstate & mask) &&
2044 WARN_ONCE(cur_ops->readlock_nesting && !(curstate & mask) &&
2045 cur_ops->readlock_nesting() > 0, ROEC_ARGS);
2087 idxnew1 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1;
2089 idxnew2 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_2) & RCUTORTURE_RDR_MASK_2;
2100 if (cur_ops->reader_blocked)
2101 rtrsp[-1].rt_preempted = cur_ops->reader_blocked();
2105 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && cur_ops->gather_gp_seqs) {
2106 rtrsp->rt_gp_seq = cur_ops->gather_gp_seqs();
2130 cur_ops->readunlock((idxold2 & RCUTORTURE_RDR_MASK_2) >> RCUTORTURE_RDR_SHIFT_2);
2137 lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff);
2140 cur_ops->readunlock((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1);
2147 cur_ops->up_read((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1);
2154 cur_ops->read_delay(trsp, rtrsp);
2176 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables;
2283 if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
2284 rtorsp->cookie = cur_ops->get_gp_state();
2285 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
2286 cur_ops->get_gp_state_full(&rtorsp->cookie_full);
2288 rtorsp->started = cur_ops->get_gp_seq();
2291 !cur_ops->readlock_held || cur_ops->readlock_held() ||
2324 completed = cur_ops->get_gp_seq();
2326 do_trace_rcu_torture_read(cur_ops->name, &rtorsp->p->rtort_rcu,
2339 if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
2340 WARN_ONCE(cur_ops->poll_gp_state(rtorsp->cookie),
2345 rtorsp->cookie, cur_ops->get_gp_state());
2346 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
2347 WARN_ONCE(cur_ops->poll_gp_state_full(&rtorsp->cookie_full),
2354 if (cur_ops->reader_blocked)
2355 preempted = cur_ops->reader_blocked();
2410 if (cur_ops->call) {
2414 cur_ops->call(rhp, rcu_torture_timer_cb);
2435 if (irqreader && cur_ops->irq_capable)
2439 if (irqreader && cur_ops->irq_capable) {
2454 if (irqreader && cur_ops->irq_capable) {
2555 rawidx = cur_ops->down_read();
2686 if (cur_ops->get_gpwrap_count)
2687 n_gpwraps += cur_ops->get_gpwrap_count(cpu);
2765 if (cur_ops->stats)
2766 cur_ops->stats();
2774 if (cur_ops->get_gp_data)
2775 cur_ops->get_gp_data(&flags, &gp_seq);
2786 if (cur_ops->gp_kthread_dbg)
2787 cur_ops->gp_kthread_dbg();
2861 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
2882 test_boost, cur_ops->can_boost,
2996 idx = cur_ops->readlock();
3018 cur_ops->readunlock(idx);
3089 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb);
3172 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();
3238 if (!cur_ops->sync)
3240 if (cur_ops->call && cur_ops->cb_barrier) {
3247 cur_ops->sync(); /* Later readers see above write. */
3250 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);
3253 gps = cur_ops->get_gp_seq();
3254 sd = cur_ops->stall_dur() + 1;
3262 idx = cur_ops->readlock();
3264 cur_ops->readunlock(idx);
3274 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
3281 cur_ops->sync(); /* Wait for running CB to complete. */
3282 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
3283 cur_ops->cb_barrier(); /* Wait for queued callbacks. */
3314 if (!cur_ops->call)
3319 cur_ops->sync(); /* Later readers see above write. */
3330 gps = cur_ops->get_gp_seq();
3347 } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) {
3361 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
3372 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
3373 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
3374 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */
3379 if (WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED) && cur_ops->gp_kthread_dbg)
3380 cur_ops->gp_kthread_dbg();
3427 cur_ops->cb_barrier();
3432 cur_ops->cb_barrier();
3483 if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) &&
3517 if ((!cur_ops->sync && !cur_ops->call) ||
3518 (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) ||
3519 cur_ops == &rcu_busted_ops) {
3594 cur_ops->call(rhp, rcu_torture_barrier_cbf);
3622 cur_ops->call(&rcu, rcu_torture_barrier_cbf);
3627 if (cur_ops->cb_barrier != NULL)
3628 cur_ops->cb_barrier();
3653 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */
3666 cur_ops->cb_barrier();
3691 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
3694 torture_type, cur_ops->name);
3744 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2)
3746 if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)
3915 cur_ops->set_gpwrap_lag(0);
3920 cur_ops->set_gpwrap_lag(gpwrap_lag_gps);
3953 cur_ops->set_gpwrap_lag(0);
3966 if (cur_ops->cb_barrier != NULL) {
3967 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
3968 cur_ops->cb_barrier();
3970 if (cur_ops->gp_slow_unregister)
3971 cur_ops->gp_slow_unregister(NULL);
3974 if (!cur_ops) {
3981 if (cur_ops->gp_kthread_dbg)
3982 cur_ops->gp_kthread_dbg();
4019 if (cur_ops->get_gp_data)
4020 cur_ops->get_gp_data(&flags, &gp_seq);
4022 cur_ops->name, (long)gp_seq, flags,
4033 if (cur_ops->cb_barrier != NULL) {
4034 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
4035 cur_ops->cb_barrier();
4037 if (cur_ops->cleanup != NULL)
4038 cur_ops->cleanup();
4068 cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) {
4073 cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq,
4075 cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end,
4120 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
4122 rcu_torture_print_module_parms(cur_ops,
4125 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
4127 if (cur_ops->gp_slow_unregister)
4128 cur_ops->gp_slow_unregister(NULL);
4130 if (gpwrap_lag && cur_ops->set_gpwrap_lag)
4165 KBUILD_MODNAME, cur_ops->name);
4169 if (WARN_ON_ONCE(cur_ops->debug_objects &&
4170 (!cur_ops->call || !cur_ops->cb_barrier)))
4177 pr_alert("%s: WARN: Duplicate call_%s() test starting.\n", KBUILD_MODNAME, cur_ops->name);
4180 idx = cur_ops->readlock(); /* Make it impossible to finish a grace period. */
4181 cur_ops->call(&rh1, rcu_torture_leak_cb); /* Start grace period. */
4182 cur_ops->call(&rh2, rcu_torture_leak_cb);
4183 cur_ops->call(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
4185 cur_ops->call(rhp, rcu_torture_leak_cb);
4186 cur_ops->call(rhp, rcu_torture_err_cb); /* Another duplicate callback. */
4188 cur_ops->readunlock(idx);
4191 cur_ops->cb_barrier();
4192 pr_alert("%s: WARN: Duplicate call_%s() test complete.\n", KBUILD_MODNAME, cur_ops->name);
4202 if (cur_ops->sync && !(++n & 0xfff))
4203 cur_ops->sync();
4407 cur_ops = torture_ops[i];
4408 if (strcmp(torture_type, cur_ops->name) == 0)
4419 cur_ops = NULL;
4422 if (cur_ops->fqs == NULL && fqs_duration != 0) {
4426 if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops ||
4429 cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU));
4432 if (cur_ops->init)
4433 cur_ops->init();
4452 rcu_torture_print_module_parms(cur_ops, "Start of test");
4453 if (cur_ops->get_gp_data)
4454 cur_ops->get_gp_data(&flags, &gp_seq);
4457 cur_ops->name, (long)gp_seq, flags);
4574 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ;
4633 if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister))
4634 cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay);
4636 if (gpwrap_lag && cur_ops->set_gpwrap_lag) {