Lines Matching defs:gp_seq
95 .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
222 return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));
523 return READ_ONCE(rcu_state.gp_seq);
550 void rcutorture_get_gp_data(int *flags, unsigned long *gp_seq)
553 *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
560 return ((READ_ONCE(rcu_state.gp_seq) & 0xffffULL) << 40) |
798 * of the rcu_node ->gp_seq counter with respect to the rcu_data counters.
805 if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + seq_gpwrap_lag,
806 rnp->gp_seq)) {
810 if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq))
811 rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4;
834 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
870 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
896 pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
898 (long)rnp->gp_seq, (long)rnp->completedqs);
962 !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
965 rdp->rcu_iw_gp_seq = rnp->gp_seq;
969 if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq != rdp->gp_seq) {
984 rsrp->gp_seq = rdp->gp_seq;
995 trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
1004 * @gp_seq_req: The gp_seq of the grace period to start.
1037 rcu_seq_started(&rnp->gp_seq, gp_seq_req) ||
1039 rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) {
1045 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
1074 trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));
1096 needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed);
1098 rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */
1099 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq,
1127 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
1132 * If there is room, assign a ->gp_seq number to any callbacks on this
1134 * that were previously assigned a ->gp_seq number that has since proven
1136 * ->gp_seq number while RCU is idle, but with reference to a non-root
1167 gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);
1196 c = rcu_seq_snap(&rcu_state.gp_seq);
1212 * assign ->gp_seq numbers to any callbacks in the RCU_NEXT_TAIL
1229 * Find all callbacks whose ->gp_seq numbers indicate that they
1232 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq);
1246 if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) || !raw_spin_trylock_rcu_node(rnp))
1249 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
1281 if (rdp->gp_seq == rnp->gp_seq)
1285 if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) ||
1290 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
1299 if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) ||
1306 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
1312 rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
1330 if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) &&
1373 !(rcu_seq_ctr(rcu_state.gp_seq) % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
1835 * A new wait segment must be started before gp_seq advanced, so
1836 * that previous gp waiters won't observe the new gp_seq.
1840 old_gp_seq = rcu_state.gp_seq;
1849 * Also note that the root rnp's gp_seq is kept separate from, and lags,
1850 * the rcu_state's gp_seq, for a reason. See the Quick-Quiz on
1853 rcu_seq_start(&rcu_state.gp_seq);
1856 rcu_seq_done_exact(&old_gp_seq, rcu_seq_snap(&rcu_state.gp_seq)));
1858 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
1859 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
1960 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);
1964 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,
1976 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2092 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2115 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2123 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2134 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2180 * Propagate new ->gp_seq value to rcu_node structures so that
2188 new_gp_seq = rcu_state.gp_seq;
2195 WRITE_ONCE(rnp->gp_seq, new_gp_seq);
2217 raw_spin_lock_irq_rcu_node(rnp); /* GP before ->gp_seq update. */
2220 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));
2221 rcu_seq_end(&rcu_state.gp_seq);
2222 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
2226 if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) {
2247 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq"));
2278 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2292 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2332 * are valid only if rnp->gp_seq is equal to gps. That structure's lock
2350 if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) {
2363 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
2373 rnp->completedqs = rnp->gp_seq;
2419 rnp->completedqs = rnp->gp_seq;
2430 /* Report up the rest of the hierarchy, tracking current ->gp_seq. */
2431 gps = rnp->gp_seq;
2452 if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq ||
2486 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2571 * completion (materialized by rnp->gp_seq update) thanks to the
2776 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
3381 rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT);
3383 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;
3414 * before the load from ->gp_seq.
3441 * before the loads from ->gp_seq and ->expedited_sequence.
3445 // Yes, rcu_state.gp_seq, not rnp_root->gp_seq, the latter's use
3449 rgosp->rgos_norm = rcu_seq_snap(&rcu_state.gp_seq);
3475 needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq));
3492 unsigned long gp_seq = get_state_synchronize_rcu();
3495 return gp_seq;
3587 * ->gp_seq field be checked instead of that of the rcu_state structure.
3589 * invoked between the time that the root rcu_node structure's ->gp_seq
3590 * field is updated and the time that the rcu_state structure's ->gp_seq
3601 rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) ||
3710 if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq ||
4152 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
4154 rdp->rcu_onl_gp_seq = rcu_state.gp_seq;
4236 * accept some slop in the rsp->gp_seq access due to the fact that this
4269 rdp->gp_seq = READ_ONCE(rnp->gp_seq);
4270 rdp->gp_seq_needed = rdp->gp_seq;
4275 rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
4276 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
4362 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);
4373 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
4426 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
4431 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
4524 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
4636 // Fix up the ->gp_seq counters.
4639 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;
4688 rnp->gp_seq = rcu_state.gp_seq;
4689 rnp->gp_seq_needed = rcu_state.gp_seq;
4690 rnp->completedqs = rcu_state.gp_seq;