103 		sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
104 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
142 	ssp->srcu_sup->node = kzalloc_objs(*ssp->srcu_sup->node, rcu_num_nodes,  in init_srcu_struct_nodes()
144 	if (!ssp->srcu_sup->node)  in init_srcu_struct_nodes()
148 	ssp->srcu_sup->level[0] = &ssp->srcu_sup->node[0];  in init_srcu_struct_nodes()
150 		ssp->srcu_sup->level[i] = ssp->srcu_sup->level[i - 1] + num_rcu_lvl[i - 1];  in init_srcu_struct_nodes()
165 		if (snp == &ssp->srcu_sup->node[0]) {  in init_srcu_struct_nodes()
172 		if (snp == ssp->srcu_sup->level[level + 1])  in init_srcu_struct_nodes()
174 		snp->srcu_parent = ssp->srcu_sup->level[level - 1] +  in init_srcu_struct_nodes()
175 				   (snp - ssp->srcu_sup->level[level]) /  in init_srcu_struct_nodes()
184 	snp_first = ssp->srcu_sup->level[level];  in init_srcu_struct_nodes()
195 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_WAIT_BARRIER);  in init_srcu_struct_nodes()
207 		ssp->srcu_sup = kzalloc_obj(*ssp->srcu_sup);  in init_srcu_struct_fields()
208 	if (!ssp->srcu_sup)  in init_srcu_struct_fields()
211 		raw_spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in init_srcu_struct_fields()
212 	ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;  in init_srcu_struct_fields()
213 	ssp->srcu_sup->node = NULL;  in init_srcu_struct_fields()
214 	mutex_init(&ssp->srcu_sup->srcu_cb_mutex);  in init_srcu_struct_fields()
215 	mutex_init(&ssp->srcu_sup->srcu_gp_mutex);  in init_srcu_struct_fields()
216 	ssp->srcu_sup->srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL;  in init_srcu_struct_fields()
217 	ssp->srcu_sup->srcu_barrier_seq = 0;  in init_srcu_struct_fields()
218 	mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);  in init_srcu_struct_fields()
219 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);  in init_srcu_struct_fields()
220 	INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);  in init_srcu_struct_fields()
221 	init_irq_work(&ssp->srcu_sup->irq_work, srcu_irq_work);  in init_srcu_struct_fields()
222 	ssp->srcu_sup->sda_is_static = is_static;  in init_srcu_struct_fields()
230 	ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL;  in init_srcu_struct_fields()
231 	ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();  in init_srcu_struct_fields()
232 	if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {  in init_srcu_struct_fields()
234 			WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC);  in init_srcu_struct_fields()
236 			WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);  in init_srcu_struct_fields()
240 	ssp->srcu_sup->srcu_ssp = ssp;  in init_srcu_struct_fields()
241 	smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed,  in init_srcu_struct_fields()
252 		kfree(ssp->srcu_sup);  in init_srcu_struct_fields()
253 		ssp->srcu_sup = NULL;  in init_srcu_struct_fields()
353 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in __srcu_transition_to_big()
354 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC);  in __srcu_transition_to_big()
365 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL)  in srcu_transition_to_big()
367 	raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
368 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) {  in srcu_transition_to_big()
369 		raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
373 	raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
384 	if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)  in raw_spin_lock_irqsave_check_contention()
387 	if (ssp->srcu_sup->srcu_size_jiffies != j) {  in raw_spin_lock_irqsave_check_contention()
388 		ssp->srcu_sup->srcu_size_jiffies = j;  in raw_spin_lock_irqsave_check_contention()
389 		ssp->srcu_sup->srcu_n_lock_retries = 0;  in raw_spin_lock_irqsave_check_contention()
391 	if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim)  in raw_spin_lock_irqsave_check_contention()
408 	raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in raw_spin_lock_irqsave_sdp_contention()
410 	raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags);  in raw_spin_lock_irqsave_sdp_contention()
422 	if (raw_spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags))  in raw_spin_lock_irqsave_ssp_contention()
424 	raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in raw_spin_lock_irqsave_ssp_contention()
441 	if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/  in check_init_srcu_struct()
443 	raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
444 	if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) {  in check_init_srcu_struct()
445 		raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
449 	raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
457 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_is_expedited()
682 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_get_delay()
684 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in srcu_get_delay()
713 	struct srcu_usage *sup = ssp->srcu_sup;  in cleanup_srcu_struct()
715 	raw_spin_lock_irq_rcu_node(ssp->srcu_sup);  in cleanup_srcu_struct()
717 	raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);  in cleanup_srcu_struct()
751 		ssp->srcu_sup = NULL;  in cleanup_srcu_struct()
851 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in srcu_gp_start()
852 	WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed));  in srcu_gp_start()
853 	WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies);  in srcu_gp_start()
854 	WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0);  in srcu_gp_start()
856 	rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
857 	state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
929 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_end()
1031 			if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) ||  in srcu_funnel_exp_start()
1044 	if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s))  in srcu_funnel_exp_start()
1045 		WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
1046 	raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_funnel_exp_start()
1071 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_funnel_gp_start()
1151 	raw_spin_lock_irq_rcu_node(ssp->srcu_sup);  in try_check_zero()
1153 	raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);  in try_check_zero()
1279 	tlast = READ_ONCE(ssp->srcu_sup->srcu_last_gp_end);  in srcu_should_expedite()
1285 	curseq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcu_should_expedite()
1287 	if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_sup->srcu_gp_seq_needed)))  in srcu_should_expedite()
1290 	if (curseq != rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq))  in srcu_should_expedite()
1324 	ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);  in srcu_gp_start_if_needed()
1368 	s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start_if_needed()
1371 				      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_gp_start_if_needed()
1598 	return rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in get_state_synchronize_srcu()
1646 	    !rcu_seq_done_exact(&ssp->srcu_sup->srcu_gp_seq, cookie))  in poll_state_synchronize_srcu()
1666 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier_cb()
1667 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier_cb()
1681 	atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1687 		atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1700 	unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1703 	mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1704 	if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) {  in srcu_barrier()
1706 		mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1709 	rcu_seq_start(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1710 	init_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1713 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 1);  in srcu_barrier()
1716 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_barrier()
1724 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier()
1725 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1726 	wait_for_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1728 	rcu_seq_end(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1729 	mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1800 	return READ_ONCE(ssp->srcu_sup->srcu_gp_seq);  in srcu_batches_completed()
1813 	mutex_lock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1825 	idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */  in srcu_advance_state()
1827 		raw_spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1828 		if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_advance_state()
1829 			WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1830 			raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1831 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1834 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1837 		raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1839 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1844 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) {  in srcu_advance_state()
1847 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1851 		raw_spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1852 		rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);  in srcu_advance_state()
1853 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1854 		raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1857 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) {  in srcu_advance_state()
1865 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1868 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1895 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_invoke_callbacks()
1944 	raw_spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1945 	if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_reschedule()
1946 		if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) {  in srcu_reschedule()
1950 	} else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) {  in srcu_reschedule()
1954 	raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1957 		queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay);  in srcu_reschedule()
1974 	raw_spin_lock_irq_rcu_node(ssp->srcu_sup);  in process_srcu()
1976 	raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);  in process_srcu()
2004 	raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in srcu_irq_work()
2006 	raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_irq_work()
2015 	*gp_seq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcutorture_get_gp_data()
2037 	int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state);  in srcu_torture_stats_print()
2044 		 tt, tf, rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq), ss_state,  in srcu_torture_stats_print()
2159 		if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) &&  in srcu_module_going()
2160 		    !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static))  in srcu_module_going()