486 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_is_expedited()  local
488 	return ULONG_CMP_LT(READ_ONCE(sup->srcu_gp_seq), READ_ONCE(sup->srcu_gp_seq_needed_exp));  in srcu_gp_is_expedited()
711 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_get_delay()  local
716 	if (rcu_seq_state(READ_ONCE(sup->srcu_gp_seq))) {  in srcu_get_delay()
718 		gpstart = READ_ONCE(sup->srcu_gp_start);  in srcu_get_delay()
722 			ASSERT_EXCLUSIVE_WRITER(sup->srcu_n_exp_nodelay);  in srcu_get_delay()
723 			WRITE_ONCE(sup->srcu_n_exp_nodelay, READ_ONCE(sup->srcu_n_exp_nodelay) + 1);  in srcu_get_delay()
724 			if (READ_ONCE(sup->srcu_n_exp_nodelay) > srcu_max_nodelay_phase)  in srcu_get_delay()
742 	struct srcu_usage *sup = ssp->srcu_sup;  in cleanup_srcu_struct()  local
751 	flush_delayed_work(&sup->work);  in cleanup_srcu_struct()
760 	if (WARN_ON(rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)) != SRCU_STATE_IDLE) ||  in cleanup_srcu_struct()
761 	    WARN_ON(rcu_seq_current(&sup->srcu_gp_seq) != sup->srcu_gp_seq_needed) ||  in cleanup_srcu_struct()
764 			__func__, ssp, rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)),  in cleanup_srcu_struct()
765 			rcu_seq_current(&sup->srcu_gp_seq), sup->srcu_gp_seq_needed);  in cleanup_srcu_struct()
771 	kfree(sup->node);  in cleanup_srcu_struct()
772 	sup->node = NULL;  in cleanup_srcu_struct()
773 	sup->srcu_size_state = SRCU_SIZE_SMALL;  in cleanup_srcu_struct()
774 	if (!sup->sda_is_static) {  in cleanup_srcu_struct()
777 		kfree(sup);  in cleanup_srcu_struct()
955 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_end()  local
958 	mutex_lock(&sup->srcu_cb_mutex);  in srcu_gp_end()
961 	spin_lock_irq_rcu_node(sup);  in srcu_gp_end()
962 	idx = rcu_seq_state(sup->srcu_gp_seq);  in srcu_gp_end()
967 	WRITE_ONCE(sup->srcu_last_gp_end, ktime_get_mono_fast_ns());  in srcu_gp_end()
968 	rcu_seq_end(&sup->srcu_gp_seq);  in srcu_gp_end()
969 	gpseq = rcu_seq_current(&sup->srcu_gp_seq);  in srcu_gp_end()
970 	if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq))  in srcu_gp_end()
971 		WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq);  in srcu_gp_end()
972 	spin_unlock_irq_rcu_node(sup);  in srcu_gp_end()
973 	mutex_unlock(&sup->srcu_gp_mutex);  in srcu_gp_end()
977 	ss_state = smp_load_acquire(&sup->srcu_size_state);  in srcu_gp_end()
986 			last_lvl = snp >= sup->level[rcu_num_lvls - 1];  in srcu_gp_end()
1018 	mutex_unlock(&sup->srcu_cb_mutex);  in srcu_gp_end()
1021 	spin_lock_irq_rcu_node(sup);  in srcu_gp_end()
1022 	gpseq = rcu_seq_current(&sup->srcu_gp_seq);  in srcu_gp_end()
1024 	    ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) {  in srcu_gp_end()
1026 		spin_unlock_irq_rcu_node(sup);  in srcu_gp_end()
1029 		spin_unlock_irq_rcu_node(sup);  in srcu_gp_end()
1037 			smp_store_release(&sup->srcu_size_state, ss_state + 1);  in srcu_gp_end()
1097 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_funnel_gp_start()  local
1100 	if (smp_load_acquire(&sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_funnel_gp_start()
1108 			if (WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && snp != snp_leaf)  in srcu_funnel_gp_start()
1135 	if (ULONG_CMP_LT(sup->srcu_gp_seq_needed, s)) {  in srcu_funnel_gp_start()
1140 		smp_store_release(&sup->srcu_gp_seq_needed, s); /*^^^*/  in srcu_funnel_gp_start()
1142 	if (!do_norm && ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, s))  in srcu_funnel_gp_start()
1143 		WRITE_ONCE(sup->srcu_gp_seq_needed_exp, s);  in srcu_funnel_gp_start()
1146 	if (!WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) &&  in srcu_funnel_gp_start()
1147 	    rcu_seq_state(sup->srcu_gp_seq) == SRCU_STATE_IDLE) {  in srcu_funnel_gp_start()
1156 			queue_delayed_work(rcu_gp_wq, &sup->work,  in srcu_funnel_gp_start()
1158 		else if (list_empty(&sup->work.work.entry))  in srcu_funnel_gp_start()
1159 			list_add(&sup->work.work.entry, &srcu_boot_list);  in srcu_funnel_gp_start()
1161 	spin_unlock_irqrestore_rcu_node(sup, flags);  in srcu_funnel_gp_start()
1990 	struct srcu_usage *sup;  in process_srcu()  local
1992 	sup = container_of(work, struct srcu_usage, work.work);  in process_srcu()
1993 	ssp = sup->srcu_ssp;  in process_srcu()
2000 		WRITE_ONCE(sup->reschedule_count, 0);  in process_srcu()
2003 		if (READ_ONCE(sup->reschedule_jiffies) == j) {  in process_srcu()
2004 			ASSERT_EXCLUSIVE_WRITER(sup->reschedule_count);  in process_srcu()
2005 			WRITE_ONCE(sup->reschedule_count, READ_ONCE(sup->reschedule_count) + 1);  in process_srcu()
2006 			if (READ_ONCE(sup->reschedule_count) > srcu_max_nodelay)  in process_srcu()
2009 			WRITE_ONCE(sup->reschedule_count, 1);  in process_srcu()
2010 			WRITE_ONCE(sup->reschedule_jiffies, j);  in process_srcu()
2106 	struct srcu_usage *sup;  in srcu_init()  local
2126 		sup = list_first_entry(&srcu_boot_list, struct srcu_usage,  in srcu_init()
2128 		list_del_init(&sup->work.work.entry);  in srcu_init()
2130 		    sup->srcu_size_state == SRCU_SIZE_SMALL)  in srcu_init()
2131 			sup->srcu_size_state = SRCU_SIZE_ALLOC;  in srcu_init()
2132 		queue_work(rcu_gp_wq, &sup->work.work);  in srcu_init()