126 	struct srcu_data *sdp;  in init_srcu_struct_data()  local
133 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_data()
134 		spin_lock_init(&ACCESS_PRIVATE(sdp, lock));  in init_srcu_struct_data()
135 		rcu_segcblist_init(&sdp->srcu_cblist);  in init_srcu_struct_data()
136 		sdp->srcu_cblist_invoking = false;  in init_srcu_struct_data()
137 		sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
138 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
139 		sdp->srcu_barrier_head.next = &sdp->srcu_barrier_head;  in init_srcu_struct_data()
140 		sdp->mynode = NULL;  in init_srcu_struct_data()
141 		sdp->cpu = cpu;  in init_srcu_struct_data()
142 		INIT_WORK(&sdp->work, srcu_invoke_callbacks);  in init_srcu_struct_data()
143 		timer_setup(&sdp->delay_work, srcu_delay_timer, 0);  in init_srcu_struct_data()
144 		sdp->ssp = ssp;  in init_srcu_struct_data()
170 	struct srcu_data *sdp;  in init_srcu_struct_nodes()  local
219 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_nodes()
220 		sdp->mynode = &snp_first[cpu / levelspread[level]];  in init_srcu_struct_nodes()
221 		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {  in init_srcu_struct_nodes()
226 		sdp->grpmask = 1UL << (cpu - sdp->mynode->grplo);  in init_srcu_struct_nodes()
431 static void spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags)  in spin_lock_irqsave_sdp_contention()  argument
433 	struct srcu_struct *ssp = sdp->ssp;  in spin_lock_irqsave_sdp_contention()
435 	if (spin_trylock_irqsave_rcu_node(sdp, *flags))  in spin_lock_irqsave_sdp_contention()
440 	spin_lock_irqsave_rcu_node(sdp, *flags);  in spin_lock_irqsave_sdp_contention()
504 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_lock_idx()  local
506 		sum += atomic_long_read(&sdp->srcu_ctrs[idx].srcu_locks);  in srcu_readers_lock_idx()
508 			mask = mask | READ_ONCE(sdp->srcu_reader_flavor);  in srcu_readers_lock_idx()
528 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_unlock_idx()  local
530 		sum += atomic_long_read(&sdp->srcu_ctrs[idx].srcu_unlocks);  in srcu_readers_unlock_idx()
531 		mask = mask | READ_ONCE(sdp->srcu_reader_flavor);  in srcu_readers_unlock_idx()
645 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_active()  local
647 		sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks);  in srcu_readers_active()
648 		sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks);  in srcu_readers_active()
649 		sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks);  in srcu_readers_active()
650 		sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks);  in srcu_readers_active()
753 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in cleanup_srcu_struct()  local
755 		timer_delete_sync(&sdp->delay_work);  in cleanup_srcu_struct()
756 		flush_work(&sdp->work);  in cleanup_srcu_struct()
757 		if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist)))  in cleanup_srcu_struct()
789 	struct srcu_data *sdp;  in __srcu_check_read_flavor()  local
795 	sdp = raw_cpu_ptr(ssp->sda);  in __srcu_check_read_flavor()
796 	old_read_flavor = READ_ONCE(sdp->srcu_reader_flavor);  in __srcu_check_read_flavor()
802 		old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor);  in __srcu_check_read_flavor()
806 …WARN_ONCE(old_read_flavor != read_flavor, "CPU %d old state %d new state %d\n", sdp->cpu, old_read…  in __srcu_check_read_flavor()
890 	struct srcu_data *sdp = container_of(t, struct srcu_data, delay_work);  in srcu_delay_timer()  local
892 	queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);  in srcu_delay_timer()
895 static void srcu_queue_delayed_work_on(struct srcu_data *sdp,  in srcu_queue_delayed_work_on()  argument
899 		queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);  in srcu_queue_delayed_work_on()
903 	timer_reduce(&sdp->delay_work, jiffies + delay);  in srcu_queue_delayed_work_on()
910 static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay)  in srcu_schedule_cbs_sdp()  argument
912 	srcu_queue_delayed_work_on(sdp, delay);  in srcu_schedule_cbs_sdp()
951 	struct srcu_data *sdp;  in srcu_gp_end()  local
1008 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_gp_end()
1009 			spin_lock_irq_rcu_node(sdp);  in srcu_gp_end()
1010 			if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed + 100))  in srcu_gp_end()
1011 				sdp->srcu_gp_seq_needed = gpseq;  in srcu_gp_end()
1012 			if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed_exp + 100))  in srcu_gp_end()
1013 				sdp->srcu_gp_seq_needed_exp = gpseq;  in srcu_gp_end()
1014 			spin_unlock_irq_rcu_node(sdp);  in srcu_gp_end()
1088 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
1092 	int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);  in srcu_funnel_gp_start()
1103 		snp_leaf = sdp->mynode;  in srcu_funnel_gp_start()
1114 					snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
1117 					srcu_schedule_cbs_sdp(sdp, do_norm ? SRCU_INTERVAL : 0);  in srcu_funnel_gp_start()
1126 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
1276 	struct srcu_data *sdp;  in srcu_should_expedite()  local
1285 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_should_expedite()
1286 	spin_lock_irqsave_rcu_node(sdp, flags);  in srcu_should_expedite()
1287 	if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) {  in srcu_should_expedite()
1288 		spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_should_expedite()
1291 	spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_should_expedite()
1335 	struct srcu_data *sdp;  in srcu_gp_start_if_needed()  local
1348 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());  in srcu_gp_start_if_needed()
1350 		sdp = raw_cpu_ptr(ssp->sda);  in srcu_gp_start_if_needed()
1351 	spin_lock_irqsave_sdp_contention(sdp, &flags);  in srcu_gp_start_if_needed()
1353 		rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);  in srcu_gp_start_if_needed()
1392 		rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_gp_start_if_needed()
1401 		WARN_ON_ONCE(!rcu_segcblist_accelerate(&sdp->srcu_cblist, s));  in srcu_gp_start_if_needed()
1403 	if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {  in srcu_gp_start_if_needed()
1404 		sdp->srcu_gp_seq_needed = s;  in srcu_gp_start_if_needed()
1407 	if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) {  in srcu_gp_start_if_needed()
1408 		sdp->srcu_gp_seq_needed_exp = s;  in srcu_gp_start_if_needed()
1411 	spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_gp_start_if_needed()
1417 		sdp_mynode = sdp->mynode;  in srcu_gp_start_if_needed()
1420 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in srcu_gp_start_if_needed()
1682 	struct srcu_data *sdp;  in srcu_barrier_cb()  local
1686 	sdp = container_of(rhp, struct srcu_data, srcu_barrier_head);  in srcu_barrier_cb()
1687 	ssp = sdp->ssp;  in srcu_barrier_cb()
1700 static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp)  in srcu_barrier_one_cpu()  argument
1702 	spin_lock_irq_rcu_node(sdp);  in srcu_barrier_one_cpu()
1704 	sdp->srcu_barrier_head.func = srcu_barrier_cb;  in srcu_barrier_one_cpu()
1705 	debug_rcu_head_queue(&sdp->srcu_barrier_head);  in srcu_barrier_one_cpu()
1706 	if (!rcu_segcblist_entrain(&sdp->srcu_cblist,  in srcu_barrier_one_cpu()
1707 				   &sdp->srcu_barrier_head)) {  in srcu_barrier_one_cpu()
1708 		debug_rcu_head_unqueue(&sdp->srcu_barrier_head);  in srcu_barrier_one_cpu()
1711 	spin_unlock_irq_rcu_node(sdp);  in srcu_barrier_one_cpu()
1760 	struct srcu_data *sdp = container_of(rhp, struct srcu_data, srcu_ec_head);  in srcu_expedite_current_cb()  local
1762 	spin_lock_irqsave_sdp_contention(sdp, &flags);  in srcu_expedite_current_cb()
1763 	if (sdp->srcu_ec_state == SRCU_EC_IDLE) {  in srcu_expedite_current_cb()
1765 	} else if (sdp->srcu_ec_state == SRCU_EC_PENDING) {  in srcu_expedite_current_cb()
1766 		sdp->srcu_ec_state = SRCU_EC_IDLE;  in srcu_expedite_current_cb()
1768 		WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST);  in srcu_expedite_current_cb()
1769 		sdp->srcu_ec_state = SRCU_EC_PENDING;  in srcu_expedite_current_cb()
1772 	spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_expedite_current_cb()
1775 		__call_srcu(sdp->ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false);  in srcu_expedite_current_cb()
1792 	struct srcu_data *sdp;  in srcu_expedite_current()  local
1795 	sdp = this_cpu_ptr(ssp->sda);  in srcu_expedite_current()
1796 	spin_lock_irqsave_sdp_contention(sdp, &flags);  in srcu_expedite_current()
1797 	if (sdp->srcu_ec_state == SRCU_EC_IDLE) {  in srcu_expedite_current()
1798 		sdp->srcu_ec_state = SRCU_EC_PENDING;  in srcu_expedite_current()
1800 	} else if (sdp->srcu_ec_state == SRCU_EC_PENDING) {  in srcu_expedite_current()
1801 		sdp->srcu_ec_state = SRCU_EC_REPOST;  in srcu_expedite_current()
1803 		WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST);  in srcu_expedite_current()
1805 	spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_expedite_current()
1808 		__call_srcu(ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false);  in srcu_expedite_current()
1907 	struct srcu_data *sdp;  in srcu_invoke_callbacks()  local
1910 	sdp = container_of(work, struct srcu_data, work);  in srcu_invoke_callbacks()
1912 	ssp = sdp->ssp;  in srcu_invoke_callbacks()
1914 	spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1915 	WARN_ON_ONCE(!rcu_segcblist_segempty(&sdp->srcu_cblist, RCU_NEXT_TAIL));  in srcu_invoke_callbacks()
1916 	rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_invoke_callbacks()
1923 	if (sdp->srcu_cblist_invoking ||  in srcu_invoke_callbacks()
1924 	    !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) {  in srcu_invoke_callbacks()
1925 		spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1930 	sdp->srcu_cblist_invoking = true;  in srcu_invoke_callbacks()
1931 	rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs);  in srcu_invoke_callbacks()
1933 	spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1948 	spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1949 	rcu_segcblist_add_len(&sdp->srcu_cblist, -len);  in srcu_invoke_callbacks()
1950 	sdp->srcu_cblist_invoking = false;  in srcu_invoke_callbacks()
1951 	more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist);  in srcu_invoke_callbacks()
1952 	spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1955 		srcu_schedule_cbs_sdp(sdp, 0);  in srcu_invoke_callbacks()
2060 			struct srcu_data *sdp;  in srcu_torture_stats_print()  local
2062 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_torture_stats_print()
2063 			u0 = data_race(atomic_long_read(&sdp->srcu_ctrs[!idx].srcu_unlocks));  in srcu_torture_stats_print()
2064 			u1 = data_race(atomic_long_read(&sdp->srcu_ctrs[idx].srcu_unlocks));  in srcu_torture_stats_print()
2072 			l0 = data_race(atomic_long_read(&sdp->srcu_ctrs[!idx].srcu_locks));  in srcu_torture_stats_print()
2073 			l1 = data_race(atomic_long_read(&sdp->srcu_ctrs[idx].srcu_locks));  in srcu_torture_stats_print()
2079 				"C."[rcu_segcblist_empty(&sdp->srcu_cblist)]);  in srcu_torture_stats_print()