1 /* SPDX-License-Identifier: GPL-2.0+ */
2 /*
3 * Read-Copy Update mechanism for mutual exclusion, adapted for tracing.
4 *
5 * Copyright (C) 2020 Paul E. McKenney.
6 */
7
8 #ifndef __LINUX_RCUPDATE_TRACE_H
9 #define __LINUX_RCUPDATE_TRACE_H
10
11 #include <linux/sched.h>
12 #include <linux/rcupdate.h>
13 #include <linux/cleanup.h>
14
15 #ifdef CONFIG_TASKS_TRACE_RCU
16 extern struct srcu_struct rcu_tasks_trace_srcu_struct;
17 #endif // #ifdef CONFIG_TASKS_TRACE_RCU
18
19 #if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_TASKS_TRACE_RCU)
20
rcu_read_lock_trace_held(void)21 static inline int rcu_read_lock_trace_held(void)
22 {
23 return srcu_read_lock_held(&rcu_tasks_trace_srcu_struct);
24 }
25
26 #else // #if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_TASKS_TRACE_RCU)
27
rcu_read_lock_trace_held(void)28 static inline int rcu_read_lock_trace_held(void)
29 {
30 return 1;
31 }
32
33 #endif // #else // #if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_TASKS_TRACE_RCU)
34
35 #ifdef CONFIG_TASKS_TRACE_RCU
36
37 /**
38 * rcu_read_lock_tasks_trace - mark beginning of RCU-trace read-side critical section
39 *
40 * When synchronize_rcu_tasks_trace() is invoked by one task, then that
41 * task is guaranteed to block until all other tasks exit their read-side
42 * critical sections. Similarly, if call_rcu_trace() is invoked on one
43 * task while other tasks are within RCU read-side critical sections,
44 * invocation of the corresponding RCU callback is deferred until after
45 * the all the other tasks exit their critical sections.
46 *
47 * For more details, please see the documentation for
48 * srcu_read_lock_fast(). For a description of how implicit RCU
49 * readers provide the needed ordering for architectures defining the
50 * ARCH_WANTS_NO_INSTR Kconfig option (and thus promising never to trace
51 * code where RCU is not watching), please see the __srcu_read_lock_fast()
52 * (non-kerneldoc) header comment. Otherwise, the smp_mb() below provided
53 * the needed ordering.
54 */
rcu_read_lock_tasks_trace(void)55 static inline struct srcu_ctr __percpu *rcu_read_lock_tasks_trace(void)
56 {
57 struct srcu_ctr __percpu *ret = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct);
58
59 rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map);
60 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
61 smp_mb(); // Provide ordering on noinstr-incomplete architectures.
62 return ret;
63 }
64
65 /**
66 * rcu_read_unlock_tasks_trace - mark end of RCU-trace read-side critical section
67 * @scp: return value from corresponding rcu_read_lock_tasks_trace().
68 *
69 * Pairs with the preceding call to rcu_read_lock_tasks_trace() that
70 * returned the value passed in via scp.
71 *
72 * For more details, please see the documentation for rcu_read_unlock().
73 * For memory-ordering information, please see the header comment for the
74 * rcu_read_lock_tasks_trace() function.
75 */
rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu * scp)76 static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp)
77 {
78 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
79 smp_mb(); // Provide ordering on noinstr-incomplete architectures.
80 __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp);
81 srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map);
82 }
83
84 /**
85 * rcu_read_lock_trace - mark beginning of RCU-trace read-side critical section
86 *
87 * When synchronize_rcu_tasks_trace() is invoked by one task, then that
88 * task is guaranteed to block until all other tasks exit their read-side
89 * critical sections. Similarly, if call_rcu_trace() is invoked on one
90 * task while other tasks are within RCU read-side critical sections,
91 * invocation of the corresponding RCU callback is deferred until after
92 * the all the other tasks exit their critical sections.
93 *
94 * For more details, please see the documentation for rcu_read_lock().
95 */
rcu_read_lock_trace(void)96 static inline void rcu_read_lock_trace(void)
97 {
98 int n;
99 struct task_struct *t = current;
100
101 rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map);
102 n = READ_ONCE(t->trc_reader_nesting);
103 WRITE_ONCE(t->trc_reader_nesting, n + 1);
104 if (n) {
105 // In case we interrupted a Tasks Trace RCU reader.
106 return;
107 }
108 barrier(); // nesting before scp to protect against interrupt handler.
109 t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct);
110 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
111 smp_mb(); // Placeholder for more selective ordering
112 }
113
114 /**
115 * rcu_read_unlock_trace - mark end of RCU-trace read-side critical section
116 *
117 * Pairs with a preceding call to rcu_read_lock_trace(), and nesting is
118 * allowed. Invoking a rcu_read_unlock_trace() when there is no matching
119 * rcu_read_lock_trace() is verboten, and will result in lockdep complaints.
120 *
121 * For more details, please see the documentation for rcu_read_unlock().
122 */
rcu_read_unlock_trace(void)123 static inline void rcu_read_unlock_trace(void)
124 {
125 int n;
126 struct srcu_ctr __percpu *scp;
127 struct task_struct *t = current;
128
129 n = READ_ONCE(t->trc_reader_nesting) - 1;
130 if (n) {
131 WRITE_ONCE(t->trc_reader_nesting, n);
132 } else {
133 scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness.
134 barrier(); // scp before nesting to protect against interrupt handler.
135 WRITE_ONCE(t->trc_reader_nesting, n);
136 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
137 smp_mb(); // Placeholder for more selective ordering
138 __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp);
139 }
140 srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map);
141 }
142
143 /**
144 * call_rcu_tasks_trace() - Queue a callback trace task-based grace period
145 * @rhp: structure to be used for queueing the RCU updates.
146 * @func: actual callback function to be invoked after the grace period
147 *
148 * The callback function will be invoked some time after a trace rcu-tasks
149 * grace period elapses, in other words after all currently executing
150 * trace rcu-tasks read-side critical sections have completed. These
151 * read-side critical sections are delimited by calls to rcu_read_lock_trace()
152 * and rcu_read_unlock_trace().
153 *
154 * See the description of call_rcu() for more detailed information on
155 * memory ordering guarantees.
156 */
call_rcu_tasks_trace(struct rcu_head * rhp,rcu_callback_t func)157 static inline void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func)
158 {
159 call_srcu(&rcu_tasks_trace_srcu_struct, rhp, func);
160 }
161
162 /**
163 * synchronize_rcu_tasks_trace - wait for a trace rcu-tasks grace period
164 *
165 * Control will return to the caller some time after a trace rcu-tasks
166 * grace period has elapsed, in other words after all currently executing
167 * trace rcu-tasks read-side critical sections have elapsed. These read-side
168 * critical sections are delimited by calls to rcu_read_lock_trace()
169 * and rcu_read_unlock_trace().
170 *
171 * This is a very specialized primitive, intended only for a few uses in
172 * tracing and other situations requiring manipulation of function preambles
173 * and profiling hooks. The synchronize_rcu_tasks_trace() function is not
174 * (yet) intended for heavy use from multiple CPUs.
175 *
176 * See the description of synchronize_rcu() for more detailed information
177 * on memory ordering guarantees.
178 */
synchronize_rcu_tasks_trace(void)179 static inline void synchronize_rcu_tasks_trace(void)
180 {
181 synchronize_srcu(&rcu_tasks_trace_srcu_struct);
182 }
183
184 /**
185 * rcu_barrier_tasks_trace - Wait for in-flight call_rcu_tasks_trace() callbacks.
186 *
187 * Note that rcu_barrier_tasks_trace() is not obligated to actually wait,
188 * for example, if there are no pending callbacks.
189 */
rcu_barrier_tasks_trace(void)190 static inline void rcu_barrier_tasks_trace(void)
191 {
192 srcu_barrier(&rcu_tasks_trace_srcu_struct);
193 }
194
195 /**
196 * rcu_tasks_trace_expedite_current - Expedite the current Tasks Trace RCU grace period
197 *
198 * Cause the current Tasks Trace RCU grace period to become expedited.
199 * The grace period following the current one might also be expedited.
200 * If there is no current grace period, one might be created. If the
201 * current grace period is currently sleeping, that sleep will complete
202 * before expediting will take effect.
203 */
rcu_tasks_trace_expedite_current(void)204 static inline void rcu_tasks_trace_expedite_current(void)
205 {
206 srcu_expedite_current(&rcu_tasks_trace_srcu_struct);
207 }
208
209 unsigned long rcu_tasks_trace_batches_completed(void);
210
211 // Placeholders to enable stepwise transition.
212 void __init rcu_tasks_trace_suppress_unused(void);
213
214 #else
rcu_tasks_trace_batches_completed(void)215 static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; }
216 /*
217 * The BPF JIT forms these addresses even when it doesn't call these
218 * functions, so provide definitions that result in runtime errors.
219 */
call_rcu_tasks_trace(struct rcu_head * rhp,rcu_callback_t func)220 static inline void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func) { BUG(); }
rcu_read_lock_trace(void)221 static inline void rcu_read_lock_trace(void) { BUG(); }
rcu_read_unlock_trace(void)222 static inline void rcu_read_unlock_trace(void) { BUG(); }
223 #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
224
225 DEFINE_LOCK_GUARD_0(rcu_tasks_trace,
226 rcu_read_lock_trace(),
227 rcu_read_unlock_trace())
228
229 #endif /* __LINUX_RCUPDATE_TRACE_H */
230