xref: /linux/include/linux/srcu.h (revision d06917d0ba226206dbd606330d73a8eb1e3029c1)
1 /* SPDX-License-Identifier: GPL-2.0+ */
2 /*
3  * Sleepable Read-Copy Update mechanism for mutual exclusion
4  *
5  * Copyright (C) IBM Corporation, 2006
6  * Copyright (C) Fujitsu, 2012
7  *
8  * Author: Paul McKenney <paulmck@linux.ibm.com>
9  *	   Lai Jiangshan <laijs@cn.fujitsu.com>
10  *
11  * For detailed explanation of Read-Copy Update mechanism see -
12  *		Documentation/RCU/ *.txt
13  *
14  */
15 
16 #ifndef _LINUX_SRCU_H
17 #define _LINUX_SRCU_H
18 
19 #include <linux/mutex.h>
20 #include <linux/rcupdate.h>
21 #include <linux/rculist.h>
22 #include <linux/workqueue.h>
23 #include <linux/rcu_segcblist.h>
24 
25 context_lock_struct(srcu_struct, __reentrant_ctx_lock);
26 
27 #ifdef CONFIG_DEBUG_LOCK_ALLOC
28 
29 int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name,
30 				     struct lock_class_key *key);
31 static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
32 				     struct lock_class_key *key)
33 {
34 	return init_srcu_struct_lockdep(ssp, name, key);
35 }
36 #ifndef CONFIG_TINY_SRCU
37 int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
38 int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name,
39 				   struct lock_class_key *key);
40 int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name,
41 			      struct lock_class_key *key);
42 #endif // #ifndef CONFIG_TINY_SRCU
43 
44 #define init_srcu_struct_fast(ssp) \
45 ({ \
46 	static struct lock_class_key __srcu_key; \
47 	\
48 	__init_srcu_struct_fast((ssp), #ssp, &__srcu_key); \
49 })
50 
51 #define init_srcu_struct_fast_updown(ssp) \
52 ({ \
53 	static struct lock_class_key __srcu_key; \
54 	\
55 	__init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \
56 })
57 
58 #define init_srcu_struct_atomic(ssp) \
59 ({ \
60 	static struct lock_class_key __srcu_key; \
61 	\
62 	__init_srcu_struct_atomic((ssp), #ssp, &__srcu_key); \
63 })
64 
65 #define __SRCU_DEP_MAP_INIT(srcu_name)	.dep_map = { .name = #srcu_name },
66 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
67 
68 int init_srcu_struct_generic(struct srcu_struct *ssp);
69 static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
70 				     struct lock_class_key *key)
71 {
72 	return init_srcu_struct_generic(ssp);
73 }
74 #ifndef CONFIG_TINY_SRCU
75 int init_srcu_struct_fast(struct srcu_struct *ssp);
76 int init_srcu_struct_fast_updown(struct srcu_struct *ssp);
77 int init_srcu_struct_atomic(struct srcu_struct *ssp);
78 #endif // #ifndef CONFIG_TINY_SRCU
79 
80 #define __SRCU_DEP_MAP_INIT(srcu_name)
81 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
82 
83 #define init_srcu_struct(ssp) \
84 ({ \
85 	static struct lock_class_key __srcu_key; \
86 	\
87 	__init_srcu_struct((ssp), #ssp, &__srcu_key); \
88 })
89 
90 /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */
91 #define SRCU_READ_FLAVOR_NORMAL		0x01		// srcu_read_lock().
92 #define SRCU_READ_FLAVOR_NMI		0x02		// srcu_read_lock_nmisafe().
93 //					0x04		// SRCU-lite is no longer with us.
94 #define SRCU_READ_FLAVOR_FAST		0x04		// srcu_read_lock_fast(), also NMI-safe.
95 #define SRCU_READ_FLAVOR_FAST_UPDOWN	0x08		// srcu_read_lock_fast_updown().
96 #define SRCU_READ_FLAVOR_ATOMIC		0x10		// srcu_read_lock_atomic().
97 #define SRCU_READ_FLAVOR_ALL		(SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
98 					 SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN | \
99 					 SRCU_READ_FLAVOR_ATOMIC)
100 						// All of the above.
101 #define SRCU_READ_FLAVOR_PREDEF		(SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_ATOMIC)
102 						// Flavors special DEFINE_SRCU() flavors.
103 #define SRCU_READ_FLAVOR_SLOWGP		(SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)
104 						// Flavors requiring synchronize_rcu()
105 						// instead of smp_mb().
106 void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases_shared(ssp);
107 
108 #ifdef CONFIG_TINY_SRCU
109 #include <linux/srcutiny.h>
110 #elif defined(CONFIG_TREE_SRCU)
111 #include <linux/srcutree.h>
112 #else
113 #error "Unknown SRCU implementation specified to kernel configuration"
114 #endif
115 
116 void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
117 		void (*func)(struct rcu_head *head));
118 void cleanup_srcu_struct(struct srcu_struct *ssp);
119 void synchronize_srcu(struct srcu_struct *ssp);
120 void synchronize_srcu_atomic(struct srcu_struct *ssp);
121 
122 /**
123  * list_splice_init_srcu - splice an SRCU-protected list into an existing list
124  * @list:	the SRCU-protected list to splice
125  * @head:	the place in the existing list to splice the first list into
126  * @ssp:	the srcu_struct that readers of @list use
127  *
128  * Like list_splice_init_rcu(), but wait for the SRCU readers of @list
129  * instead of RCU ones, so that none of them can follow a next pointer
130  * onto @head.
131  *
132  * Note that this function blocks, and that the caller must prevent any
133  * other updates to @list.
134  */
135 static inline void list_splice_init_srcu(struct list_head *list,
136 					 struct list_head *head,
137 					 struct srcu_struct *ssp)
138 {
139 	struct list_head *first = list->next;
140 	struct list_head *last = list->prev;
141 	struct list_head *next = head->next;
142 
143 	if (list_empty(list))
144 		return;
145 
146 	INIT_LIST_HEAD_RCU(list);
147 	synchronize_srcu(ssp);
148 	ASSERT_EXCLUSIVE_ACCESS(*first);
149 	ASSERT_EXCLUSIVE_ACCESS(*last);
150 
151 	last->next = next;
152 	rcu_assign_pointer(list_next_rcu(head), first);
153 	first->prev = head;
154 	next->prev = last;
155 }
156 
157 #define SRCU_GET_STATE_COMPLETED 0x1
158 
159 /**
160  * get_completed_synchronize_srcu - Return a pre-completed polled state cookie
161  *
162  * Returns a value that poll_state_synchronize_srcu() will always treat
163  * as a cookie whose grace period has already completed.
164  */
165 static inline unsigned long get_completed_synchronize_srcu(void)
166 {
167 	return SRCU_GET_STATE_COMPLETED;
168 }
169 
170 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
171 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
172 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);
173 
174 // Maximum number of unsigned long values corresponding to
175 // not-yet-completed SRCU grace periods.
176 #define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2
177 
178 /**
179  * same_state_synchronize_srcu - Are two old-state values identical?
180  * @oldstate1: First old-state value.
181  * @oldstate2: Second old-state value.
182  *
183  * The two old-state values must have been obtained from either
184  * get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or
185  * get_completed_synchronize_srcu().  Returns @true if the two values are
186  * identical and @false otherwise.  This allows structures whose lifetimes
187  * are tracked by old-state values to push these values to a list header,
188  * allowing those structures to be slightly smaller.
189  */
190 static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2)
191 {
192 	return oldstate1 == oldstate2;
193 }
194 
195 #ifdef CONFIG_NEED_SRCU_NMI_SAFE
196 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires_shared(ssp);
197 void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases_shared(ssp);
198 #else
199 static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)
200 	__acquires_shared(ssp)
201 {
202 	return __srcu_read_lock(ssp);
203 }
204 static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
205 	__releases_shared(ssp)
206 {
207 	__srcu_read_unlock(ssp, idx);
208 }
209 #endif /* CONFIG_NEED_SRCU_NMI_SAFE */
210 
211 void srcu_init(void);
212 
213 #ifdef CONFIG_DEBUG_LOCK_ALLOC
214 
215 /**
216  * srcu_read_lock_held - might we be in SRCU read-side critical section?
217  * @ssp: The srcu_struct structure to check
218  *
219  * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU
220  * read-side critical section.  In absence of CONFIG_DEBUG_LOCK_ALLOC,
221  * this assumes we are in an SRCU read-side critical section unless it can
222  * prove otherwise.
223  *
224  * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
225  * and while lockdep is disabled.
226  *
227  * Note that SRCU is based on its own statemachine and it doesn't
228  * relies on normal RCU, it can be called from the CPU which
229  * is in the idle loop from an RCU point of view or offline.
230  */
231 static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
232 {
233 	if (!debug_lockdep_rcu_enabled())
234 		return 1;
235 	return lock_is_held(&ssp->dep_map);
236 }
237 
238 /*
239  * Annotations provide deadlock detection for SRCU.
240  *
241  * Similar to other lockdep annotations, except there is an additional
242  * srcu_lock_sync(), which is basically an empty *write*-side critical section,
243  * see lock_sync() for more information.
244  */
245 
246 /* Annotates a srcu_read_lock() */
247 static inline void srcu_lock_acquire(struct lockdep_map *map)
248 {
249 	lock_map_acquire_read(map);
250 }
251 
252 /* Annotates a srcu_read_lock() */
253 static inline void srcu_lock_release(struct lockdep_map *map)
254 {
255 	lock_map_release(map);
256 }
257 
258 /* Annotates a synchronize_srcu() */
259 static inline void srcu_lock_sync(struct lockdep_map *map)
260 {
261 	lock_map_sync(map);
262 }
263 
264 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
265 
266 static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
267 {
268 	return 1;
269 }
270 
271 #define srcu_lock_acquire(m) do { } while (0)
272 #define srcu_lock_release(m) do { } while (0)
273 #define srcu_lock_sync(m) do { } while (0)
274 
275 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
276 
277 /*
278  * No-op helper to denote that ssp must be held. Because SRCU-protected pointers
279  * should still be marked with __rcu_guarded, and we do not want to mark them
280  * with __guarded_by(ssp) as it would complicate annotations for writers, we
281  * choose the following strategy: srcu_dereference_check() calls this helper
282  * that checks that the passed ssp is held, and then fake-acquires 'RCU'.
283  */
284 static inline void __srcu_read_lock_must_hold(const struct srcu_struct *ssp) __must_hold_shared(ssp) { }
285 
286 /**
287  * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
288  * @p: the pointer to fetch and protect for later dereferencing
289  * @ssp: pointer to the srcu_struct, which is used to check that we
290  *	really are in an SRCU read-side critical section.
291  * @c: condition to check for update-side use
292  *
293  * If PROVE_RCU is enabled, invoking this outside of an RCU read-side
294  * critical section will result in an RCU-lockdep splat, unless @c evaluates
295  * to 1.  The @c argument will normally be a logical expression containing
296  * lockdep_is_held() calls.
297  */
298 #define srcu_dereference_check(p, ssp, c)					\
299 ({										\
300 	__srcu_read_lock_must_hold(ssp);					\
301 	__acquire_shared_ctx_lock(RCU);					\
302 	__auto_type __v = __rcu_dereference_check((p), __UNIQUE_ID(rcu),	\
303 				(c) || srcu_read_lock_held(ssp), __rcu);	\
304 	__release_shared_ctx_lock(RCU);					\
305 	__v;									\
306 })
307 
308 /**
309  * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
310  * @p: the pointer to fetch and protect for later dereferencing
311  * @ssp: pointer to the srcu_struct, which is used to check that we
312  *	really are in an SRCU read-side critical section.
313  *
314  * Makes rcu_dereference_check() do the dirty work.  If PROVE_RCU
315  * is enabled, invoking this outside of an RCU read-side critical
316  * section will result in an RCU-lockdep splat.
317  */
318 #define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0)
319 
320 /**
321  * srcu_dereference_notrace - no tracing and no lockdep calls from here
322  * @p: the pointer to fetch and protect for later dereferencing
323  * @ssp: pointer to the srcu_struct, which is used to check that we
324  *	really are in an SRCU read-side critical section.
325  */
326 #define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1)
327 
328 /**
329  * srcu_read_lock - register a new reader for an SRCU-protected structure.
330  * @ssp: srcu_struct in which to register the new reader.
331  *
332  * Enter an SRCU read-side critical section.  Note that SRCU read-side
333  * critical sections may be nested.  However, it is illegal to
334  * call anything that waits on an SRCU grace period for the same
335  * srcu_struct, whether directly or indirectly.  Please note that
336  * one way to indirectly wait on an SRCU grace period is to acquire
337  * a mutex that is held elsewhere while calling synchronize_srcu() or
338  * synchronize_srcu_expedited().
339  *
340  * The return value from srcu_read_lock() is guaranteed to be
341  * non-negative.  This value must be passed unaltered to the matching
342  * srcu_read_unlock().  Note that srcu_read_lock() and the matching
343  * srcu_read_unlock() must occur in the same context, for example, it is
344  * illegal to invoke srcu_read_unlock() in an irq handler if the matching
345  * srcu_read_lock() was invoked in process context.  Or, for that matter to
346  * invoke srcu_read_unlock() from one task and the matching srcu_read_lock()
347  * from another.
348  */
349 static inline int srcu_read_lock(struct srcu_struct *ssp)
350 	__acquires_shared(ssp)
351 {
352 	int retval;
353 
354 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
355 	retval = __srcu_read_lock(ssp);
356 	srcu_lock_acquire(&ssp->dep_map);
357 	return retval;
358 }
359 
360 /**
361  * srcu_read_lock_atomic - register a new reader promising an atomic section
362  * @ssp: srcu_struct in which to register the new reader.
363  *
364  * As srcu_read_lock(), but the caller promises that the read-side
365  * critical section never sleeps and never blocks on anything which
366  * may itself depend on memory allocation to make progress. Preemption
367  * is disabled for the duration, which both enforces that promise (any
368  * sleepable call in the section will splat on every configuration)
369  * and bounds the section so that the update side may spin rather
370  * than sleep when waiting for readers: see synchronize_srcu_atomic().
371  *
372  * The lock and matching srcu_read_unlock_atomic() must be invoked on
373  * the same CPU, from the same context; passing the return value to
374  * another task is not permitted for this flavor.
375  */
376 static inline int srcu_read_lock_atomic(struct srcu_struct *ssp)
377 	__acquires_shared(ssp)
378 {
379 	int retval;
380 
381 	preempt_disable();
382 	/*
383 	 * Arm might_sleep() to catch even a *potentially* sleeping call
384 	 * in the section, not just an actual schedule: the atomic-domain
385 	 * promise must hold on every path, contended or not. In hardirq,
386 	 * softirq, or NMI the annotation would land on the interrupted
387 	 * task, and can also result in data races against that task's
388 	 * own non_block_start()/non_block_end() invocations; it is also
389 	 * redundant there, so skip it.
390 	 */
391 	if (in_task())
392 		non_block_start();
393 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC);
394 	retval = __srcu_read_lock(ssp);
395 	srcu_lock_acquire(&ssp->dep_map);
396 	return retval;
397 }
398 
399 /**
400  * srcu_read_lock_fast - register a new reader for an SRCU-protected structure.
401  * @ssp: srcu_struct in which to register the new reader.
402  *
403  * Enter an SRCU read-side critical section, but for a light-weight
404  * smp_mb()-free reader.  See srcu_read_lock() for more information.  This
405  * function is NMI-safe, in a manner similar to srcu_read_lock_nmisafe().
406  *
407  * For srcu_read_lock_fast() to be used on an srcu_struct structure,
408  * that structure must have been defined using either DEFINE_SRCU_FAST()
409  * or DEFINE_STATIC_SRCU_FAST() on the one hand or initialized with
410  * init_srcu_struct_fast() on the other.  Such an srcu_struct structure
411  * cannot be passed to any non-fast variant of srcu_read_{,un}lock() or
412  * srcu_{down,up}_read().  In kernels built with CONFIG_PROVE_RCU=y,
413  * __srcu_check_read_flavor() will complain bitterly if you ignore this
414  * restriction.
415  *
416  * Grace-period auto-expediting is disabled for SRCU-fast srcu_struct
417  * structures because SRCU-fast expedited grace periods invoke
418  * synchronize_rcu_expedited(), IPIs and all.  If you need expedited
419  * SRCU-fast grace periods, use synchronize_srcu_expedited().
420  *
421  * The srcu_read_lock_fast() function can be invoked only from those
422  * contexts where RCU is watching, that is, from contexts where it would
423  * be legal to invoke rcu_read_lock().  Otherwise, lockdep will complain.
424  */
425 static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires_shared(ssp)
426 	__acquires_shared(ssp)
427 {
428 	struct srcu_ctr __percpu *retval;
429 
430 	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast().");
431 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
432 	retval = __srcu_read_lock_fast(ssp);
433 	rcu_try_lock_acquire(&ssp->dep_map);
434 	return retval;
435 }
436 
437 /**
438  * srcu_read_lock_fast_updown - register a new reader for an SRCU-fast-updown structure.
439  * @ssp: srcu_struct in which to register the new reader.
440  *
441  * Enter an SRCU read-side critical section, but for a light-weight
442  * smp_mb()-free reader.  See srcu_read_lock() for more information.
443  * This function is compatible with srcu_down_read_fast(), but is not
444  * NMI-safe.
445  *
446  * For srcu_read_lock_fast_updown() to be used on an srcu_struct
447  * structure, that structure must have been defined using either
448  * DEFINE_SRCU_FAST_UPDOWN() or DEFINE_STATIC_SRCU_FAST_UPDOWN() on the one
449  * hand or initialized with init_srcu_struct_fast_updown() on the other.
450  * Such an srcu_struct structure cannot be passed to any non-fast-updown
451  * variant of srcu_read_{,un}lock() or srcu_{down,up}_read().  In kernels
452  * built with CONFIG_PROVE_RCU=y, __srcu_check_read_flavor() will complain
453  * bitterly if you ignore this * restriction.
454  *
455  * Grace-period auto-expediting is disabled for SRCU-fast-updown
456  * srcu_struct structures because SRCU-fast-updown expedited grace periods
457  * invoke synchronize_rcu_expedited(), IPIs and all.  If you need expedited
458  * SRCU-fast-updown grace periods, use synchronize_srcu_expedited().
459  *
460  * The srcu_read_lock_fast_updown() function can be invoked only from
461  * those contexts where RCU is watching, that is, from contexts where
462  * it would be legal to invoke rcu_read_lock().  Otherwise, lockdep will
463  * complain.
464  */
465 static inline struct srcu_ctr __percpu *srcu_read_lock_fast_updown(struct srcu_struct *ssp)
466 	__acquires_shared(ssp)
467 {
468 	struct srcu_ctr __percpu *retval;
469 
470 	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast_updown().");
471 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
472 	retval = __srcu_read_lock_fast_updown(ssp);
473 	rcu_try_lock_acquire(&ssp->dep_map);
474 	return retval;
475 }
476 
477 /*
478  * Used by tracing, cannot be traced and cannot call lockdep.
479  * See srcu_read_lock_fast() for more information.
480  */
481 static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp)
482 	__acquires_shared(ssp)
483 {
484 	struct srcu_ctr __percpu *retval;
485 
486 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
487 	retval = __srcu_read_lock_fast(ssp);
488 	return retval;
489 }
490 
491 /**
492  * srcu_down_read_fast - register a new reader for an SRCU-protected structure.
493  * @ssp: srcu_struct in which to register the new reader.
494  *
495  * Enter a semaphore-like SRCU read-side critical section, but for
496  * a light-weight smp_mb()-free reader.  See srcu_read_lock_fast() and
497  * srcu_down_read() for more information.
498  *
499  * The same srcu_struct may be used concurrently by srcu_down_read_fast()
500  * and srcu_read_lock_fast().  However, the same definition/initialization
501  * requirements called out for srcu_read_lock_fast_updown() apply.
502  */
503 static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp)
504 {
505 	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
506 	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast().");
507 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
508 	return __srcu_read_lock_fast_updown(ssp);
509 }
510 
511 /**
512  * srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure.
513  * @ssp: srcu_struct in which to register the new reader.
514  *
515  * Enter an SRCU read-side critical section, but in an NMI-safe manner.
516  * See srcu_read_lock() for more information.
517  *
518  * If srcu_read_lock_nmisafe() is ever used on an srcu_struct structure,
519  * then none of the other flavors may be used, whether before, during,
520  * or after.
521  */
522 static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp)
523 	__acquires_shared(ssp)
524 {
525 	int retval;
526 
527 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
528 	retval = __srcu_read_lock_nmisafe(ssp);
529 	rcu_try_lock_acquire(&ssp->dep_map);
530 	return retval;
531 }
532 
533 /* Used by tracing, cannot be traced and cannot invoke lockdep. */
534 static inline notrace int
535 srcu_read_lock_notrace(struct srcu_struct *ssp)
536 	__acquires_shared(ssp)
537 {
538 	int retval;
539 
540 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
541 	retval = __srcu_read_lock(ssp);
542 	return retval;
543 }
544 
545 /**
546  * srcu_down_read - register a new reader for an SRCU-protected structure.
547  * @ssp: srcu_struct in which to register the new reader.
548  *
549  * Enter a semaphore-like SRCU read-side critical section.  Note that
550  * SRCU read-side critical sections may be nested.  However, it is
551  * illegal to call anything that waits on an SRCU grace period for the
552  * same srcu_struct, whether directly or indirectly.  Please note that
553  * one way to indirectly wait on an SRCU grace period is to acquire
554  * a mutex that is held elsewhere while calling synchronize_srcu() or
555  * synchronize_srcu_expedited().  But if you want lockdep to help you
556  * keep this stuff straight, you should instead use srcu_read_lock().
557  *
558  * The semaphore-like nature of srcu_down_read() means that the matching
559  * srcu_up_read() can be invoked from some other context, for example,
560  * from some other task or from an irq handler.  However, neither
561  * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler.
562  *
563  * Calls to srcu_down_read() may be nested, similar to the manner in
564  * which calls to down_read() may be nested.  The same srcu_struct may be
565  * used concurrently by srcu_down_read() and srcu_read_lock().
566  */
567 static inline int srcu_down_read(struct srcu_struct *ssp)
568 	__acquires_shared(ssp)
569 {
570 	WARN_ON_ONCE(in_nmi());
571 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
572 	return __srcu_read_lock(ssp);
573 }
574 
575 /**
576  * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
577  * @ssp: srcu_struct in which to unregister the old reader.
578  * @idx: return value from corresponding srcu_read_lock().
579  *
580  * Exit an SRCU read-side critical section.
581  */
582 static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
583 	__releases_shared(ssp)
584 {
585 	WARN_ON_ONCE(idx & ~0x1);
586 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
587 	srcu_lock_release(&ssp->dep_map);
588 	__srcu_read_unlock(ssp, idx);
589 }
590 
591 /**
592  * srcu_read_unlock_atomic - unregister an atomic-section reader
593  * @ssp: srcu_struct from which to unregister the old reader.
594  * @idx: return value from corresponding srcu_read_lock_atomic().
595  */
596 static inline void srcu_read_unlock_atomic(struct srcu_struct *ssp, int idx)
597 	__releases_shared(ssp)
598 {
599 	WARN_ON_ONCE(idx & ~0x1);
600 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC);
601 	srcu_lock_release(&ssp->dep_map);
602 	__srcu_read_unlock(ssp, idx);
603 	if (in_task())
604 		non_block_end();
605 	preempt_enable();
606 }
607 
608 /**
609  * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure.
610  * @ssp: srcu_struct in which to unregister the old reader.
611  * @scp: return value from corresponding srcu_read_lock_fast().
612  *
613  * Exit a light-weight SRCU read-side critical section.
614  */
615 static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
616 	__releases_shared(ssp)
617 {
618 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
619 	srcu_lock_release(&ssp->dep_map);
620 	__srcu_read_unlock_fast(ssp, scp);
621 	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast().");
622 }
623 
624 /**
625  * srcu_read_unlock_fast_updown - unregister a old reader from an SRCU-fast-updown structure.
626  * @ssp: srcu_struct in which to unregister the old reader.
627  * @scp: return value from corresponding srcu_read_lock_fast_updown().
628  *
629  * Exit an SRCU-fast-updown read-side critical section.
630  */
631 static inline void
632 srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases_shared(ssp)
633 {
634 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
635 	srcu_lock_release(&ssp->dep_map);
636 	__srcu_read_unlock_fast_updown(ssp, scp);
637 	RCU_LOCKDEP_WARN(!rcu_is_watching(),
638 			 "RCU must be watching srcu_read_unlock_fast_updown().");
639 }
640 
641 /*
642  * Used by tracing, cannot be traced and cannot call lockdep.
643  * See srcu_read_unlock_fast() for more information.
644  */
645 static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp,
646 						 struct srcu_ctr __percpu *scp) __releases_shared(ssp)
647 {
648 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
649 	__srcu_read_unlock_fast(ssp, scp);
650 }
651 
652 /**
653  * srcu_up_read_fast - unregister a old reader from an SRCU-protected structure.
654  * @ssp: srcu_struct in which to unregister the old reader.
655  * @scp: return value from corresponding srcu_read_lock_fast().
656  *
657  * Exit an SRCU read-side critical section, but not necessarily from
658  * the same context as the maching srcu_down_read_fast().
659  */
660 static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
661 	__releases_shared(ssp)
662 {
663 	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
664 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
665 	__srcu_read_unlock_fast_updown(ssp, scp);
666 	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_up_read_fast_updown().");
667 }
668 
669 /**
670  * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure.
671  * @ssp: srcu_struct in which to unregister the old reader.
672  * @idx: return value from corresponding srcu_read_lock_nmisafe().
673  *
674  * Exit an SRCU read-side critical section, but in an NMI-safe manner.
675  */
676 static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
677 	__releases_shared(ssp)
678 {
679 	WARN_ON_ONCE(idx & ~0x1);
680 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
681 	rcu_lock_release(&ssp->dep_map);
682 	__srcu_read_unlock_nmisafe(ssp, idx);
683 }
684 
685 /* Used by tracing, cannot be traced and cannot call lockdep. */
686 static inline notrace void
687 srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases_shared(ssp)
688 {
689 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
690 	__srcu_read_unlock(ssp, idx);
691 }
692 
693 /**
694  * srcu_up_read - unregister a old reader from an SRCU-protected structure.
695  * @ssp: srcu_struct in which to unregister the old reader.
696  * @idx: return value from corresponding srcu_read_lock().
697  *
698  * Exit an SRCU read-side critical section, but not necessarily from
699  * the same context as the maching srcu_down_read().
700  */
701 static inline void srcu_up_read(struct srcu_struct *ssp, int idx)
702 	__releases_shared(ssp)
703 {
704 	WARN_ON_ONCE(idx & ~0x1);
705 	WARN_ON_ONCE(in_nmi());
706 	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
707 	__srcu_read_unlock(ssp, idx);
708 }
709 
710 /**
711  * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock
712  *
713  * Converts the preceding srcu_read_unlock into a two-way memory barrier.
714  *
715  * Call this after srcu_read_unlock, to guarantee that all memory operations
716  * that occur after smp_mb__after_srcu_read_unlock will appear to happen after
717  * the preceding srcu_read_unlock.
718  */
719 static inline void smp_mb__after_srcu_read_unlock(void)
720 {
721 	/* __srcu_read_unlock has smp_mb() internally so nothing to do here. */
722 }
723 
724 /**
725  * smp_mb__after_srcu_read_lock - ensure full ordering after srcu_read_lock
726  *
727  * Converts the preceding srcu_read_lock into a two-way memory barrier.
728  *
729  * Call this after srcu_read_lock, to guarantee that all memory operations
730  * that occur after smp_mb__after_srcu_read_lock will appear to happen after
731  * the preceding srcu_read_lock.
732  */
733 static inline void smp_mb__after_srcu_read_lock(void)
734 {
735 	/* __srcu_read_lock has smp_mb() internally so nothing to do here. */
736 }
737 
738 DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct,
739 		    _T->idx = srcu_read_lock(_T->lock),
740 		    srcu_read_unlock(_T->lock, _T->idx),
741 		    int idx)
742 DECLARE_LOCK_GUARD_1_ATTRS(srcu, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T))
743 #define class_srcu_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu, _T)
744 
745 DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct,
746 		    _T->scp = srcu_read_lock_fast(_T->lock),
747 		    srcu_read_unlock_fast(_T->lock, _T->scp),
748 		    struct srcu_ctr __percpu *scp)
749 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T))
750 #define class_srcu_fast_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast, _T)
751 
752 DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct,
753 		    _T->scp = srcu_read_lock_fast_notrace(_T->lock),
754 		    srcu_read_unlock_fast_notrace(_T->lock, _T->scp),
755 		    struct srcu_ctr __percpu *scp)
756 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T))
757 #define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T)
758 
759 DEFINE_LOCK_GUARD_1(srcu_fast_updown, struct srcu_struct,
760 		    _T->scp = srcu_read_lock_fast_updown(_T->lock),
761 		    srcu_read_unlock_fast_updown(_T->lock, _T->scp),
762 		    struct srcu_ctr __percpu *scp)
763 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_updown, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T))
764 #define class_srcu_fast_updown_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_updown, _T)
765 
766 #endif
767