1 /* SPDX-License-Identifier: GPL-2.0+ */ 2 /* 3 * Sleepable Read-Copy Update mechanism for mutual exclusion 4 * 5 * Copyright (C) IBM Corporation, 2006 6 * Copyright (C) Fujitsu, 2012 7 * 8 * Author: Paul McKenney <paulmck@linux.ibm.com> 9 * Lai Jiangshan <laijs@cn.fujitsu.com> 10 * 11 * For detailed explanation of Read-Copy Update mechanism see - 12 * Documentation/RCU/ *.txt 13 * 14 */ 15 16 #ifndef _LINUX_SRCU_H 17 #define _LINUX_SRCU_H 18 19 #include <linux/mutex.h> 20 #include <linux/rcupdate.h> 21 #include <linux/workqueue.h> 22 #include <linux/rcu_segcblist.h> 23 24 context_lock_struct(srcu_struct, __reentrant_ctx_lock); 25 26 #ifdef CONFIG_DEBUG_LOCK_ALLOC 27 28 int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); 29 #ifndef CONFIG_TINY_SRCU 30 int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); 31 int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, 32 struct lock_class_key *key); 33 #endif // #ifndef CONFIG_TINY_SRCU 34 35 #define init_srcu_struct(ssp) \ 36 ({ \ 37 static struct lock_class_key __srcu_key; \ 38 \ 39 __init_srcu_struct((ssp), #ssp, &__srcu_key); \ 40 }) 41 42 #define init_srcu_struct_fast(ssp) \ 43 ({ \ 44 static struct lock_class_key __srcu_key; \ 45 \ 46 __init_srcu_struct_fast((ssp), #ssp, &__srcu_key); \ 47 }) 48 49 #define init_srcu_struct_fast_updown(ssp) \ 50 ({ \ 51 static struct lock_class_key __srcu_key; \ 52 \ 53 __init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \ 54 }) 55 56 #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, 57 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 58 59 int init_srcu_struct(struct srcu_struct *ssp); 60 #ifndef CONFIG_TINY_SRCU 61 int init_srcu_struct_fast(struct srcu_struct *ssp); 62 int init_srcu_struct_fast_updown(struct srcu_struct *ssp); 63 #endif // #ifndef CONFIG_TINY_SRCU 64 65 #define __SRCU_DEP_MAP_INIT(srcu_name) 66 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 67 68 /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ 69 #define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). 70 #define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). 71 // 0x4 // SRCU-lite is no longer with us. 72 #define SRCU_READ_FLAVOR_FAST 0x4 // srcu_read_lock_fast(). 73 #define SRCU_READ_FLAVOR_FAST_UPDOWN 0x8 // srcu_read_lock_fast(). 74 #define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ 75 SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) 76 // All of the above. 77 #define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) 78 // Flavors requiring synchronize_rcu() 79 // instead of smp_mb(). 80 void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases_shared(ssp); 81 82 #ifdef CONFIG_TINY_SRCU 83 #include <linux/srcutiny.h> 84 #elif defined(CONFIG_TREE_SRCU) 85 #include <linux/srcutree.h> 86 #else 87 #error "Unknown SRCU implementation specified to kernel configuration" 88 #endif 89 90 void call_srcu(struct srcu_struct *ssp, struct rcu_head *head, 91 void (*func)(struct rcu_head *head)); 92 void cleanup_srcu_struct(struct srcu_struct *ssp); 93 void synchronize_srcu(struct srcu_struct *ssp); 94 95 #define SRCU_GET_STATE_COMPLETED 0x1 96 97 /** 98 * get_completed_synchronize_srcu - Return a pre-completed polled state cookie 99 * 100 * Returns a value that poll_state_synchronize_srcu() will always treat 101 * as a cookie whose grace period has already completed. 102 */ 103 static inline unsigned long get_completed_synchronize_srcu(void) 104 { 105 return SRCU_GET_STATE_COMPLETED; 106 } 107 108 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp); 109 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp); 110 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie); 111 112 // Maximum number of unsigned long values corresponding to 113 // not-yet-completed SRCU grace periods. 114 #define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2 115 116 /** 117 * same_state_synchronize_srcu - Are two old-state values identical? 118 * @oldstate1: First old-state value. 119 * @oldstate2: Second old-state value. 120 * 121 * The two old-state values must have been obtained from either 122 * get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or 123 * get_completed_synchronize_srcu(). Returns @true if the two values are 124 * identical and @false otherwise. This allows structures whose lifetimes 125 * are tracked by old-state values to push these values to a list header, 126 * allowing those structures to be slightly smaller. 127 */ 128 static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2) 129 { 130 return oldstate1 == oldstate2; 131 } 132 133 #ifdef CONFIG_NEED_SRCU_NMI_SAFE 134 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires_shared(ssp); 135 void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases_shared(ssp); 136 #else 137 static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) 138 __acquires_shared(ssp) 139 { 140 return __srcu_read_lock(ssp); 141 } 142 static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) 143 __releases_shared(ssp) 144 { 145 __srcu_read_unlock(ssp, idx); 146 } 147 #endif /* CONFIG_NEED_SRCU_NMI_SAFE */ 148 149 void srcu_init(void); 150 151 #ifdef CONFIG_DEBUG_LOCK_ALLOC 152 153 /** 154 * srcu_read_lock_held - might we be in SRCU read-side critical section? 155 * @ssp: The srcu_struct structure to check 156 * 157 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU 158 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, 159 * this assumes we are in an SRCU read-side critical section unless it can 160 * prove otherwise. 161 * 162 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot 163 * and while lockdep is disabled. 164 * 165 * Note that SRCU is based on its own statemachine and it doesn't 166 * relies on normal RCU, it can be called from the CPU which 167 * is in the idle loop from an RCU point of view or offline. 168 */ 169 static inline int srcu_read_lock_held(const struct srcu_struct *ssp) 170 { 171 if (!debug_lockdep_rcu_enabled()) 172 return 1; 173 return lock_is_held(&ssp->dep_map); 174 } 175 176 /* 177 * Annotations provide deadlock detection for SRCU. 178 * 179 * Similar to other lockdep annotations, except there is an additional 180 * srcu_lock_sync(), which is basically an empty *write*-side critical section, 181 * see lock_sync() for more information. 182 */ 183 184 /* Annotates a srcu_read_lock() */ 185 static inline void srcu_lock_acquire(struct lockdep_map *map) 186 { 187 lock_map_acquire_read(map); 188 } 189 190 /* Annotates a srcu_read_lock() */ 191 static inline void srcu_lock_release(struct lockdep_map *map) 192 { 193 lock_map_release(map); 194 } 195 196 /* Annotates a synchronize_srcu() */ 197 static inline void srcu_lock_sync(struct lockdep_map *map) 198 { 199 lock_map_sync(map); 200 } 201 202 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 203 204 static inline int srcu_read_lock_held(const struct srcu_struct *ssp) 205 { 206 return 1; 207 } 208 209 #define srcu_lock_acquire(m) do { } while (0) 210 #define srcu_lock_release(m) do { } while (0) 211 #define srcu_lock_sync(m) do { } while (0) 212 213 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 214 215 /* 216 * No-op helper to denote that ssp must be held. Because SRCU-protected pointers 217 * should still be marked with __rcu_guarded, and we do not want to mark them 218 * with __guarded_by(ssp) as it would complicate annotations for writers, we 219 * choose the following strategy: srcu_dereference_check() calls this helper 220 * that checks that the passed ssp is held, and then fake-acquires 'RCU'. 221 */ 222 static inline void __srcu_read_lock_must_hold(const struct srcu_struct *ssp) __must_hold_shared(ssp) { } 223 224 /** 225 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing 226 * @p: the pointer to fetch and protect for later dereferencing 227 * @ssp: pointer to the srcu_struct, which is used to check that we 228 * really are in an SRCU read-side critical section. 229 * @c: condition to check for update-side use 230 * 231 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side 232 * critical section will result in an RCU-lockdep splat, unless @c evaluates 233 * to 1. The @c argument will normally be a logical expression containing 234 * lockdep_is_held() calls. 235 */ 236 #define srcu_dereference_check(p, ssp, c) \ 237 ({ \ 238 __srcu_read_lock_must_hold(ssp); \ 239 __acquire_shared_ctx_lock(RCU); \ 240 __auto_type __v = __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ 241 (c) || srcu_read_lock_held(ssp), __rcu); \ 242 __release_shared_ctx_lock(RCU); \ 243 __v; \ 244 }) 245 246 /** 247 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing 248 * @p: the pointer to fetch and protect for later dereferencing 249 * @ssp: pointer to the srcu_struct, which is used to check that we 250 * really are in an SRCU read-side critical section. 251 * 252 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU 253 * is enabled, invoking this outside of an RCU read-side critical 254 * section will result in an RCU-lockdep splat. 255 */ 256 #define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0) 257 258 /** 259 * srcu_dereference_notrace - no tracing and no lockdep calls from here 260 * @p: the pointer to fetch and protect for later dereferencing 261 * @ssp: pointer to the srcu_struct, which is used to check that we 262 * really are in an SRCU read-side critical section. 263 */ 264 #define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1) 265 266 /** 267 * srcu_read_lock - register a new reader for an SRCU-protected structure. 268 * @ssp: srcu_struct in which to register the new reader. 269 * 270 * Enter an SRCU read-side critical section. Note that SRCU read-side 271 * critical sections may be nested. However, it is illegal to 272 * call anything that waits on an SRCU grace period for the same 273 * srcu_struct, whether directly or indirectly. Please note that 274 * one way to indirectly wait on an SRCU grace period is to acquire 275 * a mutex that is held elsewhere while calling synchronize_srcu() or 276 * synchronize_srcu_expedited(). 277 * 278 * The return value from srcu_read_lock() is guaranteed to be 279 * non-negative. This value must be passed unaltered to the matching 280 * srcu_read_unlock(). Note that srcu_read_lock() and the matching 281 * srcu_read_unlock() must occur in the same context, for example, it is 282 * illegal to invoke srcu_read_unlock() in an irq handler if the matching 283 * srcu_read_lock() was invoked in process context. Or, for that matter to 284 * invoke srcu_read_unlock() from one task and the matching srcu_read_lock() 285 * from another. 286 */ 287 static inline int srcu_read_lock(struct srcu_struct *ssp) 288 __acquires_shared(ssp) 289 { 290 int retval; 291 292 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 293 retval = __srcu_read_lock(ssp); 294 srcu_lock_acquire(&ssp->dep_map); 295 return retval; 296 } 297 298 /** 299 * srcu_read_lock_fast - register a new reader for an SRCU-protected structure. 300 * @ssp: srcu_struct in which to register the new reader. 301 * 302 * Enter an SRCU read-side critical section, but for a light-weight 303 * smp_mb()-free reader. See srcu_read_lock() for more information. This 304 * function is NMI-safe, in a manner similar to srcu_read_lock_nmisafe(). 305 * 306 * For srcu_read_lock_fast() to be used on an srcu_struct structure, 307 * that structure must have been defined using either DEFINE_SRCU_FAST() 308 * or DEFINE_STATIC_SRCU_FAST() on the one hand or initialized with 309 * init_srcu_struct_fast() on the other. Such an srcu_struct structure 310 * cannot be passed to any non-fast variant of srcu_read_{,un}lock() or 311 * srcu_{down,up}_read(). In kernels built with CONFIG_PROVE_RCU=y, 312 * __srcu_check_read_flavor() will complain bitterly if you ignore this 313 * restriction. 314 * 315 * Grace-period auto-expediting is disabled for SRCU-fast srcu_struct 316 * structures because SRCU-fast expedited grace periods invoke 317 * synchronize_rcu_expedited(), IPIs and all. If you need expedited 318 * SRCU-fast grace periods, use synchronize_srcu_expedited(). 319 * 320 * The srcu_read_lock_fast() function can be invoked only from those 321 * contexts where RCU is watching, that is, from contexts where it would 322 * be legal to invoke rcu_read_lock(). Otherwise, lockdep will complain. 323 */ 324 static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires_shared(ssp) 325 __acquires_shared(ssp) 326 { 327 struct srcu_ctr __percpu *retval; 328 329 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast()."); 330 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 331 retval = __srcu_read_lock_fast(ssp); 332 rcu_try_lock_acquire(&ssp->dep_map); 333 return retval; 334 } 335 336 /** 337 * srcu_read_lock_fast_updown - register a new reader for an SRCU-fast-updown structure. 338 * @ssp: srcu_struct in which to register the new reader. 339 * 340 * Enter an SRCU read-side critical section, but for a light-weight 341 * smp_mb()-free reader. See srcu_read_lock() for more information. 342 * This function is compatible with srcu_down_read_fast(), but is not 343 * NMI-safe. 344 * 345 * For srcu_read_lock_fast_updown() to be used on an srcu_struct 346 * structure, that structure must have been defined using either 347 * DEFINE_SRCU_FAST_UPDOWN() or DEFINE_STATIC_SRCU_FAST_UPDOWN() on the one 348 * hand or initialized with init_srcu_struct_fast_updown() on the other. 349 * Such an srcu_struct structure cannot be passed to any non-fast-updown 350 * variant of srcu_read_{,un}lock() or srcu_{down,up}_read(). In kernels 351 * built with CONFIG_PROVE_RCU=y, __srcu_check_read_flavor() will complain 352 * bitterly if you ignore this * restriction. 353 * 354 * Grace-period auto-expediting is disabled for SRCU-fast-updown 355 * srcu_struct structures because SRCU-fast-updown expedited grace periods 356 * invoke synchronize_rcu_expedited(), IPIs and all. If you need expedited 357 * SRCU-fast-updown grace periods, use synchronize_srcu_expedited(). 358 * 359 * The srcu_read_lock_fast_updown() function can be invoked only from 360 * those contexts where RCU is watching, that is, from contexts where 361 * it would be legal to invoke rcu_read_lock(). Otherwise, lockdep will 362 * complain. 363 */ 364 static inline struct srcu_ctr __percpu *srcu_read_lock_fast_updown(struct srcu_struct *ssp) 365 __acquires_shared(ssp) 366 { 367 struct srcu_ctr __percpu *retval; 368 369 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast_updown()."); 370 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 371 retval = __srcu_read_lock_fast_updown(ssp); 372 rcu_try_lock_acquire(&ssp->dep_map); 373 return retval; 374 } 375 376 /* 377 * Used by tracing, cannot be traced and cannot call lockdep. 378 * See srcu_read_lock_fast() for more information. 379 */ 380 static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp) 381 __acquires_shared(ssp) 382 { 383 struct srcu_ctr __percpu *retval; 384 385 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 386 retval = __srcu_read_lock_fast(ssp); 387 return retval; 388 } 389 390 /** 391 * srcu_down_read_fast - register a new reader for an SRCU-protected structure. 392 * @ssp: srcu_struct in which to register the new reader. 393 * 394 * Enter a semaphore-like SRCU read-side critical section, but for 395 * a light-weight smp_mb()-free reader. See srcu_read_lock_fast() and 396 * srcu_down_read() for more information. 397 * 398 * The same srcu_struct may be used concurrently by srcu_down_read_fast() 399 * and srcu_read_lock_fast(). However, the same definition/initialization 400 * requirements called out for srcu_read_lock_safe() apply. 401 */ 402 static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp) 403 { 404 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); 405 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast()."); 406 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 407 return __srcu_read_lock_fast_updown(ssp); 408 } 409 410 /** 411 * srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure. 412 * @ssp: srcu_struct in which to register the new reader. 413 * 414 * Enter an SRCU read-side critical section, but in an NMI-safe manner. 415 * See srcu_read_lock() for more information. 416 * 417 * If srcu_read_lock_nmisafe() is ever used on an srcu_struct structure, 418 * then none of the other flavors may be used, whether before, during, 419 * or after. 420 */ 421 static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) 422 __acquires_shared(ssp) 423 { 424 int retval; 425 426 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI); 427 retval = __srcu_read_lock_nmisafe(ssp); 428 rcu_try_lock_acquire(&ssp->dep_map); 429 return retval; 430 } 431 432 /* Used by tracing, cannot be traced and cannot invoke lockdep. */ 433 static inline notrace int 434 srcu_read_lock_notrace(struct srcu_struct *ssp) 435 __acquires_shared(ssp) 436 { 437 int retval; 438 439 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 440 retval = __srcu_read_lock(ssp); 441 return retval; 442 } 443 444 /** 445 * srcu_down_read - register a new reader for an SRCU-protected structure. 446 * @ssp: srcu_struct in which to register the new reader. 447 * 448 * Enter a semaphore-like SRCU read-side critical section. Note that 449 * SRCU read-side critical sections may be nested. However, it is 450 * illegal to call anything that waits on an SRCU grace period for the 451 * same srcu_struct, whether directly or indirectly. Please note that 452 * one way to indirectly wait on an SRCU grace period is to acquire 453 * a mutex that is held elsewhere while calling synchronize_srcu() or 454 * synchronize_srcu_expedited(). But if you want lockdep to help you 455 * keep this stuff straight, you should instead use srcu_read_lock(). 456 * 457 * The semaphore-like nature of srcu_down_read() means that the matching 458 * srcu_up_read() can be invoked from some other context, for example, 459 * from some other task or from an irq handler. However, neither 460 * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler. 461 * 462 * Calls to srcu_down_read() may be nested, similar to the manner in 463 * which calls to down_read() may be nested. The same srcu_struct may be 464 * used concurrently by srcu_down_read() and srcu_read_lock(). 465 */ 466 static inline int srcu_down_read(struct srcu_struct *ssp) 467 __acquires_shared(ssp) 468 { 469 WARN_ON_ONCE(in_nmi()); 470 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 471 return __srcu_read_lock(ssp); 472 } 473 474 /** 475 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. 476 * @ssp: srcu_struct in which to unregister the old reader. 477 * @idx: return value from corresponding srcu_read_lock(). 478 * 479 * Exit an SRCU read-side critical section. 480 */ 481 static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) 482 __releases_shared(ssp) 483 { 484 WARN_ON_ONCE(idx & ~0x1); 485 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 486 srcu_lock_release(&ssp->dep_map); 487 __srcu_read_unlock(ssp, idx); 488 } 489 490 /** 491 * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. 492 * @ssp: srcu_struct in which to unregister the old reader. 493 * @scp: return value from corresponding srcu_read_lock_fast(). 494 * 495 * Exit a light-weight SRCU read-side critical section. 496 */ 497 static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) 498 __releases_shared(ssp) 499 { 500 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 501 srcu_lock_release(&ssp->dep_map); 502 __srcu_read_unlock_fast(ssp, scp); 503 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast()."); 504 } 505 506 /** 507 * srcu_read_unlock_fast_updown - unregister a old reader from an SRCU-fast-updown structure. 508 * @ssp: srcu_struct in which to unregister the old reader. 509 * @scp: return value from corresponding srcu_read_lock_fast_updown(). 510 * 511 * Exit an SRCU-fast-updown read-side critical section. 512 */ 513 static inline void 514 srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases_shared(ssp) 515 { 516 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 517 srcu_lock_release(&ssp->dep_map); 518 __srcu_read_unlock_fast_updown(ssp, scp); 519 RCU_LOCKDEP_WARN(!rcu_is_watching(), 520 "RCU must be watching srcu_read_unlock_fast_updown()."); 521 } 522 523 /* 524 * Used by tracing, cannot be traced and cannot call lockdep. 525 * See srcu_read_unlock_fast() for more information. 526 */ 527 static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp, 528 struct srcu_ctr __percpu *scp) __releases_shared(ssp) 529 { 530 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 531 __srcu_read_unlock_fast(ssp, scp); 532 } 533 534 /** 535 * srcu_up_read_fast - unregister a old reader from an SRCU-protected structure. 536 * @ssp: srcu_struct in which to unregister the old reader. 537 * @scp: return value from corresponding srcu_read_lock_fast(). 538 * 539 * Exit an SRCU read-side critical section, but not necessarily from 540 * the same context as the maching srcu_down_read_fast(). 541 */ 542 static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) 543 __releases_shared(ssp) 544 { 545 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); 546 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 547 __srcu_read_unlock_fast_updown(ssp, scp); 548 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_up_read_fast_updown()."); 549 } 550 551 /** 552 * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure. 553 * @ssp: srcu_struct in which to unregister the old reader. 554 * @idx: return value from corresponding srcu_read_lock_nmisafe(). 555 * 556 * Exit an SRCU read-side critical section, but in an NMI-safe manner. 557 */ 558 static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) 559 __releases_shared(ssp) 560 { 561 WARN_ON_ONCE(idx & ~0x1); 562 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI); 563 rcu_lock_release(&ssp->dep_map); 564 __srcu_read_unlock_nmisafe(ssp, idx); 565 } 566 567 /* Used by tracing, cannot be traced and cannot call lockdep. */ 568 static inline notrace void 569 srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases_shared(ssp) 570 { 571 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 572 __srcu_read_unlock(ssp, idx); 573 } 574 575 /** 576 * srcu_up_read - unregister a old reader from an SRCU-protected structure. 577 * @ssp: srcu_struct in which to unregister the old reader. 578 * @idx: return value from corresponding srcu_read_lock(). 579 * 580 * Exit an SRCU read-side critical section, but not necessarily from 581 * the same context as the maching srcu_down_read(). 582 */ 583 static inline void srcu_up_read(struct srcu_struct *ssp, int idx) 584 __releases_shared(ssp) 585 { 586 WARN_ON_ONCE(idx & ~0x1); 587 WARN_ON_ONCE(in_nmi()); 588 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 589 __srcu_read_unlock(ssp, idx); 590 } 591 592 /** 593 * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock 594 * 595 * Converts the preceding srcu_read_unlock into a two-way memory barrier. 596 * 597 * Call this after srcu_read_unlock, to guarantee that all memory operations 598 * that occur after smp_mb__after_srcu_read_unlock will appear to happen after 599 * the preceding srcu_read_unlock. 600 */ 601 static inline void smp_mb__after_srcu_read_unlock(void) 602 { 603 /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */ 604 } 605 606 /** 607 * smp_mb__after_srcu_read_lock - ensure full ordering after srcu_read_lock 608 * 609 * Converts the preceding srcu_read_lock into a two-way memory barrier. 610 * 611 * Call this after srcu_read_lock, to guarantee that all memory operations 612 * that occur after smp_mb__after_srcu_read_lock will appear to happen after 613 * the preceding srcu_read_lock. 614 */ 615 static inline void smp_mb__after_srcu_read_lock(void) 616 { 617 /* __srcu_read_lock has smp_mb() internally so nothing to do here. */ 618 } 619 620 DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct, 621 _T->idx = srcu_read_lock(_T->lock), 622 srcu_read_unlock(_T->lock, _T->idx), 623 int idx) 624 DECLARE_LOCK_GUARD_1_ATTRS(srcu, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 625 #define class_srcu_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu, _T) 626 627 DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct, 628 _T->scp = srcu_read_lock_fast(_T->lock), 629 srcu_read_unlock_fast(_T->lock, _T->scp), 630 struct srcu_ctr __percpu *scp) 631 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 632 #define class_srcu_fast_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast, _T) 633 634 DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct, 635 _T->scp = srcu_read_lock_fast_notrace(_T->lock), 636 srcu_read_unlock_fast_notrace(_T->lock, _T->scp), 637 struct srcu_ctr __percpu *scp) 638 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 639 #define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T) 640 641 #endif 642