1 /* SPDX-License-Identifier: GPL-2.0+ */ 2 /* 3 * Sleepable Read-Copy Update mechanism for mutual exclusion 4 * 5 * Copyright (C) IBM Corporation, 2006 6 * Copyright (C) Fujitsu, 2012 7 * 8 * Author: Paul McKenney <paulmck@linux.ibm.com> 9 * Lai Jiangshan <laijs@cn.fujitsu.com> 10 * 11 * For detailed explanation of Read-Copy Update mechanism see - 12 * Documentation/RCU/ *.txt 13 * 14 */ 15 16 #ifndef _LINUX_SRCU_H 17 #define _LINUX_SRCU_H 18 19 #include <linux/mutex.h> 20 #include <linux/rcupdate.h> 21 #include <linux/workqueue.h> 22 #include <linux/rcu_segcblist.h> 23 24 context_lock_struct(srcu_struct, __reentrant_ctx_lock); 25 26 #ifdef CONFIG_DEBUG_LOCK_ALLOC 27 28 int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, 29 struct lock_class_key *key); 30 static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, 31 struct lock_class_key *key) 32 { 33 return init_srcu_struct_lockdep(ssp, name, key); 34 } 35 #ifndef CONFIG_TINY_SRCU 36 int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); 37 int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, 38 struct lock_class_key *key); 39 int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name, 40 struct lock_class_key *key); 41 #endif // #ifndef CONFIG_TINY_SRCU 42 43 #define init_srcu_struct_fast(ssp) \ 44 ({ \ 45 static struct lock_class_key __srcu_key; \ 46 \ 47 __init_srcu_struct_fast((ssp), #ssp, &__srcu_key); \ 48 }) 49 50 #define init_srcu_struct_fast_updown(ssp) \ 51 ({ \ 52 static struct lock_class_key __srcu_key; \ 53 \ 54 __init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \ 55 }) 56 57 #define init_srcu_struct_atomic(ssp) \ 58 ({ \ 59 static struct lock_class_key __srcu_key; \ 60 \ 61 __init_srcu_struct_atomic((ssp), #ssp, &__srcu_key); \ 62 }) 63 64 #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, 65 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 66 67 int init_srcu_struct_generic(struct srcu_struct *ssp); 68 static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, 69 struct lock_class_key *key) 70 { 71 return init_srcu_struct_generic(ssp); 72 } 73 #ifndef CONFIG_TINY_SRCU 74 int init_srcu_struct_fast(struct srcu_struct *ssp); 75 int init_srcu_struct_fast_updown(struct srcu_struct *ssp); 76 int init_srcu_struct_atomic(struct srcu_struct *ssp); 77 #endif // #ifndef CONFIG_TINY_SRCU 78 79 #define __SRCU_DEP_MAP_INIT(srcu_name) 80 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 81 82 #define init_srcu_struct(ssp) \ 83 ({ \ 84 static struct lock_class_key __srcu_key; \ 85 \ 86 __init_srcu_struct((ssp), #ssp, &__srcu_key); \ 87 }) 88 89 /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ 90 #define SRCU_READ_FLAVOR_NORMAL 0x01 // srcu_read_lock(). 91 #define SRCU_READ_FLAVOR_NMI 0x02 // srcu_read_lock_nmisafe(). 92 // 0x04 // SRCU-lite is no longer with us. 93 #define SRCU_READ_FLAVOR_FAST 0x04 // srcu_read_lock_fast(), also NMI-safe. 94 #define SRCU_READ_FLAVOR_FAST_UPDOWN 0x08 // srcu_read_lock_fast_updown(). 95 #define SRCU_READ_FLAVOR_ATOMIC 0x10 // srcu_read_lock_atomic(). 96 #define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ 97 SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN | \ 98 SRCU_READ_FLAVOR_ATOMIC) 99 // All of the above. 100 #define SRCU_READ_FLAVOR_PREDEF (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_ATOMIC) 101 // Flavors special DEFINE_SRCU() flavors. 102 #define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) 103 // Flavors requiring synchronize_rcu() 104 // instead of smp_mb(). 105 void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases_shared(ssp); 106 107 #ifdef CONFIG_TINY_SRCU 108 #include <linux/srcutiny.h> 109 #elif defined(CONFIG_TREE_SRCU) 110 #include <linux/srcutree.h> 111 #else 112 #error "Unknown SRCU implementation specified to kernel configuration" 113 #endif 114 115 void call_srcu(struct srcu_struct *ssp, struct rcu_head *head, 116 void (*func)(struct rcu_head *head)); 117 void cleanup_srcu_struct(struct srcu_struct *ssp); 118 void synchronize_srcu(struct srcu_struct *ssp); 119 void synchronize_srcu_atomic(struct srcu_struct *ssp); 120 121 #define SRCU_GET_STATE_COMPLETED 0x1 122 123 /** 124 * get_completed_synchronize_srcu - Return a pre-completed polled state cookie 125 * 126 * Returns a value that poll_state_synchronize_srcu() will always treat 127 * as a cookie whose grace period has already completed. 128 */ 129 static inline unsigned long get_completed_synchronize_srcu(void) 130 { 131 return SRCU_GET_STATE_COMPLETED; 132 } 133 134 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp); 135 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp); 136 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie); 137 138 // Maximum number of unsigned long values corresponding to 139 // not-yet-completed SRCU grace periods. 140 #define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2 141 142 /** 143 * same_state_synchronize_srcu - Are two old-state values identical? 144 * @oldstate1: First old-state value. 145 * @oldstate2: Second old-state value. 146 * 147 * The two old-state values must have been obtained from either 148 * get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or 149 * get_completed_synchronize_srcu(). Returns @true if the two values are 150 * identical and @false otherwise. This allows structures whose lifetimes 151 * are tracked by old-state values to push these values to a list header, 152 * allowing those structures to be slightly smaller. 153 */ 154 static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2) 155 { 156 return oldstate1 == oldstate2; 157 } 158 159 #ifdef CONFIG_NEED_SRCU_NMI_SAFE 160 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires_shared(ssp); 161 void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases_shared(ssp); 162 #else 163 static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) 164 __acquires_shared(ssp) 165 { 166 return __srcu_read_lock(ssp); 167 } 168 static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) 169 __releases_shared(ssp) 170 { 171 __srcu_read_unlock(ssp, idx); 172 } 173 #endif /* CONFIG_NEED_SRCU_NMI_SAFE */ 174 175 void srcu_init(void); 176 177 #ifdef CONFIG_DEBUG_LOCK_ALLOC 178 179 /** 180 * srcu_read_lock_held - might we be in SRCU read-side critical section? 181 * @ssp: The srcu_struct structure to check 182 * 183 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU 184 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, 185 * this assumes we are in an SRCU read-side critical section unless it can 186 * prove otherwise. 187 * 188 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot 189 * and while lockdep is disabled. 190 * 191 * Note that SRCU is based on its own statemachine and it doesn't 192 * relies on normal RCU, it can be called from the CPU which 193 * is in the idle loop from an RCU point of view or offline. 194 */ 195 static inline int srcu_read_lock_held(const struct srcu_struct *ssp) 196 { 197 if (!debug_lockdep_rcu_enabled()) 198 return 1; 199 return lock_is_held(&ssp->dep_map); 200 } 201 202 /* 203 * Annotations provide deadlock detection for SRCU. 204 * 205 * Similar to other lockdep annotations, except there is an additional 206 * srcu_lock_sync(), which is basically an empty *write*-side critical section, 207 * see lock_sync() for more information. 208 */ 209 210 /* Annotates a srcu_read_lock() */ 211 static inline void srcu_lock_acquire(struct lockdep_map *map) 212 { 213 lock_map_acquire_read(map); 214 } 215 216 /* Annotates a srcu_read_lock() */ 217 static inline void srcu_lock_release(struct lockdep_map *map) 218 { 219 lock_map_release(map); 220 } 221 222 /* Annotates a synchronize_srcu() */ 223 static inline void srcu_lock_sync(struct lockdep_map *map) 224 { 225 lock_map_sync(map); 226 } 227 228 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 229 230 static inline int srcu_read_lock_held(const struct srcu_struct *ssp) 231 { 232 return 1; 233 } 234 235 #define srcu_lock_acquire(m) do { } while (0) 236 #define srcu_lock_release(m) do { } while (0) 237 #define srcu_lock_sync(m) do { } while (0) 238 239 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 240 241 /* 242 * No-op helper to denote that ssp must be held. Because SRCU-protected pointers 243 * should still be marked with __rcu_guarded, and we do not want to mark them 244 * with __guarded_by(ssp) as it would complicate annotations for writers, we 245 * choose the following strategy: srcu_dereference_check() calls this helper 246 * that checks that the passed ssp is held, and then fake-acquires 'RCU'. 247 */ 248 static inline void __srcu_read_lock_must_hold(const struct srcu_struct *ssp) __must_hold_shared(ssp) { } 249 250 /** 251 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing 252 * @p: the pointer to fetch and protect for later dereferencing 253 * @ssp: pointer to the srcu_struct, which is used to check that we 254 * really are in an SRCU read-side critical section. 255 * @c: condition to check for update-side use 256 * 257 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side 258 * critical section will result in an RCU-lockdep splat, unless @c evaluates 259 * to 1. The @c argument will normally be a logical expression containing 260 * lockdep_is_held() calls. 261 */ 262 #define srcu_dereference_check(p, ssp, c) \ 263 ({ \ 264 __srcu_read_lock_must_hold(ssp); \ 265 __acquire_shared_ctx_lock(RCU); \ 266 __auto_type __v = __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ 267 (c) || srcu_read_lock_held(ssp), __rcu); \ 268 __release_shared_ctx_lock(RCU); \ 269 __v; \ 270 }) 271 272 /** 273 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing 274 * @p: the pointer to fetch and protect for later dereferencing 275 * @ssp: pointer to the srcu_struct, which is used to check that we 276 * really are in an SRCU read-side critical section. 277 * 278 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU 279 * is enabled, invoking this outside of an RCU read-side critical 280 * section will result in an RCU-lockdep splat. 281 */ 282 #define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0) 283 284 /** 285 * srcu_dereference_notrace - no tracing and no lockdep calls from here 286 * @p: the pointer to fetch and protect for later dereferencing 287 * @ssp: pointer to the srcu_struct, which is used to check that we 288 * really are in an SRCU read-side critical section. 289 */ 290 #define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1) 291 292 /** 293 * srcu_read_lock - register a new reader for an SRCU-protected structure. 294 * @ssp: srcu_struct in which to register the new reader. 295 * 296 * Enter an SRCU read-side critical section. Note that SRCU read-side 297 * critical sections may be nested. However, it is illegal to 298 * call anything that waits on an SRCU grace period for the same 299 * srcu_struct, whether directly or indirectly. Please note that 300 * one way to indirectly wait on an SRCU grace period is to acquire 301 * a mutex that is held elsewhere while calling synchronize_srcu() or 302 * synchronize_srcu_expedited(). 303 * 304 * The return value from srcu_read_lock() is guaranteed to be 305 * non-negative. This value must be passed unaltered to the matching 306 * srcu_read_unlock(). Note that srcu_read_lock() and the matching 307 * srcu_read_unlock() must occur in the same context, for example, it is 308 * illegal to invoke srcu_read_unlock() in an irq handler if the matching 309 * srcu_read_lock() was invoked in process context. Or, for that matter to 310 * invoke srcu_read_unlock() from one task and the matching srcu_read_lock() 311 * from another. 312 */ 313 static inline int srcu_read_lock(struct srcu_struct *ssp) 314 __acquires_shared(ssp) 315 { 316 int retval; 317 318 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 319 retval = __srcu_read_lock(ssp); 320 srcu_lock_acquire(&ssp->dep_map); 321 return retval; 322 } 323 324 /** 325 * srcu_read_lock_atomic - register a new reader promising an atomic section 326 * @ssp: srcu_struct in which to register the new reader. 327 * 328 * As srcu_read_lock(), but the caller promises that the read-side 329 * critical section never sleeps and never blocks on anything which 330 * may itself depend on memory allocation to make progress. Preemption 331 * is disabled for the duration, which both enforces that promise (any 332 * sleepable call in the section will splat on every configuration) 333 * and bounds the section so that the update side may spin rather 334 * than sleep when waiting for readers: see synchronize_srcu_atomic(). 335 * 336 * The lock and matching srcu_read_unlock_atomic() must be invoked on 337 * the same CPU, from the same context; passing the return value to 338 * another task is not permitted for this flavor. 339 */ 340 static inline int srcu_read_lock_atomic(struct srcu_struct *ssp) 341 __acquires_shared(ssp) 342 { 343 int retval; 344 345 preempt_disable(); 346 /* 347 * Arm might_sleep() to catch even a *potentially* sleeping call 348 * in the section, not just an actual schedule: the atomic-domain 349 * promise must hold on every path, contended or not. In hardirq, 350 * softirq, or NMI the annotation would land on the interrupted 351 * task, and can also result in data races against that task's 352 * own non_block_start()/non_block_end() invocations; it is also 353 * redundant there, so skip it. 354 */ 355 if (in_task()) 356 non_block_start(); 357 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); 358 retval = __srcu_read_lock(ssp); 359 srcu_lock_acquire(&ssp->dep_map); 360 return retval; 361 } 362 363 /** 364 * srcu_read_lock_fast - register a new reader for an SRCU-protected structure. 365 * @ssp: srcu_struct in which to register the new reader. 366 * 367 * Enter an SRCU read-side critical section, but for a light-weight 368 * smp_mb()-free reader. See srcu_read_lock() for more information. This 369 * function is NMI-safe, in a manner similar to srcu_read_lock_nmisafe(). 370 * 371 * For srcu_read_lock_fast() to be used on an srcu_struct structure, 372 * that structure must have been defined using either DEFINE_SRCU_FAST() 373 * or DEFINE_STATIC_SRCU_FAST() on the one hand or initialized with 374 * init_srcu_struct_fast() on the other. Such an srcu_struct structure 375 * cannot be passed to any non-fast variant of srcu_read_{,un}lock() or 376 * srcu_{down,up}_read(). In kernels built with CONFIG_PROVE_RCU=y, 377 * __srcu_check_read_flavor() will complain bitterly if you ignore this 378 * restriction. 379 * 380 * Grace-period auto-expediting is disabled for SRCU-fast srcu_struct 381 * structures because SRCU-fast expedited grace periods invoke 382 * synchronize_rcu_expedited(), IPIs and all. If you need expedited 383 * SRCU-fast grace periods, use synchronize_srcu_expedited(). 384 * 385 * The srcu_read_lock_fast() function can be invoked only from those 386 * contexts where RCU is watching, that is, from contexts where it would 387 * be legal to invoke rcu_read_lock(). Otherwise, lockdep will complain. 388 */ 389 static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires_shared(ssp) 390 __acquires_shared(ssp) 391 { 392 struct srcu_ctr __percpu *retval; 393 394 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast()."); 395 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 396 retval = __srcu_read_lock_fast(ssp); 397 rcu_try_lock_acquire(&ssp->dep_map); 398 return retval; 399 } 400 401 /** 402 * srcu_read_lock_fast_updown - register a new reader for an SRCU-fast-updown structure. 403 * @ssp: srcu_struct in which to register the new reader. 404 * 405 * Enter an SRCU read-side critical section, but for a light-weight 406 * smp_mb()-free reader. See srcu_read_lock() for more information. 407 * This function is compatible with srcu_down_read_fast(), but is not 408 * NMI-safe. 409 * 410 * For srcu_read_lock_fast_updown() to be used on an srcu_struct 411 * structure, that structure must have been defined using either 412 * DEFINE_SRCU_FAST_UPDOWN() or DEFINE_STATIC_SRCU_FAST_UPDOWN() on the one 413 * hand or initialized with init_srcu_struct_fast_updown() on the other. 414 * Such an srcu_struct structure cannot be passed to any non-fast-updown 415 * variant of srcu_read_{,un}lock() or srcu_{down,up}_read(). In kernels 416 * built with CONFIG_PROVE_RCU=y, __srcu_check_read_flavor() will complain 417 * bitterly if you ignore this * restriction. 418 * 419 * Grace-period auto-expediting is disabled for SRCU-fast-updown 420 * srcu_struct structures because SRCU-fast-updown expedited grace periods 421 * invoke synchronize_rcu_expedited(), IPIs and all. If you need expedited 422 * SRCU-fast-updown grace periods, use synchronize_srcu_expedited(). 423 * 424 * The srcu_read_lock_fast_updown() function can be invoked only from 425 * those contexts where RCU is watching, that is, from contexts where 426 * it would be legal to invoke rcu_read_lock(). Otherwise, lockdep will 427 * complain. 428 */ 429 static inline struct srcu_ctr __percpu *srcu_read_lock_fast_updown(struct srcu_struct *ssp) 430 __acquires_shared(ssp) 431 { 432 struct srcu_ctr __percpu *retval; 433 434 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast_updown()."); 435 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 436 retval = __srcu_read_lock_fast_updown(ssp); 437 rcu_try_lock_acquire(&ssp->dep_map); 438 return retval; 439 } 440 441 /* 442 * Used by tracing, cannot be traced and cannot call lockdep. 443 * See srcu_read_lock_fast() for more information. 444 */ 445 static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp) 446 __acquires_shared(ssp) 447 { 448 struct srcu_ctr __percpu *retval; 449 450 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 451 retval = __srcu_read_lock_fast(ssp); 452 return retval; 453 } 454 455 /** 456 * srcu_down_read_fast - register a new reader for an SRCU-protected structure. 457 * @ssp: srcu_struct in which to register the new reader. 458 * 459 * Enter a semaphore-like SRCU read-side critical section, but for 460 * a light-weight smp_mb()-free reader. See srcu_read_lock_fast() and 461 * srcu_down_read() for more information. 462 * 463 * The same srcu_struct may be used concurrently by srcu_down_read_fast() 464 * and srcu_read_lock_fast(). However, the same definition/initialization 465 * requirements called out for srcu_read_lock_fast_updown() apply. 466 */ 467 static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp) 468 { 469 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); 470 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast()."); 471 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 472 return __srcu_read_lock_fast_updown(ssp); 473 } 474 475 /** 476 * srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure. 477 * @ssp: srcu_struct in which to register the new reader. 478 * 479 * Enter an SRCU read-side critical section, but in an NMI-safe manner. 480 * See srcu_read_lock() for more information. 481 * 482 * If srcu_read_lock_nmisafe() is ever used on an srcu_struct structure, 483 * then none of the other flavors may be used, whether before, during, 484 * or after. 485 */ 486 static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) 487 __acquires_shared(ssp) 488 { 489 int retval; 490 491 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI); 492 retval = __srcu_read_lock_nmisafe(ssp); 493 rcu_try_lock_acquire(&ssp->dep_map); 494 return retval; 495 } 496 497 /* Used by tracing, cannot be traced and cannot invoke lockdep. */ 498 static inline notrace int 499 srcu_read_lock_notrace(struct srcu_struct *ssp) 500 __acquires_shared(ssp) 501 { 502 int retval; 503 504 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 505 retval = __srcu_read_lock(ssp); 506 return retval; 507 } 508 509 /** 510 * srcu_down_read - register a new reader for an SRCU-protected structure. 511 * @ssp: srcu_struct in which to register the new reader. 512 * 513 * Enter a semaphore-like SRCU read-side critical section. Note that 514 * SRCU read-side critical sections may be nested. However, it is 515 * illegal to call anything that waits on an SRCU grace period for the 516 * same srcu_struct, whether directly or indirectly. Please note that 517 * one way to indirectly wait on an SRCU grace period is to acquire 518 * a mutex that is held elsewhere while calling synchronize_srcu() or 519 * synchronize_srcu_expedited(). But if you want lockdep to help you 520 * keep this stuff straight, you should instead use srcu_read_lock(). 521 * 522 * The semaphore-like nature of srcu_down_read() means that the matching 523 * srcu_up_read() can be invoked from some other context, for example, 524 * from some other task or from an irq handler. However, neither 525 * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler. 526 * 527 * Calls to srcu_down_read() may be nested, similar to the manner in 528 * which calls to down_read() may be nested. The same srcu_struct may be 529 * used concurrently by srcu_down_read() and srcu_read_lock(). 530 */ 531 static inline int srcu_down_read(struct srcu_struct *ssp) 532 __acquires_shared(ssp) 533 { 534 WARN_ON_ONCE(in_nmi()); 535 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 536 return __srcu_read_lock(ssp); 537 } 538 539 /** 540 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. 541 * @ssp: srcu_struct in which to unregister the old reader. 542 * @idx: return value from corresponding srcu_read_lock(). 543 * 544 * Exit an SRCU read-side critical section. 545 */ 546 static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) 547 __releases_shared(ssp) 548 { 549 WARN_ON_ONCE(idx & ~0x1); 550 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 551 srcu_lock_release(&ssp->dep_map); 552 __srcu_read_unlock(ssp, idx); 553 } 554 555 /** 556 * srcu_read_unlock_atomic - unregister an atomic-section reader 557 * @ssp: srcu_struct from which to unregister the old reader. 558 * @idx: return value from corresponding srcu_read_lock_atomic(). 559 */ 560 static inline void srcu_read_unlock_atomic(struct srcu_struct *ssp, int idx) 561 __releases_shared(ssp) 562 { 563 WARN_ON_ONCE(idx & ~0x1); 564 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); 565 srcu_lock_release(&ssp->dep_map); 566 __srcu_read_unlock(ssp, idx); 567 if (in_task()) 568 non_block_end(); 569 preempt_enable(); 570 } 571 572 /** 573 * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. 574 * @ssp: srcu_struct in which to unregister the old reader. 575 * @scp: return value from corresponding srcu_read_lock_fast(). 576 * 577 * Exit a light-weight SRCU read-side critical section. 578 */ 579 static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) 580 __releases_shared(ssp) 581 { 582 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 583 srcu_lock_release(&ssp->dep_map); 584 __srcu_read_unlock_fast(ssp, scp); 585 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast()."); 586 } 587 588 /** 589 * srcu_read_unlock_fast_updown - unregister a old reader from an SRCU-fast-updown structure. 590 * @ssp: srcu_struct in which to unregister the old reader. 591 * @scp: return value from corresponding srcu_read_lock_fast_updown(). 592 * 593 * Exit an SRCU-fast-updown read-side critical section. 594 */ 595 static inline void 596 srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases_shared(ssp) 597 { 598 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 599 srcu_lock_release(&ssp->dep_map); 600 __srcu_read_unlock_fast_updown(ssp, scp); 601 RCU_LOCKDEP_WARN(!rcu_is_watching(), 602 "RCU must be watching srcu_read_unlock_fast_updown()."); 603 } 604 605 /* 606 * Used by tracing, cannot be traced and cannot call lockdep. 607 * See srcu_read_unlock_fast() for more information. 608 */ 609 static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp, 610 struct srcu_ctr __percpu *scp) __releases_shared(ssp) 611 { 612 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 613 __srcu_read_unlock_fast(ssp, scp); 614 } 615 616 /** 617 * srcu_up_read_fast - unregister a old reader from an SRCU-protected structure. 618 * @ssp: srcu_struct in which to unregister the old reader. 619 * @scp: return value from corresponding srcu_read_lock_fast(). 620 * 621 * Exit an SRCU read-side critical section, but not necessarily from 622 * the same context as the maching srcu_down_read_fast(). 623 */ 624 static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) 625 __releases_shared(ssp) 626 { 627 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); 628 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 629 __srcu_read_unlock_fast_updown(ssp, scp); 630 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_up_read_fast_updown()."); 631 } 632 633 /** 634 * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure. 635 * @ssp: srcu_struct in which to unregister the old reader. 636 * @idx: return value from corresponding srcu_read_lock_nmisafe(). 637 * 638 * Exit an SRCU read-side critical section, but in an NMI-safe manner. 639 */ 640 static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) 641 __releases_shared(ssp) 642 { 643 WARN_ON_ONCE(idx & ~0x1); 644 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI); 645 rcu_lock_release(&ssp->dep_map); 646 __srcu_read_unlock_nmisafe(ssp, idx); 647 } 648 649 /* Used by tracing, cannot be traced and cannot call lockdep. */ 650 static inline notrace void 651 srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases_shared(ssp) 652 { 653 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 654 __srcu_read_unlock(ssp, idx); 655 } 656 657 /** 658 * srcu_up_read - unregister a old reader from an SRCU-protected structure. 659 * @ssp: srcu_struct in which to unregister the old reader. 660 * @idx: return value from corresponding srcu_read_lock(). 661 * 662 * Exit an SRCU read-side critical section, but not necessarily from 663 * the same context as the maching srcu_down_read(). 664 */ 665 static inline void srcu_up_read(struct srcu_struct *ssp, int idx) 666 __releases_shared(ssp) 667 { 668 WARN_ON_ONCE(idx & ~0x1); 669 WARN_ON_ONCE(in_nmi()); 670 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 671 __srcu_read_unlock(ssp, idx); 672 } 673 674 /** 675 * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock 676 * 677 * Converts the preceding srcu_read_unlock into a two-way memory barrier. 678 * 679 * Call this after srcu_read_unlock, to guarantee that all memory operations 680 * that occur after smp_mb__after_srcu_read_unlock will appear to happen after 681 * the preceding srcu_read_unlock. 682 */ 683 static inline void smp_mb__after_srcu_read_unlock(void) 684 { 685 /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */ 686 } 687 688 /** 689 * smp_mb__after_srcu_read_lock - ensure full ordering after srcu_read_lock 690 * 691 * Converts the preceding srcu_read_lock into a two-way memory barrier. 692 * 693 * Call this after srcu_read_lock, to guarantee that all memory operations 694 * that occur after smp_mb__after_srcu_read_lock will appear to happen after 695 * the preceding srcu_read_lock. 696 */ 697 static inline void smp_mb__after_srcu_read_lock(void) 698 { 699 /* __srcu_read_lock has smp_mb() internally so nothing to do here. */ 700 } 701 702 DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct, 703 _T->idx = srcu_read_lock(_T->lock), 704 srcu_read_unlock(_T->lock, _T->idx), 705 int idx) 706 DECLARE_LOCK_GUARD_1_ATTRS(srcu, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 707 #define class_srcu_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu, _T) 708 709 DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct, 710 _T->scp = srcu_read_lock_fast(_T->lock), 711 srcu_read_unlock_fast(_T->lock, _T->scp), 712 struct srcu_ctr __percpu *scp) 713 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 714 #define class_srcu_fast_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast, _T) 715 716 DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct, 717 _T->scp = srcu_read_lock_fast_notrace(_T->lock), 718 srcu_read_unlock_fast_notrace(_T->lock, _T->scp), 719 struct srcu_ctr __percpu *scp) 720 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 721 #define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T) 722 723 DEFINE_LOCK_GUARD_1(srcu_fast_updown, struct srcu_struct, 724 _T->scp = srcu_read_lock_fast_updown(_T->lock), 725 srcu_read_unlock_fast_updown(_T->lock, _T->scp), 726 struct srcu_ctr __percpu *scp) 727 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_updown, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 728 #define class_srcu_fast_updown_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_updown, _T) 729 730 #endif 731