1 /* SPDX-License-Identifier: GPL-2.0+ */ 2 /* 3 * Sleepable Read-Copy Update mechanism for mutual exclusion 4 * 5 * Copyright (C) IBM Corporation, 2006 6 * Copyright (C) Fujitsu, 2012 7 * 8 * Author: Paul McKenney <paulmck@linux.ibm.com> 9 * Lai Jiangshan <laijs@cn.fujitsu.com> 10 * 11 * For detailed explanation of Read-Copy Update mechanism see - 12 * Documentation/RCU/ *.txt 13 * 14 */ 15 16 #ifndef _LINUX_SRCU_H 17 #define _LINUX_SRCU_H 18 19 #include <linux/mutex.h> 20 #include <linux/rcupdate.h> 21 #include <linux/rculist.h> 22 #include <linux/workqueue.h> 23 #include <linux/rcu_segcblist.h> 24 25 context_lock_struct(srcu_struct, __reentrant_ctx_lock); 26 27 #ifdef CONFIG_DEBUG_LOCK_ALLOC 28 29 int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, 30 struct lock_class_key *key); 31 static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, 32 struct lock_class_key *key) 33 { 34 return init_srcu_struct_lockdep(ssp, name, key); 35 } 36 #ifndef CONFIG_TINY_SRCU 37 int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); 38 int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, 39 struct lock_class_key *key); 40 int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name, 41 struct lock_class_key *key); 42 #endif // #ifndef CONFIG_TINY_SRCU 43 44 #define init_srcu_struct_fast(ssp) \ 45 ({ \ 46 static struct lock_class_key __srcu_key; \ 47 \ 48 __init_srcu_struct_fast((ssp), #ssp, &__srcu_key); \ 49 }) 50 51 #define init_srcu_struct_fast_updown(ssp) \ 52 ({ \ 53 static struct lock_class_key __srcu_key; \ 54 \ 55 __init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \ 56 }) 57 58 #define init_srcu_struct_atomic(ssp) \ 59 ({ \ 60 static struct lock_class_key __srcu_key; \ 61 \ 62 __init_srcu_struct_atomic((ssp), #ssp, &__srcu_key); \ 63 }) 64 65 #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, 66 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 67 68 int init_srcu_struct_generic(struct srcu_struct *ssp); 69 static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, 70 struct lock_class_key *key) 71 { 72 return init_srcu_struct_generic(ssp); 73 } 74 #ifndef CONFIG_TINY_SRCU 75 int init_srcu_struct_fast(struct srcu_struct *ssp); 76 int init_srcu_struct_fast_updown(struct srcu_struct *ssp); 77 int init_srcu_struct_atomic(struct srcu_struct *ssp); 78 #endif // #ifndef CONFIG_TINY_SRCU 79 80 #define __SRCU_DEP_MAP_INIT(srcu_name) 81 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 82 83 #define init_srcu_struct(ssp) \ 84 ({ \ 85 static struct lock_class_key __srcu_key; \ 86 \ 87 __init_srcu_struct((ssp), #ssp, &__srcu_key); \ 88 }) 89 90 /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ 91 #define SRCU_READ_FLAVOR_NORMAL 0x01 // srcu_read_lock(). 92 #define SRCU_READ_FLAVOR_NMI 0x02 // srcu_read_lock_nmisafe(). 93 // 0x04 // SRCU-lite is no longer with us. 94 #define SRCU_READ_FLAVOR_FAST 0x04 // srcu_read_lock_fast(), also NMI-safe. 95 #define SRCU_READ_FLAVOR_FAST_UPDOWN 0x08 // srcu_read_lock_fast_updown(). 96 #define SRCU_READ_FLAVOR_ATOMIC 0x10 // srcu_read_lock_atomic(). 97 #define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ 98 SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN | \ 99 SRCU_READ_FLAVOR_ATOMIC) 100 // All of the above. 101 #define SRCU_READ_FLAVOR_PREDEF (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_ATOMIC) 102 // Flavors special DEFINE_SRCU() flavors. 103 #define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) 104 // Flavors requiring synchronize_rcu() 105 // instead of smp_mb(). 106 void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases_shared(ssp); 107 108 #ifdef CONFIG_TINY_SRCU 109 #include <linux/srcutiny.h> 110 #elif defined(CONFIG_TREE_SRCU) 111 #include <linux/srcutree.h> 112 #else 113 #error "Unknown SRCU implementation specified to kernel configuration" 114 #endif 115 116 void call_srcu(struct srcu_struct *ssp, struct rcu_head *head, 117 void (*func)(struct rcu_head *head)); 118 void cleanup_srcu_struct(struct srcu_struct *ssp); 119 void synchronize_srcu(struct srcu_struct *ssp); 120 void synchronize_srcu_atomic(struct srcu_struct *ssp); 121 122 /** 123 * list_splice_init_srcu - splice an SRCU-protected list into an existing list 124 * @list: the SRCU-protected list to splice 125 * @head: the place in the existing list to splice the first list into 126 * @ssp: the srcu_struct that readers of @list use 127 * 128 * Like list_splice_init_rcu(), but wait for the SRCU readers of @list 129 * instead of RCU ones, so that none of them can follow a next pointer 130 * onto @head. 131 * 132 * Note that this function blocks, and that the caller must prevent any 133 * other updates to @list. 134 */ 135 static inline void list_splice_init_srcu(struct list_head *list, 136 struct list_head *head, 137 struct srcu_struct *ssp) 138 { 139 struct list_head *first = list->next; 140 struct list_head *last = list->prev; 141 struct list_head *next = head->next; 142 143 if (list_empty(list)) 144 return; 145 146 INIT_LIST_HEAD_RCU(list); 147 synchronize_srcu(ssp); 148 ASSERT_EXCLUSIVE_ACCESS(*first); 149 ASSERT_EXCLUSIVE_ACCESS(*last); 150 151 last->next = next; 152 rcu_assign_pointer(list_next_rcu(head), first); 153 first->prev = head; 154 next->prev = last; 155 } 156 157 #define SRCU_GET_STATE_COMPLETED 0x1 158 159 /** 160 * get_completed_synchronize_srcu - Return a pre-completed polled state cookie 161 * 162 * Returns a value that poll_state_synchronize_srcu() will always treat 163 * as a cookie whose grace period has already completed. 164 */ 165 static inline unsigned long get_completed_synchronize_srcu(void) 166 { 167 return SRCU_GET_STATE_COMPLETED; 168 } 169 170 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp); 171 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp); 172 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie); 173 174 // Maximum number of unsigned long values corresponding to 175 // not-yet-completed SRCU grace periods. 176 #define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2 177 178 /** 179 * same_state_synchronize_srcu - Are two old-state values identical? 180 * @oldstate1: First old-state value. 181 * @oldstate2: Second old-state value. 182 * 183 * The two old-state values must have been obtained from either 184 * get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or 185 * get_completed_synchronize_srcu(). Returns @true if the two values are 186 * identical and @false otherwise. This allows structures whose lifetimes 187 * are tracked by old-state values to push these values to a list header, 188 * allowing those structures to be slightly smaller. 189 */ 190 static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2) 191 { 192 return oldstate1 == oldstate2; 193 } 194 195 #ifdef CONFIG_NEED_SRCU_NMI_SAFE 196 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires_shared(ssp); 197 void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases_shared(ssp); 198 #else 199 static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) 200 __acquires_shared(ssp) 201 { 202 return __srcu_read_lock(ssp); 203 } 204 static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) 205 __releases_shared(ssp) 206 { 207 __srcu_read_unlock(ssp, idx); 208 } 209 #endif /* CONFIG_NEED_SRCU_NMI_SAFE */ 210 211 void srcu_init(void); 212 213 #ifdef CONFIG_DEBUG_LOCK_ALLOC 214 215 /** 216 * srcu_read_lock_held - might we be in SRCU read-side critical section? 217 * @ssp: The srcu_struct structure to check 218 * 219 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU 220 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, 221 * this assumes we are in an SRCU read-side critical section unless it can 222 * prove otherwise. 223 * 224 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot 225 * and while lockdep is disabled. 226 * 227 * Note that SRCU is based on its own statemachine and it doesn't 228 * relies on normal RCU, it can be called from the CPU which 229 * is in the idle loop from an RCU point of view or offline. 230 */ 231 static inline int srcu_read_lock_held(const struct srcu_struct *ssp) 232 { 233 if (!debug_lockdep_rcu_enabled()) 234 return 1; 235 return lock_is_held(&ssp->dep_map); 236 } 237 238 /* 239 * Annotations provide deadlock detection for SRCU. 240 * 241 * Similar to other lockdep annotations, except there is an additional 242 * srcu_lock_sync(), which is basically an empty *write*-side critical section, 243 * see lock_sync() for more information. 244 */ 245 246 /* Annotates a srcu_read_lock() */ 247 static inline void srcu_lock_acquire(struct lockdep_map *map) 248 { 249 lock_map_acquire_read(map); 250 } 251 252 /* Annotates a srcu_read_lock() */ 253 static inline void srcu_lock_release(struct lockdep_map *map) 254 { 255 lock_map_release(map); 256 } 257 258 /* Annotates a synchronize_srcu() */ 259 static inline void srcu_lock_sync(struct lockdep_map *map) 260 { 261 lock_map_sync(map); 262 } 263 264 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 265 266 static inline int srcu_read_lock_held(const struct srcu_struct *ssp) 267 { 268 return 1; 269 } 270 271 #define srcu_lock_acquire(m) do { } while (0) 272 #define srcu_lock_release(m) do { } while (0) 273 #define srcu_lock_sync(m) do { } while (0) 274 275 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 276 277 /* 278 * No-op helper to denote that ssp must be held. Because SRCU-protected pointers 279 * should still be marked with __rcu_guarded, and we do not want to mark them 280 * with __guarded_by(ssp) as it would complicate annotations for writers, we 281 * choose the following strategy: srcu_dereference_check() calls this helper 282 * that checks that the passed ssp is held, and then fake-acquires 'RCU'. 283 */ 284 static inline void __srcu_read_lock_must_hold(const struct srcu_struct *ssp) __must_hold_shared(ssp) { } 285 286 /** 287 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing 288 * @p: the pointer to fetch and protect for later dereferencing 289 * @ssp: pointer to the srcu_struct, which is used to check that we 290 * really are in an SRCU read-side critical section. 291 * @c: condition to check for update-side use 292 * 293 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side 294 * critical section will result in an RCU-lockdep splat, unless @c evaluates 295 * to 1. The @c argument will normally be a logical expression containing 296 * lockdep_is_held() calls. 297 */ 298 #define srcu_dereference_check(p, ssp, c) \ 299 ({ \ 300 __srcu_read_lock_must_hold(ssp); \ 301 __acquire_shared_ctx_lock(RCU); \ 302 __auto_type __v = __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ 303 (c) || srcu_read_lock_held(ssp), __rcu); \ 304 __release_shared_ctx_lock(RCU); \ 305 __v; \ 306 }) 307 308 /** 309 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing 310 * @p: the pointer to fetch and protect for later dereferencing 311 * @ssp: pointer to the srcu_struct, which is used to check that we 312 * really are in an SRCU read-side critical section. 313 * 314 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU 315 * is enabled, invoking this outside of an RCU read-side critical 316 * section will result in an RCU-lockdep splat. 317 */ 318 #define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0) 319 320 /** 321 * srcu_dereference_notrace - no tracing and no lockdep calls from here 322 * @p: the pointer to fetch and protect for later dereferencing 323 * @ssp: pointer to the srcu_struct, which is used to check that we 324 * really are in an SRCU read-side critical section. 325 */ 326 #define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1) 327 328 /** 329 * srcu_read_lock - register a new reader for an SRCU-protected structure. 330 * @ssp: srcu_struct in which to register the new reader. 331 * 332 * Enter an SRCU read-side critical section. Note that SRCU read-side 333 * critical sections may be nested. However, it is illegal to 334 * call anything that waits on an SRCU grace period for the same 335 * srcu_struct, whether directly or indirectly. Please note that 336 * one way to indirectly wait on an SRCU grace period is to acquire 337 * a mutex that is held elsewhere while calling synchronize_srcu() or 338 * synchronize_srcu_expedited(). 339 * 340 * The return value from srcu_read_lock() is guaranteed to be 341 * non-negative. This value must be passed unaltered to the matching 342 * srcu_read_unlock(). Note that srcu_read_lock() and the matching 343 * srcu_read_unlock() must occur in the same context, for example, it is 344 * illegal to invoke srcu_read_unlock() in an irq handler if the matching 345 * srcu_read_lock() was invoked in process context. Or, for that matter to 346 * invoke srcu_read_unlock() from one task and the matching srcu_read_lock() 347 * from another. 348 */ 349 static inline int srcu_read_lock(struct srcu_struct *ssp) 350 __acquires_shared(ssp) 351 { 352 int retval; 353 354 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 355 retval = __srcu_read_lock(ssp); 356 srcu_lock_acquire(&ssp->dep_map); 357 return retval; 358 } 359 360 /** 361 * srcu_read_lock_atomic - register a new reader promising an atomic section 362 * @ssp: srcu_struct in which to register the new reader. 363 * 364 * As srcu_read_lock(), but the caller promises that the read-side 365 * critical section never sleeps and never blocks on anything which 366 * may itself depend on memory allocation to make progress. Preemption 367 * is disabled for the duration, which both enforces that promise (any 368 * sleepable call in the section will splat on every configuration) 369 * and bounds the section so that the update side may spin rather 370 * than sleep when waiting for readers: see synchronize_srcu_atomic(). 371 * 372 * The lock and matching srcu_read_unlock_atomic() must be invoked on 373 * the same CPU, from the same context; passing the return value to 374 * another task is not permitted for this flavor. 375 */ 376 static inline int srcu_read_lock_atomic(struct srcu_struct *ssp) 377 __acquires_shared(ssp) 378 { 379 int retval; 380 381 preempt_disable(); 382 /* 383 * Arm might_sleep() to catch even a *potentially* sleeping call 384 * in the section, not just an actual schedule: the atomic-domain 385 * promise must hold on every path, contended or not. In hardirq, 386 * softirq, or NMI the annotation would land on the interrupted 387 * task, and can also result in data races against that task's 388 * own non_block_start()/non_block_end() invocations; it is also 389 * redundant there, so skip it. 390 */ 391 if (in_task()) 392 non_block_start(); 393 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); 394 retval = __srcu_read_lock(ssp); 395 srcu_lock_acquire(&ssp->dep_map); 396 return retval; 397 } 398 399 /** 400 * srcu_read_lock_fast - register a new reader for an SRCU-protected structure. 401 * @ssp: srcu_struct in which to register the new reader. 402 * 403 * Enter an SRCU read-side critical section, but for a light-weight 404 * smp_mb()-free reader. See srcu_read_lock() for more information. This 405 * function is NMI-safe, in a manner similar to srcu_read_lock_nmisafe(). 406 * 407 * For srcu_read_lock_fast() to be used on an srcu_struct structure, 408 * that structure must have been defined using either DEFINE_SRCU_FAST() 409 * or DEFINE_STATIC_SRCU_FAST() on the one hand or initialized with 410 * init_srcu_struct_fast() on the other. Such an srcu_struct structure 411 * cannot be passed to any non-fast variant of srcu_read_{,un}lock() or 412 * srcu_{down,up}_read(). In kernels built with CONFIG_PROVE_RCU=y, 413 * __srcu_check_read_flavor() will complain bitterly if you ignore this 414 * restriction. 415 * 416 * Grace-period auto-expediting is disabled for SRCU-fast srcu_struct 417 * structures because SRCU-fast expedited grace periods invoke 418 * synchronize_rcu_expedited(), IPIs and all. If you need expedited 419 * SRCU-fast grace periods, use synchronize_srcu_expedited(). 420 * 421 * The srcu_read_lock_fast() function can be invoked only from those 422 * contexts where RCU is watching, that is, from contexts where it would 423 * be legal to invoke rcu_read_lock(). Otherwise, lockdep will complain. 424 */ 425 static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires_shared(ssp) 426 __acquires_shared(ssp) 427 { 428 struct srcu_ctr __percpu *retval; 429 430 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast()."); 431 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 432 retval = __srcu_read_lock_fast(ssp); 433 rcu_try_lock_acquire(&ssp->dep_map); 434 return retval; 435 } 436 437 /** 438 * srcu_read_lock_fast_updown - register a new reader for an SRCU-fast-updown structure. 439 * @ssp: srcu_struct in which to register the new reader. 440 * 441 * Enter an SRCU read-side critical section, but for a light-weight 442 * smp_mb()-free reader. See srcu_read_lock() for more information. 443 * This function is compatible with srcu_down_read_fast(), but is not 444 * NMI-safe. 445 * 446 * For srcu_read_lock_fast_updown() to be used on an srcu_struct 447 * structure, that structure must have been defined using either 448 * DEFINE_SRCU_FAST_UPDOWN() or DEFINE_STATIC_SRCU_FAST_UPDOWN() on the one 449 * hand or initialized with init_srcu_struct_fast_updown() on the other. 450 * Such an srcu_struct structure cannot be passed to any non-fast-updown 451 * variant of srcu_read_{,un}lock() or srcu_{down,up}_read(). In kernels 452 * built with CONFIG_PROVE_RCU=y, __srcu_check_read_flavor() will complain 453 * bitterly if you ignore this * restriction. 454 * 455 * Grace-period auto-expediting is disabled for SRCU-fast-updown 456 * srcu_struct structures because SRCU-fast-updown expedited grace periods 457 * invoke synchronize_rcu_expedited(), IPIs and all. If you need expedited 458 * SRCU-fast-updown grace periods, use synchronize_srcu_expedited(). 459 * 460 * The srcu_read_lock_fast_updown() function can be invoked only from 461 * those contexts where RCU is watching, that is, from contexts where 462 * it would be legal to invoke rcu_read_lock(). Otherwise, lockdep will 463 * complain. 464 */ 465 static inline struct srcu_ctr __percpu *srcu_read_lock_fast_updown(struct srcu_struct *ssp) 466 __acquires_shared(ssp) 467 { 468 struct srcu_ctr __percpu *retval; 469 470 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast_updown()."); 471 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 472 retval = __srcu_read_lock_fast_updown(ssp); 473 rcu_try_lock_acquire(&ssp->dep_map); 474 return retval; 475 } 476 477 /* 478 * Used by tracing, cannot be traced and cannot call lockdep. 479 * See srcu_read_lock_fast() for more information. 480 */ 481 static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp) 482 __acquires_shared(ssp) 483 { 484 struct srcu_ctr __percpu *retval; 485 486 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 487 retval = __srcu_read_lock_fast(ssp); 488 return retval; 489 } 490 491 /** 492 * srcu_down_read_fast - register a new reader for an SRCU-protected structure. 493 * @ssp: srcu_struct in which to register the new reader. 494 * 495 * Enter a semaphore-like SRCU read-side critical section, but for 496 * a light-weight smp_mb()-free reader. See srcu_read_lock_fast() and 497 * srcu_down_read() for more information. 498 * 499 * The same srcu_struct may be used concurrently by srcu_down_read_fast() 500 * and srcu_read_lock_fast(). However, the same definition/initialization 501 * requirements called out for srcu_read_lock_fast_updown() apply. 502 */ 503 static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp) 504 { 505 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); 506 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast()."); 507 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 508 return __srcu_read_lock_fast_updown(ssp); 509 } 510 511 /** 512 * srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure. 513 * @ssp: srcu_struct in which to register the new reader. 514 * 515 * Enter an SRCU read-side critical section, but in an NMI-safe manner. 516 * See srcu_read_lock() for more information. 517 * 518 * If srcu_read_lock_nmisafe() is ever used on an srcu_struct structure, 519 * then none of the other flavors may be used, whether before, during, 520 * or after. 521 */ 522 static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) 523 __acquires_shared(ssp) 524 { 525 int retval; 526 527 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI); 528 retval = __srcu_read_lock_nmisafe(ssp); 529 rcu_try_lock_acquire(&ssp->dep_map); 530 return retval; 531 } 532 533 /* Used by tracing, cannot be traced and cannot invoke lockdep. */ 534 static inline notrace int 535 srcu_read_lock_notrace(struct srcu_struct *ssp) 536 __acquires_shared(ssp) 537 { 538 int retval; 539 540 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 541 retval = __srcu_read_lock(ssp); 542 return retval; 543 } 544 545 /** 546 * srcu_down_read - register a new reader for an SRCU-protected structure. 547 * @ssp: srcu_struct in which to register the new reader. 548 * 549 * Enter a semaphore-like SRCU read-side critical section. Note that 550 * SRCU read-side critical sections may be nested. However, it is 551 * illegal to call anything that waits on an SRCU grace period for the 552 * same srcu_struct, whether directly or indirectly. Please note that 553 * one way to indirectly wait on an SRCU grace period is to acquire 554 * a mutex that is held elsewhere while calling synchronize_srcu() or 555 * synchronize_srcu_expedited(). But if you want lockdep to help you 556 * keep this stuff straight, you should instead use srcu_read_lock(). 557 * 558 * The semaphore-like nature of srcu_down_read() means that the matching 559 * srcu_up_read() can be invoked from some other context, for example, 560 * from some other task or from an irq handler. However, neither 561 * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler. 562 * 563 * Calls to srcu_down_read() may be nested, similar to the manner in 564 * which calls to down_read() may be nested. The same srcu_struct may be 565 * used concurrently by srcu_down_read() and srcu_read_lock(). 566 */ 567 static inline int srcu_down_read(struct srcu_struct *ssp) 568 __acquires_shared(ssp) 569 { 570 WARN_ON_ONCE(in_nmi()); 571 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 572 return __srcu_read_lock(ssp); 573 } 574 575 /** 576 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. 577 * @ssp: srcu_struct in which to unregister the old reader. 578 * @idx: return value from corresponding srcu_read_lock(). 579 * 580 * Exit an SRCU read-side critical section. 581 */ 582 static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) 583 __releases_shared(ssp) 584 { 585 WARN_ON_ONCE(idx & ~0x1); 586 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 587 srcu_lock_release(&ssp->dep_map); 588 __srcu_read_unlock(ssp, idx); 589 } 590 591 /** 592 * srcu_read_unlock_atomic - unregister an atomic-section reader 593 * @ssp: srcu_struct from which to unregister the old reader. 594 * @idx: return value from corresponding srcu_read_lock_atomic(). 595 */ 596 static inline void srcu_read_unlock_atomic(struct srcu_struct *ssp, int idx) 597 __releases_shared(ssp) 598 { 599 WARN_ON_ONCE(idx & ~0x1); 600 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); 601 srcu_lock_release(&ssp->dep_map); 602 __srcu_read_unlock(ssp, idx); 603 if (in_task()) 604 non_block_end(); 605 preempt_enable(); 606 } 607 608 /** 609 * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. 610 * @ssp: srcu_struct in which to unregister the old reader. 611 * @scp: return value from corresponding srcu_read_lock_fast(). 612 * 613 * Exit a light-weight SRCU read-side critical section. 614 */ 615 static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) 616 __releases_shared(ssp) 617 { 618 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 619 srcu_lock_release(&ssp->dep_map); 620 __srcu_read_unlock_fast(ssp, scp); 621 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast()."); 622 } 623 624 /** 625 * srcu_read_unlock_fast_updown - unregister a old reader from an SRCU-fast-updown structure. 626 * @ssp: srcu_struct in which to unregister the old reader. 627 * @scp: return value from corresponding srcu_read_lock_fast_updown(). 628 * 629 * Exit an SRCU-fast-updown read-side critical section. 630 */ 631 static inline void 632 srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases_shared(ssp) 633 { 634 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 635 srcu_lock_release(&ssp->dep_map); 636 __srcu_read_unlock_fast_updown(ssp, scp); 637 RCU_LOCKDEP_WARN(!rcu_is_watching(), 638 "RCU must be watching srcu_read_unlock_fast_updown()."); 639 } 640 641 /* 642 * Used by tracing, cannot be traced and cannot call lockdep. 643 * See srcu_read_unlock_fast() for more information. 644 */ 645 static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp, 646 struct srcu_ctr __percpu *scp) __releases_shared(ssp) 647 { 648 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 649 __srcu_read_unlock_fast(ssp, scp); 650 } 651 652 /** 653 * srcu_up_read_fast - unregister a old reader from an SRCU-protected structure. 654 * @ssp: srcu_struct in which to unregister the old reader. 655 * @scp: return value from corresponding srcu_read_lock_fast(). 656 * 657 * Exit an SRCU read-side critical section, but not necessarily from 658 * the same context as the maching srcu_down_read_fast(). 659 */ 660 static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) 661 __releases_shared(ssp) 662 { 663 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); 664 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); 665 __srcu_read_unlock_fast_updown(ssp, scp); 666 RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_up_read_fast_updown()."); 667 } 668 669 /** 670 * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure. 671 * @ssp: srcu_struct in which to unregister the old reader. 672 * @idx: return value from corresponding srcu_read_lock_nmisafe(). 673 * 674 * Exit an SRCU read-side critical section, but in an NMI-safe manner. 675 */ 676 static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) 677 __releases_shared(ssp) 678 { 679 WARN_ON_ONCE(idx & ~0x1); 680 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI); 681 rcu_lock_release(&ssp->dep_map); 682 __srcu_read_unlock_nmisafe(ssp, idx); 683 } 684 685 /* Used by tracing, cannot be traced and cannot call lockdep. */ 686 static inline notrace void 687 srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases_shared(ssp) 688 { 689 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 690 __srcu_read_unlock(ssp, idx); 691 } 692 693 /** 694 * srcu_up_read - unregister a old reader from an SRCU-protected structure. 695 * @ssp: srcu_struct in which to unregister the old reader. 696 * @idx: return value from corresponding srcu_read_lock(). 697 * 698 * Exit an SRCU read-side critical section, but not necessarily from 699 * the same context as the maching srcu_down_read(). 700 */ 701 static inline void srcu_up_read(struct srcu_struct *ssp, int idx) 702 __releases_shared(ssp) 703 { 704 WARN_ON_ONCE(idx & ~0x1); 705 WARN_ON_ONCE(in_nmi()); 706 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); 707 __srcu_read_unlock(ssp, idx); 708 } 709 710 /** 711 * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock 712 * 713 * Converts the preceding srcu_read_unlock into a two-way memory barrier. 714 * 715 * Call this after srcu_read_unlock, to guarantee that all memory operations 716 * that occur after smp_mb__after_srcu_read_unlock will appear to happen after 717 * the preceding srcu_read_unlock. 718 */ 719 static inline void smp_mb__after_srcu_read_unlock(void) 720 { 721 /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */ 722 } 723 724 /** 725 * smp_mb__after_srcu_read_lock - ensure full ordering after srcu_read_lock 726 * 727 * Converts the preceding srcu_read_lock into a two-way memory barrier. 728 * 729 * Call this after srcu_read_lock, to guarantee that all memory operations 730 * that occur after smp_mb__after_srcu_read_lock will appear to happen after 731 * the preceding srcu_read_lock. 732 */ 733 static inline void smp_mb__after_srcu_read_lock(void) 734 { 735 /* __srcu_read_lock has smp_mb() internally so nothing to do here. */ 736 } 737 738 DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct, 739 _T->idx = srcu_read_lock(_T->lock), 740 srcu_read_unlock(_T->lock, _T->idx), 741 int idx) 742 DECLARE_LOCK_GUARD_1_ATTRS(srcu, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 743 #define class_srcu_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu, _T) 744 745 DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct, 746 _T->scp = srcu_read_lock_fast(_T->lock), 747 srcu_read_unlock_fast(_T->lock, _T->scp), 748 struct srcu_ctr __percpu *scp) 749 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 750 #define class_srcu_fast_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast, _T) 751 752 DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct, 753 _T->scp = srcu_read_lock_fast_notrace(_T->lock), 754 srcu_read_unlock_fast_notrace(_T->lock, _T->scp), 755 struct srcu_ctr __percpu *scp) 756 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 757 #define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T) 758 759 DEFINE_LOCK_GUARD_1(srcu_fast_updown, struct srcu_struct, 760 _T->scp = srcu_read_lock_fast_updown(_T->lock), 761 srcu_read_unlock_fast_updown(_T->lock, _T->scp), 762 struct srcu_ctr __percpu *scp) 763 DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_updown, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) 764 #define class_srcu_fast_updown_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_updown, _T) 765 766 #endif 767