1 /* SPDX-License-Identifier: GPL-2.0+ */ 2 /* 3 * RCU segmented callback lists, internal-to-rcu header file 4 * 5 * Copyright IBM Corporation, 2017 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 */ 9 10 #include <linux/rcu_segcblist.h> 11 12 /* Return number of callbacks in the specified callback list. */ 13 static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) 14 { 15 return READ_ONCE(rclp->len); 16 } 17 18 long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg); 19 20 /* Return number of callbacks in segmented callback list by summing seglen. */ 21 long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp); 22 23 void rcu_cblist_init(struct rcu_cblist *rclp); 24 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); 25 void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, 26 struct rcu_cblist *srclp, 27 struct rcu_head *rhp); 28 struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp); 29 30 /* 31 * Is the specified rcu_segcblist structure empty? 32 * 33 * But careful! The fact that the ->head field is NULL does not 34 * necessarily imply that there are no callbacks associated with 35 * this structure. When callbacks are being invoked, they are 36 * removed as a group. If callback invocation must be preempted, 37 * the remaining callbacks will be added back to the list. Either 38 * way, the counts are updated later. 39 * 40 * So it is often the case that rcu_segcblist_n_cbs() should be used 41 * instead. 42 */ 43 static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp) 44 { 45 return !READ_ONCE(rsclp->head); 46 } 47 48 /* Return number of callbacks in segmented callback list. */ 49 static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) 50 { 51 #ifdef CONFIG_RCU_NOCB_CPU 52 return atomic_long_read(&rsclp->len); 53 #else 54 return READ_ONCE(rsclp->len); 55 #endif 56 } 57 58 static inline void rcu_segcblist_set_flags(struct rcu_segcblist *rsclp, 59 int flags) 60 { 61 WRITE_ONCE(rsclp->flags, rsclp->flags | flags); 62 } 63 64 static inline void rcu_segcblist_clear_flags(struct rcu_segcblist *rsclp, 65 int flags) 66 { 67 WRITE_ONCE(rsclp->flags, rsclp->flags & ~flags); 68 } 69 70 static inline bool rcu_segcblist_test_flags(struct rcu_segcblist *rsclp, 71 int flags) 72 { 73 return READ_ONCE(rsclp->flags) & flags; 74 } 75 76 /* 77 * Is the specified rcu_segcblist enabled, for example, not corresponding 78 * to an offline CPU? 79 */ 80 static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) 81 { 82 return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED); 83 } 84 85 /* 86 * Is the specified rcu_segcblist NOCB offloaded (or in the middle of the 87 * [de]offloading process)? 88 */ 89 static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) 90 { 91 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && 92 rcu_segcblist_test_flags(rsclp, SEGCBLIST_OFFLOADED)) 93 return true; 94 95 return false; 96 } 97 98 /* 99 * Are all segments following the specified segment of the specified 100 * rcu_segcblist structure empty of callbacks? (The specified 101 * segment might well contain callbacks.) 102 */ 103 static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) 104 { 105 return !READ_ONCE(*READ_ONCE(rsclp->tails[seg])); 106 } 107 108 /* 109 * Is the specified segment of the specified rcu_segcblist structure 110 * empty of callbacks? 111 */ 112 static inline bool rcu_segcblist_segempty(struct rcu_segcblist *rsclp, int seg) 113 { 114 if (seg == RCU_DONE_TAIL) 115 return &rsclp->head == rsclp->tails[RCU_DONE_TAIL]; 116 return rsclp->tails[seg - 1] == rsclp->tails[seg]; 117 } 118 119 void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp); 120 void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v); 121 void rcu_segcblist_init(struct rcu_segcblist *rsclp); 122 void rcu_segcblist_disable(struct rcu_segcblist *rsclp); 123 void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload); 124 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); 125 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); 126 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); 127 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); 128 bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); 129 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, 130 struct rcu_head *rhp); 131 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, 132 struct rcu_head *rhp); 133 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, 134 struct rcu_cblist *rclp); 135 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, 136 struct rcu_cblist *rclp); 137 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, 138 struct rcu_cblist *rclp); 139 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, 140 struct rcu_cblist *rclp); 141 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, 142 struct rcu_cblist *rclp); 143 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); 144 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); 145 void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, 146 struct rcu_segcblist *src_rsclp); 147