1 /* 2 * Variant of atomic_t specialized for reference counts. 3 * 4 * The interface matches the atomic_t interface (to aid in porting) but only 5 * provides the few functions one should use for reference counting. 6 * 7 * It differs in that the counter saturates at UINT_MAX and will not move once 8 * there. This avoids wrapping the counter and causing 'spurious' 9 * use-after-free issues. 10 * 11 * Memory ordering rules are slightly relaxed wrt regular atomic_t functions 12 * and provide only what is strictly required for refcounts. 13 * 14 * The increments are fully relaxed; these will not provide ordering. The 15 * rationale is that whatever is used to obtain the object we're increasing the 16 * reference count on will provide the ordering. For locked data structures, 17 * its the lock acquire, for RCU/lockless data structures its the dependent 18 * load. 19 * 20 * Do note that inc_not_zero() provides a control dependency which will order 21 * future stores against the inc, this ensures we'll never modify the object 22 * if we did not in fact acquire a reference. 23 * 24 * The decrements will provide release order, such that all the prior loads and 25 * stores will be issued before, it also provides a control dependency, which 26 * will order us against the subsequent free(). 27 * 28 * The control dependency is against the load of the cmpxchg (ll/sc) that 29 * succeeded. This means the stores aren't fully ordered, but this is fine 30 * because the 1->0 transition indicates no concurrency. 31 * 32 * Note that the allocator is responsible for ordering things between free() 33 * and alloc(). 34 * 35 */ 36 37 #include <linux/refcount.h> 38 #include <linux/bug.h> 39 40 #ifdef CONFIG_REFCOUNT_FULL 41 42 /** 43 * refcount_add_not_zero - add a value to a refcount unless it is 0 44 * @i: the value to add to the refcount 45 * @r: the refcount 46 * 47 * Will saturate at UINT_MAX and WARN. 48 * 49 * Provides no memory ordering, it is assumed the caller has guaranteed the 50 * object memory to be stable (RCU, etc.). It does provide a control dependency 51 * and thereby orders future stores. See the comment on top. 52 * 53 * Use of this function is not recommended for the normal reference counting 54 * use case in which references are taken and released one at a time. In these 55 * cases, refcount_inc(), or one of its variants, should instead be used to 56 * increment a reference count. 57 * 58 * Return: false if the passed refcount is 0, true otherwise 59 */ 60 bool refcount_add_not_zero(unsigned int i, refcount_t *r) 61 { 62 unsigned int new, val = atomic_read(&r->refs); 63 64 do { 65 if (!val) 66 return false; 67 68 if (unlikely(val == UINT_MAX)) 69 return true; 70 71 new = val + i; 72 if (new < val) 73 new = UINT_MAX; 74 75 } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); 76 77 WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n"); 78 79 return true; 80 } 81 EXPORT_SYMBOL(refcount_add_not_zero); 82 83 /** 84 * refcount_add - add a value to a refcount 85 * @i: the value to add to the refcount 86 * @r: the refcount 87 * 88 * Similar to atomic_add(), but will saturate at UINT_MAX and WARN. 89 * 90 * Provides no memory ordering, it is assumed the caller has guaranteed the 91 * object memory to be stable (RCU, etc.). It does provide a control dependency 92 * and thereby orders future stores. See the comment on top. 93 * 94 * Use of this function is not recommended for the normal reference counting 95 * use case in which references are taken and released one at a time. In these 96 * cases, refcount_inc(), or one of its variants, should instead be used to 97 * increment a reference count. 98 */ 99 void refcount_add(unsigned int i, refcount_t *r) 100 { 101 WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n"); 102 } 103 EXPORT_SYMBOL(refcount_add); 104 105 /** 106 * refcount_inc_not_zero - increment a refcount unless it is 0 107 * @r: the refcount to increment 108 * 109 * Similar to atomic_inc_not_zero(), but will saturate at UINT_MAX and WARN. 110 * 111 * Provides no memory ordering, it is assumed the caller has guaranteed the 112 * object memory to be stable (RCU, etc.). It does provide a control dependency 113 * and thereby orders future stores. See the comment on top. 114 * 115 * Return: true if the increment was successful, false otherwise 116 */ 117 bool refcount_inc_not_zero(refcount_t *r) 118 { 119 unsigned int new, val = atomic_read(&r->refs); 120 121 do { 122 new = val + 1; 123 124 if (!val) 125 return false; 126 127 if (unlikely(!new)) 128 return true; 129 130 } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); 131 132 WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n"); 133 134 return true; 135 } 136 EXPORT_SYMBOL(refcount_inc_not_zero); 137 138 /** 139 * refcount_inc - increment a refcount 140 * @r: the refcount to increment 141 * 142 * Similar to atomic_inc(), but will saturate at UINT_MAX and WARN. 143 * 144 * Provides no memory ordering, it is assumed the caller already has a 145 * reference on the object. 146 * 147 * Will WARN if the refcount is 0, as this represents a possible use-after-free 148 * condition. 149 */ 150 void refcount_inc(refcount_t *r) 151 { 152 WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n"); 153 } 154 EXPORT_SYMBOL(refcount_inc); 155 156 /** 157 * refcount_sub_and_test - subtract from a refcount and test if it is 0 158 * @i: amount to subtract from the refcount 159 * @r: the refcount 160 * 161 * Similar to atomic_dec_and_test(), but it will WARN, return false and 162 * ultimately leak on underflow and will fail to decrement when saturated 163 * at UINT_MAX. 164 * 165 * Provides release memory ordering, such that prior loads and stores are done 166 * before, and provides a control dependency such that free() must come after. 167 * See the comment on top. 168 * 169 * Use of this function is not recommended for the normal reference counting 170 * use case in which references are taken and released one at a time. In these 171 * cases, refcount_dec(), or one of its variants, should instead be used to 172 * decrement a reference count. 173 * 174 * Return: true if the resulting refcount is 0, false otherwise 175 */ 176 bool refcount_sub_and_test(unsigned int i, refcount_t *r) 177 { 178 unsigned int new, val = atomic_read(&r->refs); 179 180 do { 181 if (unlikely(val == UINT_MAX)) 182 return false; 183 184 new = val - i; 185 if (new > val) { 186 WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n"); 187 return false; 188 } 189 190 } while (!atomic_try_cmpxchg_release(&r->refs, &val, new)); 191 192 return !new; 193 } 194 EXPORT_SYMBOL(refcount_sub_and_test); 195 196 /** 197 * refcount_dec_and_test - decrement a refcount and test if it is 0 198 * @r: the refcount 199 * 200 * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to 201 * decrement when saturated at UINT_MAX. 202 * 203 * Provides release memory ordering, such that prior loads and stores are done 204 * before, and provides a control dependency such that free() must come after. 205 * See the comment on top. 206 * 207 * Return: true if the resulting refcount is 0, false otherwise 208 */ 209 bool refcount_dec_and_test(refcount_t *r) 210 { 211 return refcount_sub_and_test(1, r); 212 } 213 EXPORT_SYMBOL(refcount_dec_and_test); 214 215 /** 216 * refcount_dec - decrement a refcount 217 * @r: the refcount 218 * 219 * Similar to atomic_dec(), it will WARN on underflow and fail to decrement 220 * when saturated at UINT_MAX. 221 * 222 * Provides release memory ordering, such that prior loads and stores are done 223 * before. 224 */ 225 void refcount_dec(refcount_t *r) 226 { 227 WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n"); 228 } 229 EXPORT_SYMBOL(refcount_dec); 230 #endif /* CONFIG_REFCOUNT_FULL */ 231 232 /** 233 * refcount_dec_if_one - decrement a refcount if it is 1 234 * @r: the refcount 235 * 236 * No atomic_t counterpart, it attempts a 1 -> 0 transition and returns the 237 * success thereof. 238 * 239 * Like all decrement operations, it provides release memory order and provides 240 * a control dependency. 241 * 242 * It can be used like a try-delete operator; this explicit case is provided 243 * and not cmpxchg in generic, because that would allow implementing unsafe 244 * operations. 245 * 246 * Return: true if the resulting refcount is 0, false otherwise 247 */ 248 bool refcount_dec_if_one(refcount_t *r) 249 { 250 int val = 1; 251 252 return atomic_try_cmpxchg_release(&r->refs, &val, 0); 253 } 254 EXPORT_SYMBOL(refcount_dec_if_one); 255 256 /** 257 * refcount_dec_not_one - decrement a refcount if it is not 1 258 * @r: the refcount 259 * 260 * No atomic_t counterpart, it decrements unless the value is 1, in which case 261 * it will return false. 262 * 263 * Was often done like: atomic_add_unless(&var, -1, 1) 264 * 265 * Return: true if the decrement operation was successful, false otherwise 266 */ 267 bool refcount_dec_not_one(refcount_t *r) 268 { 269 unsigned int new, val = atomic_read(&r->refs); 270 271 do { 272 if (unlikely(val == UINT_MAX)) 273 return true; 274 275 if (val == 1) 276 return false; 277 278 new = val - 1; 279 if (new > val) { 280 WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n"); 281 return true; 282 } 283 284 } while (!atomic_try_cmpxchg_release(&r->refs, &val, new)); 285 286 return true; 287 } 288 EXPORT_SYMBOL(refcount_dec_not_one); 289 290 /** 291 * refcount_dec_and_mutex_lock - return holding mutex if able to decrement 292 * refcount to 0 293 * @r: the refcount 294 * @lock: the mutex to be locked 295 * 296 * Similar to atomic_dec_and_mutex_lock(), it will WARN on underflow and fail 297 * to decrement when saturated at UINT_MAX. 298 * 299 * Provides release memory ordering, such that prior loads and stores are done 300 * before, and provides a control dependency such that free() must come after. 301 * See the comment on top. 302 * 303 * Return: true and hold mutex if able to decrement refcount to 0, false 304 * otherwise 305 */ 306 bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock) 307 { 308 if (refcount_dec_not_one(r)) 309 return false; 310 311 mutex_lock(lock); 312 if (!refcount_dec_and_test(r)) { 313 mutex_unlock(lock); 314 return false; 315 } 316 317 return true; 318 } 319 EXPORT_SYMBOL(refcount_dec_and_mutex_lock); 320 321 /** 322 * refcount_dec_and_lock - return holding spinlock if able to decrement 323 * refcount to 0 324 * @r: the refcount 325 * @lock: the spinlock to be locked 326 * 327 * Similar to atomic_dec_and_lock(), it will WARN on underflow and fail to 328 * decrement when saturated at UINT_MAX. 329 * 330 * Provides release memory ordering, such that prior loads and stores are done 331 * before, and provides a control dependency such that free() must come after. 332 * See the comment on top. 333 * 334 * Return: true and hold spinlock if able to decrement refcount to 0, false 335 * otherwise 336 */ 337 bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock) 338 { 339 if (refcount_dec_not_one(r)) 340 return false; 341 342 spin_lock(lock); 343 if (!refcount_dec_and_test(r)) { 344 spin_unlock(lock); 345 return false; 346 } 347 348 return true; 349 } 350 EXPORT_SYMBOL(refcount_dec_and_lock); 351 352