1 /* 2 * Variant of atomic_t specialized for reference counts. 3 * 4 * The interface matches the atomic_t interface (to aid in porting) but only 5 * provides the few functions one should use for reference counting. 6 * 7 * It differs in that the counter saturates at UINT_MAX and will not move once 8 * there. This avoids wrapping the counter and causing 'spurious' 9 * use-after-free issues. 10 * 11 * Memory ordering rules are slightly relaxed wrt regular atomic_t functions 12 * and provide only what is strictly required for refcounts. 13 * 14 * The increments are fully relaxed; these will not provide ordering. The 15 * rationale is that whatever is used to obtain the object we're increasing the 16 * reference count on will provide the ordering. For locked data structures, 17 * its the lock acquire, for RCU/lockless data structures its the dependent 18 * load. 19 * 20 * Do note that inc_not_zero() provides a control dependency which will order 21 * future stores against the inc, this ensures we'll never modify the object 22 * if we did not in fact acquire a reference. 23 * 24 * The decrements will provide release order, such that all the prior loads and 25 * stores will be issued before, it also provides a control dependency, which 26 * will order us against the subsequent free(). 27 * 28 * The control dependency is against the load of the cmpxchg (ll/sc) that 29 * succeeded. This means the stores aren't fully ordered, but this is fine 30 * because the 1->0 transition indicates no concurrency. 31 * 32 * Note that the allocator is responsible for ordering things between free() 33 * and alloc(). 34 * 35 */ 36 37 #include <linux/refcount.h> 38 #include <linux/bug.h> 39 40 /** 41 * refcount_add_not_zero - add a value to a refcount unless it is 0 42 * @i: the value to add to the refcount 43 * @r: the refcount 44 * 45 * Will saturate at UINT_MAX and WARN. 46 * 47 * Provides no memory ordering, it is assumed the caller has guaranteed the 48 * object memory to be stable (RCU, etc.). It does provide a control dependency 49 * and thereby orders future stores. See the comment on top. 50 * 51 * Use of this function is not recommended for the normal reference counting 52 * use case in which references are taken and released one at a time. In these 53 * cases, refcount_inc(), or one of its variants, should instead be used to 54 * increment a reference count. 55 * 56 * Return: false if the passed refcount is 0, true otherwise 57 */ 58 bool refcount_add_not_zero(unsigned int i, refcount_t *r) 59 { 60 unsigned int new, val = atomic_read(&r->refs); 61 62 do { 63 if (!val) 64 return false; 65 66 if (unlikely(val == UINT_MAX)) 67 return true; 68 69 new = val + i; 70 if (new < val) 71 new = UINT_MAX; 72 73 } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); 74 75 WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n"); 76 77 return true; 78 } 79 EXPORT_SYMBOL(refcount_add_not_zero); 80 81 /** 82 * refcount_add - add a value to a refcount 83 * @i: the value to add to the refcount 84 * @r: the refcount 85 * 86 * Similar to atomic_add(), but will saturate at UINT_MAX and WARN. 87 * 88 * Provides no memory ordering, it is assumed the caller has guaranteed the 89 * object memory to be stable (RCU, etc.). It does provide a control dependency 90 * and thereby orders future stores. See the comment on top. 91 * 92 * Use of this function is not recommended for the normal reference counting 93 * use case in which references are taken and released one at a time. In these 94 * cases, refcount_inc(), or one of its variants, should instead be used to 95 * increment a reference count. 96 */ 97 void refcount_add(unsigned int i, refcount_t *r) 98 { 99 WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n"); 100 } 101 EXPORT_SYMBOL(refcount_add); 102 103 /** 104 * refcount_inc_not_zero - increment a refcount unless it is 0 105 * @r: the refcount to increment 106 * 107 * Similar to atomic_inc_not_zero(), but will saturate at UINT_MAX and WARN. 108 * 109 * Provides no memory ordering, it is assumed the caller has guaranteed the 110 * object memory to be stable (RCU, etc.). It does provide a control dependency 111 * and thereby orders future stores. See the comment on top. 112 * 113 * Return: true if the increment was successful, false otherwise 114 */ 115 bool refcount_inc_not_zero(refcount_t *r) 116 { 117 unsigned int new, val = atomic_read(&r->refs); 118 119 do { 120 new = val + 1; 121 122 if (!val) 123 return false; 124 125 if (unlikely(!new)) 126 return true; 127 128 } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new)); 129 130 WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n"); 131 132 return true; 133 } 134 EXPORT_SYMBOL(refcount_inc_not_zero); 135 136 /** 137 * refcount_inc - increment a refcount 138 * @r: the refcount to increment 139 * 140 * Similar to atomic_inc(), but will saturate at UINT_MAX and WARN. 141 * 142 * Provides no memory ordering, it is assumed the caller already has a 143 * reference on the object. 144 * 145 * Will WARN if the refcount is 0, as this represents a possible use-after-free 146 * condition. 147 */ 148 void refcount_inc(refcount_t *r) 149 { 150 WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n"); 151 } 152 EXPORT_SYMBOL(refcount_inc); 153 154 /** 155 * refcount_sub_and_test - subtract from a refcount and test if it is 0 156 * @i: amount to subtract from the refcount 157 * @r: the refcount 158 * 159 * Similar to atomic_dec_and_test(), but it will WARN, return false and 160 * ultimately leak on underflow and will fail to decrement when saturated 161 * at UINT_MAX. 162 * 163 * Provides release memory ordering, such that prior loads and stores are done 164 * before, and provides a control dependency such that free() must come after. 165 * See the comment on top. 166 * 167 * Use of this function is not recommended for the normal reference counting 168 * use case in which references are taken and released one at a time. In these 169 * cases, refcount_dec(), or one of its variants, should instead be used to 170 * decrement a reference count. 171 * 172 * Return: true if the resulting refcount is 0, false otherwise 173 */ 174 bool refcount_sub_and_test(unsigned int i, refcount_t *r) 175 { 176 unsigned int new, val = atomic_read(&r->refs); 177 178 do { 179 if (unlikely(val == UINT_MAX)) 180 return false; 181 182 new = val - i; 183 if (new > val) { 184 WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n"); 185 return false; 186 } 187 188 } while (!atomic_try_cmpxchg_release(&r->refs, &val, new)); 189 190 return !new; 191 } 192 EXPORT_SYMBOL(refcount_sub_and_test); 193 194 /** 195 * refcount_dec_and_test - decrement a refcount and test if it is 0 196 * @r: the refcount 197 * 198 * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to 199 * decrement when saturated at UINT_MAX. 200 * 201 * Provides release memory ordering, such that prior loads and stores are done 202 * before, and provides a control dependency such that free() must come after. 203 * See the comment on top. 204 * 205 * Return: true if the resulting refcount is 0, false otherwise 206 */ 207 bool refcount_dec_and_test(refcount_t *r) 208 { 209 return refcount_sub_and_test(1, r); 210 } 211 EXPORT_SYMBOL(refcount_dec_and_test); 212 213 /** 214 * refcount_dec - decrement a refcount 215 * @r: the refcount 216 * 217 * Similar to atomic_dec(), it will WARN on underflow and fail to decrement 218 * when saturated at UINT_MAX. 219 * 220 * Provides release memory ordering, such that prior loads and stores are done 221 * before. 222 */ 223 void refcount_dec(refcount_t *r) 224 { 225 WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n"); 226 } 227 EXPORT_SYMBOL(refcount_dec); 228 229 /** 230 * refcount_dec_if_one - decrement a refcount if it is 1 231 * @r: the refcount 232 * 233 * No atomic_t counterpart, it attempts a 1 -> 0 transition and returns the 234 * success thereof. 235 * 236 * Like all decrement operations, it provides release memory order and provides 237 * a control dependency. 238 * 239 * It can be used like a try-delete operator; this explicit case is provided 240 * and not cmpxchg in generic, because that would allow implementing unsafe 241 * operations. 242 * 243 * Return: true if the resulting refcount is 0, false otherwise 244 */ 245 bool refcount_dec_if_one(refcount_t *r) 246 { 247 int val = 1; 248 249 return atomic_try_cmpxchg_release(&r->refs, &val, 0); 250 } 251 EXPORT_SYMBOL(refcount_dec_if_one); 252 253 /** 254 * refcount_dec_not_one - decrement a refcount if it is not 1 255 * @r: the refcount 256 * 257 * No atomic_t counterpart, it decrements unless the value is 1, in which case 258 * it will return false. 259 * 260 * Was often done like: atomic_add_unless(&var, -1, 1) 261 * 262 * Return: true if the decrement operation was successful, false otherwise 263 */ 264 bool refcount_dec_not_one(refcount_t *r) 265 { 266 unsigned int new, val = atomic_read(&r->refs); 267 268 do { 269 if (unlikely(val == UINT_MAX)) 270 return true; 271 272 if (val == 1) 273 return false; 274 275 new = val - 1; 276 if (new > val) { 277 WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n"); 278 return true; 279 } 280 281 } while (!atomic_try_cmpxchg_release(&r->refs, &val, new)); 282 283 return true; 284 } 285 EXPORT_SYMBOL(refcount_dec_not_one); 286 287 /** 288 * refcount_dec_and_mutex_lock - return holding mutex if able to decrement 289 * refcount to 0 290 * @r: the refcount 291 * @lock: the mutex to be locked 292 * 293 * Similar to atomic_dec_and_mutex_lock(), it will WARN on underflow and fail 294 * to decrement when saturated at UINT_MAX. 295 * 296 * Provides release memory ordering, such that prior loads and stores are done 297 * before, and provides a control dependency such that free() must come after. 298 * See the comment on top. 299 * 300 * Return: true and hold mutex if able to decrement refcount to 0, false 301 * otherwise 302 */ 303 bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock) 304 { 305 if (refcount_dec_not_one(r)) 306 return false; 307 308 mutex_lock(lock); 309 if (!refcount_dec_and_test(r)) { 310 mutex_unlock(lock); 311 return false; 312 } 313 314 return true; 315 } 316 EXPORT_SYMBOL(refcount_dec_and_mutex_lock); 317 318 /** 319 * refcount_dec_and_lock - return holding spinlock if able to decrement 320 * refcount to 0 321 * @r: the refcount 322 * @lock: the spinlock to be locked 323 * 324 * Similar to atomic_dec_and_lock(), it will WARN on underflow and fail to 325 * decrement when saturated at UINT_MAX. 326 * 327 * Provides release memory ordering, such that prior loads and stores are done 328 * before, and provides a control dependency such that free() must come after. 329 * See the comment on top. 330 * 331 * Return: true and hold spinlock if able to decrement refcount to 0, false 332 * otherwise 333 */ 334 bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock) 335 { 336 if (refcount_dec_not_one(r)) 337 return false; 338 339 spin_lock(lock); 340 if (!refcount_dec_and_test(r)) { 341 spin_unlock(lock); 342 return false; 343 } 344 345 return true; 346 } 347 EXPORT_SYMBOL(refcount_dec_and_lock); 348 349