1 #include <linux/export.h> 2 #include <linux/lockref.h> 3 4 #if USE_CMPXCHG_LOCKREF 5 6 /* 7 * Allow weakly-ordered memory architectures to provide barrier-less 8 * cmpxchg semantics for lockref updates. 9 */ 10 #ifndef cmpxchg64_relaxed 11 # define cmpxchg64_relaxed cmpxchg64 12 #endif 13 14 /* 15 * Note that the "cmpxchg()" reloads the "old" value for the 16 * failure case. 17 */ 18 #define CMPXCHG_LOOP(CODE, SUCCESS) do { \ 19 struct lockref old; \ 20 BUILD_BUG_ON(sizeof(old) != 8); \ 21 old.lock_count = READ_ONCE(lockref->lock_count); \ 22 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \ 23 struct lockref new = old, prev = old; \ 24 CODE \ 25 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \ 26 old.lock_count, \ 27 new.lock_count); \ 28 if (likely(old.lock_count == prev.lock_count)) { \ 29 SUCCESS; \ 30 } \ 31 cpu_relax_lowlatency(); \ 32 } \ 33 } while (0) 34 35 #else 36 37 #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0) 38 39 #endif 40 41 /** 42 * lockref_get - Increments reference count unconditionally 43 * @lockref: pointer to lockref structure 44 * 45 * This operation is only valid if you already hold a reference 46 * to the object, so you know the count cannot be zero. 47 */ 48 void lockref_get(struct lockref *lockref) 49 { 50 CMPXCHG_LOOP( 51 new.count++; 52 , 53 return; 54 ); 55 56 spin_lock(&lockref->lock); 57 lockref->count++; 58 spin_unlock(&lockref->lock); 59 } 60 EXPORT_SYMBOL(lockref_get); 61 62 /** 63 * lockref_get_not_zero - Increments count unless the count is 0 or dead 64 * @lockref: pointer to lockref structure 65 * Return: 1 if count updated successfully or 0 if count was zero 66 */ 67 int lockref_get_not_zero(struct lockref *lockref) 68 { 69 int retval; 70 71 CMPXCHG_LOOP( 72 new.count++; 73 if (old.count <= 0) 74 return 0; 75 , 76 return 1; 77 ); 78 79 spin_lock(&lockref->lock); 80 retval = 0; 81 if (lockref->count > 0) { 82 lockref->count++; 83 retval = 1; 84 } 85 spin_unlock(&lockref->lock); 86 return retval; 87 } 88 EXPORT_SYMBOL(lockref_get_not_zero); 89 90 /** 91 * lockref_get_or_lock - Increments count unless the count is 0 or dead 92 * @lockref: pointer to lockref structure 93 * Return: 1 if count updated successfully or 0 if count was zero 94 * and we got the lock instead. 95 */ 96 int lockref_get_or_lock(struct lockref *lockref) 97 { 98 CMPXCHG_LOOP( 99 new.count++; 100 if (old.count <= 0) 101 break; 102 , 103 return 1; 104 ); 105 106 spin_lock(&lockref->lock); 107 if (lockref->count <= 0) 108 return 0; 109 lockref->count++; 110 spin_unlock(&lockref->lock); 111 return 1; 112 } 113 EXPORT_SYMBOL(lockref_get_or_lock); 114 115 /** 116 * lockref_put_return - Decrement reference count if possible 117 * @lockref: pointer to lockref structure 118 * 119 * Decrement the reference count and return the new value. 120 * If the lockref was dead or locked, return an error. 121 */ 122 int lockref_put_return(struct lockref *lockref) 123 { 124 CMPXCHG_LOOP( 125 new.count--; 126 if (old.count <= 0) 127 return -1; 128 , 129 return new.count; 130 ); 131 return -1; 132 } 133 EXPORT_SYMBOL(lockref_put_return); 134 135 /** 136 * lockref_put_or_lock - decrements count unless count <= 1 before decrement 137 * @lockref: pointer to lockref structure 138 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken 139 */ 140 int lockref_put_or_lock(struct lockref *lockref) 141 { 142 CMPXCHG_LOOP( 143 new.count--; 144 if (old.count <= 1) 145 break; 146 , 147 return 1; 148 ); 149 150 spin_lock(&lockref->lock); 151 if (lockref->count <= 1) 152 return 0; 153 lockref->count--; 154 spin_unlock(&lockref->lock); 155 return 1; 156 } 157 EXPORT_SYMBOL(lockref_put_or_lock); 158 159 /** 160 * lockref_mark_dead - mark lockref dead 161 * @lockref: pointer to lockref structure 162 */ 163 void lockref_mark_dead(struct lockref *lockref) 164 { 165 assert_spin_locked(&lockref->lock); 166 lockref->count = -128; 167 } 168 EXPORT_SYMBOL(lockref_mark_dead); 169 170 /** 171 * lockref_get_not_dead - Increments count unless the ref is dead 172 * @lockref: pointer to lockref structure 173 * Return: 1 if count updated successfully or 0 if lockref was dead 174 */ 175 int lockref_get_not_dead(struct lockref *lockref) 176 { 177 int retval; 178 179 CMPXCHG_LOOP( 180 new.count++; 181 if (old.count < 0) 182 return 0; 183 , 184 return 1; 185 ); 186 187 spin_lock(&lockref->lock); 188 retval = 0; 189 if (lockref->count >= 0) { 190 lockref->count++; 191 retval = 1; 192 } 193 spin_unlock(&lockref->lock); 194 return retval; 195 } 196 EXPORT_SYMBOL(lockref_get_not_dead); 197