1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/atomic.h 4 * 5 * Copyright (C) 1996 Russell King. 6 * Copyright (C) 2002 Deep Blue Solutions Ltd. 7 * Copyright (C) 2012 ARM Ltd. 8 */ 9 10 #ifndef __ASM_ATOMIC_LL_SC_H 11 #define __ASM_ATOMIC_LL_SC_H 12 13 #include <linux/stringify.h> 14 #include <asm/xwreg.h> 15 16 #ifndef CONFIG_CC_HAS_K_CONSTRAINT 17 #define K 18 #endif 19 20 /* 21 * AArch64 UP and SMP safe atomic ops. We use load exclusive and 22 * store exclusive to ensure that these are atomic. We may loop 23 * to ensure that the update happens. 24 */ 25 26 #define ATOMIC_OP(op, asm_op, constraint) \ 27 static __always_inline void \ 28 __ll_sc_atomic_##op(int i, atomic_t *v) \ 29 { \ 30 unsigned long tmp; \ 31 int result; \ 32 \ 33 asm volatile("// atomic_" #op "\n" \ 34 " prfm pstl1strm, %2\n" \ 35 "1: ldxr %w0, %2\n" \ 36 " " #asm_op " %w0, %w0, %w3\n" \ 37 " stxr %w1, %w0, %2\n" \ 38 " cbnz %w1, 1b\n" \ 39 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 40 : __stringify(constraint) "r" (i)); \ 41 } 42 43 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\ 44 static __always_inline int \ 45 __ll_sc_atomic_##op##_return##name(int i, atomic_t *v) \ 46 { \ 47 unsigned long tmp; \ 48 int result; \ 49 \ 50 asm volatile("// atomic_" #op "_return" #name "\n" \ 51 " prfm pstl1strm, %2\n" \ 52 "1: ld" #acq "xr %w0, %2\n" \ 53 " " #asm_op " %w0, %w0, %w3\n" \ 54 " st" #rel "xr %w1, %w0, %2\n" \ 55 " cbnz %w1, 1b\n" \ 56 " " #mb \ 57 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 58 : __stringify(constraint) "r" (i) \ 59 : cl); \ 60 \ 61 return result; \ 62 } 63 64 #define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \ 65 static __always_inline int \ 66 __ll_sc_atomic_fetch_##op##name(int i, atomic_t *v) \ 67 { \ 68 unsigned long tmp; \ 69 int val, result; \ 70 \ 71 asm volatile("// atomic_fetch_" #op #name "\n" \ 72 " prfm pstl1strm, %3\n" \ 73 "1: ld" #acq "xr %w0, %3\n" \ 74 " " #asm_op " %w1, %w0, %w4\n" \ 75 " st" #rel "xr %w2, %w1, %3\n" \ 76 " cbnz %w2, 1b\n" \ 77 " " #mb \ 78 : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ 79 : __stringify(constraint) "r" (i) \ 80 : cl); \ 81 \ 82 return result; \ 83 } 84 85 #define ATOMIC_OPS(...) \ 86 ATOMIC_OP(__VA_ARGS__) \ 87 ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)\ 88 ATOMIC_OP_RETURN(_relaxed, , , , , __VA_ARGS__)\ 89 ATOMIC_OP_RETURN(_acquire, , a, , "memory", __VA_ARGS__)\ 90 ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__)\ 91 ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ 92 ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ 93 ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ 94 ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) 95 96 ATOMIC_OPS(add, add, I) 97 ATOMIC_OPS(sub, sub, J) 98 99 #undef ATOMIC_OPS 100 #define ATOMIC_OPS(...) \ 101 ATOMIC_OP(__VA_ARGS__) \ 102 ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ 103 ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ 104 ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ 105 ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) 106 107 ATOMIC_OPS(and, and, K) 108 ATOMIC_OPS(or, orr, K) 109 ATOMIC_OPS(xor, eor, K) 110 /* 111 * GAS converts the mysterious and undocumented BIC (immediate) alias to 112 * an AND (immediate) instruction with the immediate inverted. We don't 113 * have a constraint for this, so fall back to register. 114 */ 115 ATOMIC_OPS(andnot, bic, ) 116 117 #undef ATOMIC_OPS 118 #undef ATOMIC_FETCH_OP 119 #undef ATOMIC_OP_RETURN 120 #undef ATOMIC_OP 121 122 #define ATOMIC64_OP(op, asm_op, constraint) \ 123 static __always_inline void \ 124 __ll_sc_atomic64_##op(s64 i, atomic64_t *v) \ 125 { \ 126 s64 result; \ 127 unsigned long tmp; \ 128 \ 129 asm volatile("// atomic64_" #op "\n" \ 130 " prfm pstl1strm, %2\n" \ 131 "1: ldxr %0, %2\n" \ 132 " " #asm_op " %0, %0, %3\n" \ 133 " stxr %w1, %0, %2\n" \ 134 " cbnz %w1, 1b" \ 135 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 136 : __stringify(constraint) "r" (i)); \ 137 } 138 139 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\ 140 static __always_inline long \ 141 __ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v) \ 142 { \ 143 s64 result; \ 144 unsigned long tmp; \ 145 \ 146 asm volatile("// atomic64_" #op "_return" #name "\n" \ 147 " prfm pstl1strm, %2\n" \ 148 "1: ld" #acq "xr %0, %2\n" \ 149 " " #asm_op " %0, %0, %3\n" \ 150 " st" #rel "xr %w1, %0, %2\n" \ 151 " cbnz %w1, 1b\n" \ 152 " " #mb \ 153 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 154 : __stringify(constraint) "r" (i) \ 155 : cl); \ 156 \ 157 return result; \ 158 } 159 160 #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\ 161 static __always_inline long \ 162 __ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v) \ 163 { \ 164 s64 result, val; \ 165 unsigned long tmp; \ 166 \ 167 asm volatile("// atomic64_fetch_" #op #name "\n" \ 168 " prfm pstl1strm, %3\n" \ 169 "1: ld" #acq "xr %0, %3\n" \ 170 " " #asm_op " %1, %0, %4\n" \ 171 " st" #rel "xr %w2, %1, %3\n" \ 172 " cbnz %w2, 1b\n" \ 173 " " #mb \ 174 : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ 175 : __stringify(constraint) "r" (i) \ 176 : cl); \ 177 \ 178 return result; \ 179 } 180 181 #define ATOMIC64_OPS(...) \ 182 ATOMIC64_OP(__VA_ARGS__) \ 183 ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__) \ 184 ATOMIC64_OP_RETURN(_relaxed,, , , , __VA_ARGS__) \ 185 ATOMIC64_OP_RETURN(_acquire,, a, , "memory", __VA_ARGS__) \ 186 ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__) \ 187 ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ 188 ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ 189 ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ 190 ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) 191 192 ATOMIC64_OPS(add, add, I) 193 ATOMIC64_OPS(sub, sub, J) 194 195 #undef ATOMIC64_OPS 196 #define ATOMIC64_OPS(...) \ 197 ATOMIC64_OP(__VA_ARGS__) \ 198 ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ 199 ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ 200 ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ 201 ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) 202 203 ATOMIC64_OPS(and, and, L) 204 ATOMIC64_OPS(or, orr, L) 205 ATOMIC64_OPS(xor, eor, L) 206 /* 207 * GAS converts the mysterious and undocumented BIC (immediate) alias to 208 * an AND (immediate) instruction with the immediate inverted. We don't 209 * have a constraint for this, so fall back to register. 210 */ 211 ATOMIC64_OPS(andnot, bic, ) 212 213 #undef ATOMIC64_OPS 214 #undef ATOMIC64_FETCH_OP 215 #undef ATOMIC64_OP_RETURN 216 #undef ATOMIC64_OP 217 218 static __always_inline s64 219 __ll_sc_atomic64_dec_if_positive(atomic64_t *v) 220 { 221 s64 result; 222 unsigned long tmp; 223 224 asm volatile("// atomic64_dec_if_positive\n" 225 " prfm pstl1strm, %2\n" 226 "1: ldxr %0, %2\n" 227 " subs %0, %0, #1\n" 228 " b.lt 2f\n" 229 " stlxr %w1, %0, %2\n" 230 " cbnz %w1, 1b\n" 231 " dmb ish\n" 232 "2:" 233 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) 234 : 235 : "cc", "memory"); 236 237 return result; 238 } 239 240 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint) \ 241 static __always_inline u##sz \ 242 __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr, \ 243 u##sz old, \ 244 u##sz new) \ 245 { \ 246 /* \ 247 * Sub-word sizes require zero extension so that EOR+CBNZ won't \ 248 * consume non-zero upper bits of the register containing "old".\ 249 */ \ 250 xwreg_t(w) cmpval = xwreg_zero_extend(old, w, sz); \ 251 unsigned long tmp; \ 252 u##sz oldval; \ 253 \ 254 asm volatile( \ 255 " prfm pstl1strm, %[v]\n" \ 256 "1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \ 257 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ 258 " cbnz %" #w "[tmp], 2f\n" \ 259 " st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \ 260 " cbnz %w[tmp], 1b\n" \ 261 " " #mb "\n" \ 262 "2:" \ 263 : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \ 264 [v] "+Q" (*(u##sz *)ptr) \ 265 : [old] __stringify(constraint) "r" (cmpval), \ 266 [new] "r" (new) \ 267 : cl); \ 268 \ 269 return oldval; \ 270 } 271 272 /* 273 * Earlier versions of GCC (no later than 8.1.0) appear to incorrectly 274 * handle the 'K' constraint for the value 4294967295 - thus we use no 275 * constraint for 32 bit operations. 276 */ 277 __CMPXCHG_CASE(w, b, , 8, , , , , K) 278 __CMPXCHG_CASE(w, h, , 16, , , , , K) 279 __CMPXCHG_CASE(w, , , 32, , , , , K) 280 __CMPXCHG_CASE(x, , , 64, , , , , L) 281 __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) 282 __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) 283 __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 284 __CMPXCHG_CASE(x, , acq_, 64, , a, , "memory", L) 285 __CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory", K) 286 __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) 287 __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) 288 __CMPXCHG_CASE(x, , rel_, 64, , , l, "memory", L) 289 __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) 290 __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) 291 __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 292 __CMPXCHG_CASE(x, , mb_, 64, dmb ish, , l, "memory", L) 293 294 #undef __CMPXCHG_CASE 295 296 union __u128_halves { 297 u128 full; 298 struct { 299 u64 low, high; 300 }; 301 }; 302 303 #define __CMPXCHG128(name, mb, rel, cl...) \ 304 static __always_inline u128 \ 305 __ll_sc__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \ 306 { \ 307 union __u128_halves r, o = { .full = (old) }, \ 308 n = { .full = (new) }; \ 309 unsigned int tmp; \ 310 \ 311 asm volatile("// __cmpxchg128" #name "\n" \ 312 " prfm pstl1strm, %[v]\n" \ 313 "1: ldxp %[rl], %[rh], %[v]\n" \ 314 " cmp %[rl], %[ol]\n" \ 315 " ccmp %[rh], %[oh], 0, eq\n" \ 316 " b.ne 2f\n" \ 317 " st" #rel "xp %w[tmp], %[nl], %[nh], %[v]\n" \ 318 " cbnz %w[tmp], 1b\n" \ 319 " " #mb "\n" \ 320 "2:" \ 321 : [v] "+Q" (*(u128 *)ptr), \ 322 [rl] "=&r" (r.low), [rh] "=&r" (r.high), \ 323 [tmp] "=&r" (tmp) \ 324 : [ol] "r" (o.low), [oh] "r" (o.high), \ 325 [nl] "r" (n.low), [nh] "r" (n.high) \ 326 : "cc", ##cl); \ 327 \ 328 return r.full; \ 329 } 330 331 __CMPXCHG128( , , ) 332 __CMPXCHG128(_mb, dmb ish, l, "memory") 333 334 #undef __CMPXCHG128 335 336 #undef K 337 338 #endif /* __ASM_ATOMIC_LL_SC_H */ 339