1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/atomic.h 4 * 5 * Copyright (C) 1996 Russell King. 6 * Copyright (C) 2002 Deep Blue Solutions Ltd. 7 * Copyright (C) 2012 ARM Ltd. 8 */ 9 10 #ifndef __ASM_ATOMIC_LL_SC_H 11 #define __ASM_ATOMIC_LL_SC_H 12 13 #include <linux/stringify.h> 14 15 #ifndef CONFIG_CC_HAS_K_CONSTRAINT 16 #define K 17 #endif 18 19 /* 20 * AArch64 UP and SMP safe atomic ops. We use load exclusive and 21 * store exclusive to ensure that these are atomic. We may loop 22 * to ensure that the update happens. 23 */ 24 25 #define ATOMIC_OP(op, asm_op, constraint) \ 26 static __always_inline void \ 27 __ll_sc_atomic_##op(int i, atomic_t *v) \ 28 { \ 29 unsigned long tmp; \ 30 int result; \ 31 \ 32 asm volatile("// atomic_" #op "\n" \ 33 " prfm pstl1strm, %2\n" \ 34 "1: ldxr %w0, %2\n" \ 35 " " #asm_op " %w0, %w0, %w3\n" \ 36 " stxr %w1, %w0, %2\n" \ 37 " cbnz %w1, 1b\n" \ 38 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 39 : __stringify(constraint) "r" (i)); \ 40 } 41 42 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\ 43 static __always_inline int \ 44 __ll_sc_atomic_##op##_return##name(int i, atomic_t *v) \ 45 { \ 46 unsigned long tmp; \ 47 int result; \ 48 \ 49 asm volatile("// atomic_" #op "_return" #name "\n" \ 50 " prfm pstl1strm, %2\n" \ 51 "1: ld" #acq "xr %w0, %2\n" \ 52 " " #asm_op " %w0, %w0, %w3\n" \ 53 " st" #rel "xr %w1, %w0, %2\n" \ 54 " cbnz %w1, 1b\n" \ 55 " " #mb \ 56 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 57 : __stringify(constraint) "r" (i) \ 58 : cl); \ 59 \ 60 return result; \ 61 } 62 63 #define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \ 64 static __always_inline int \ 65 __ll_sc_atomic_fetch_##op##name(int i, atomic_t *v) \ 66 { \ 67 unsigned long tmp; \ 68 int val, result; \ 69 \ 70 asm volatile("// atomic_fetch_" #op #name "\n" \ 71 " prfm pstl1strm, %3\n" \ 72 "1: ld" #acq "xr %w0, %3\n" \ 73 " " #asm_op " %w1, %w0, %w4\n" \ 74 " st" #rel "xr %w2, %w1, %3\n" \ 75 " cbnz %w2, 1b\n" \ 76 " " #mb \ 77 : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ 78 : __stringify(constraint) "r" (i) \ 79 : cl); \ 80 \ 81 return result; \ 82 } 83 84 #define ATOMIC_OPS(...) \ 85 ATOMIC_OP(__VA_ARGS__) \ 86 ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)\ 87 ATOMIC_OP_RETURN(_relaxed, , , , , __VA_ARGS__)\ 88 ATOMIC_OP_RETURN(_acquire, , a, , "memory", __VA_ARGS__)\ 89 ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__)\ 90 ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ 91 ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ 92 ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ 93 ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) 94 95 ATOMIC_OPS(add, add, I) 96 ATOMIC_OPS(sub, sub, J) 97 98 #undef ATOMIC_OPS 99 #define ATOMIC_OPS(...) \ 100 ATOMIC_OP(__VA_ARGS__) \ 101 ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ 102 ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ 103 ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ 104 ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) 105 106 ATOMIC_OPS(and, and, K) 107 ATOMIC_OPS(or, orr, K) 108 ATOMIC_OPS(xor, eor, K) 109 /* 110 * GAS converts the mysterious and undocumented BIC (immediate) alias to 111 * an AND (immediate) instruction with the immediate inverted. We don't 112 * have a constraint for this, so fall back to register. 113 */ 114 ATOMIC_OPS(andnot, bic, ) 115 116 #undef ATOMIC_OPS 117 #undef ATOMIC_FETCH_OP 118 #undef ATOMIC_OP_RETURN 119 #undef ATOMIC_OP 120 121 #define ATOMIC64_OP(op, asm_op, constraint) \ 122 static __always_inline void \ 123 __ll_sc_atomic64_##op(s64 i, atomic64_t *v) \ 124 { \ 125 s64 result; \ 126 unsigned long tmp; \ 127 \ 128 asm volatile("// atomic64_" #op "\n" \ 129 " prfm pstl1strm, %2\n" \ 130 "1: ldxr %0, %2\n" \ 131 " " #asm_op " %0, %0, %3\n" \ 132 " stxr %w1, %0, %2\n" \ 133 " cbnz %w1, 1b" \ 134 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 135 : __stringify(constraint) "r" (i)); \ 136 } 137 138 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\ 139 static __always_inline long \ 140 __ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v) \ 141 { \ 142 s64 result; \ 143 unsigned long tmp; \ 144 \ 145 asm volatile("// atomic64_" #op "_return" #name "\n" \ 146 " prfm pstl1strm, %2\n" \ 147 "1: ld" #acq "xr %0, %2\n" \ 148 " " #asm_op " %0, %0, %3\n" \ 149 " st" #rel "xr %w1, %0, %2\n" \ 150 " cbnz %w1, 1b\n" \ 151 " " #mb \ 152 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 153 : __stringify(constraint) "r" (i) \ 154 : cl); \ 155 \ 156 return result; \ 157 } 158 159 #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\ 160 static __always_inline long \ 161 __ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v) \ 162 { \ 163 s64 result, val; \ 164 unsigned long tmp; \ 165 \ 166 asm volatile("// atomic64_fetch_" #op #name "\n" \ 167 " prfm pstl1strm, %3\n" \ 168 "1: ld" #acq "xr %0, %3\n" \ 169 " " #asm_op " %1, %0, %4\n" \ 170 " st" #rel "xr %w2, %1, %3\n" \ 171 " cbnz %w2, 1b\n" \ 172 " " #mb \ 173 : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ 174 : __stringify(constraint) "r" (i) \ 175 : cl); \ 176 \ 177 return result; \ 178 } 179 180 #define ATOMIC64_OPS(...) \ 181 ATOMIC64_OP(__VA_ARGS__) \ 182 ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__) \ 183 ATOMIC64_OP_RETURN(_relaxed,, , , , __VA_ARGS__) \ 184 ATOMIC64_OP_RETURN(_acquire,, a, , "memory", __VA_ARGS__) \ 185 ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__) \ 186 ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ 187 ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ 188 ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ 189 ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) 190 191 ATOMIC64_OPS(add, add, I) 192 ATOMIC64_OPS(sub, sub, J) 193 194 #undef ATOMIC64_OPS 195 #define ATOMIC64_OPS(...) \ 196 ATOMIC64_OP(__VA_ARGS__) \ 197 ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ 198 ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ 199 ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ 200 ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) 201 202 ATOMIC64_OPS(and, and, L) 203 ATOMIC64_OPS(or, orr, L) 204 ATOMIC64_OPS(xor, eor, L) 205 /* 206 * GAS converts the mysterious and undocumented BIC (immediate) alias to 207 * an AND (immediate) instruction with the immediate inverted. We don't 208 * have a constraint for this, so fall back to register. 209 */ 210 ATOMIC64_OPS(andnot, bic, ) 211 212 #undef ATOMIC64_OPS 213 #undef ATOMIC64_FETCH_OP 214 #undef ATOMIC64_OP_RETURN 215 #undef ATOMIC64_OP 216 217 static __always_inline s64 218 __ll_sc_atomic64_dec_if_positive(atomic64_t *v) 219 { 220 s64 result; 221 unsigned long tmp; 222 223 asm volatile("// atomic64_dec_if_positive\n" 224 " prfm pstl1strm, %2\n" 225 "1: ldxr %0, %2\n" 226 " subs %0, %0, #1\n" 227 " b.lt 2f\n" 228 " stlxr %w1, %0, %2\n" 229 " cbnz %w1, 1b\n" 230 " dmb ish\n" 231 "2:" 232 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) 233 : 234 : "cc", "memory"); 235 236 return result; 237 } 238 239 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint) \ 240 static __always_inline u##sz \ 241 __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr, \ 242 unsigned long old, \ 243 u##sz new) \ 244 { \ 245 unsigned long tmp; \ 246 u##sz oldval; \ 247 \ 248 /* \ 249 * Sub-word sizes require explicit casting so that the compare \ 250 * part of the cmpxchg doesn't end up interpreting non-zero \ 251 * upper bits of the register containing "old". \ 252 */ \ 253 if (sz < 32) \ 254 old = (u##sz)old; \ 255 \ 256 asm volatile( \ 257 " prfm pstl1strm, %[v]\n" \ 258 "1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \ 259 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ 260 " cbnz %" #w "[tmp], 2f\n" \ 261 " st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \ 262 " cbnz %w[tmp], 1b\n" \ 263 " " #mb "\n" \ 264 "2:" \ 265 : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \ 266 [v] "+Q" (*(u##sz *)ptr) \ 267 : [old] __stringify(constraint) "r" (old), [new] "r" (new) \ 268 : cl); \ 269 \ 270 return oldval; \ 271 } 272 273 /* 274 * Earlier versions of GCC (no later than 8.1.0) appear to incorrectly 275 * handle the 'K' constraint for the value 4294967295 - thus we use no 276 * constraint for 32 bit operations. 277 */ 278 __CMPXCHG_CASE(w, b, , 8, , , , , K) 279 __CMPXCHG_CASE(w, h, , 16, , , , , K) 280 __CMPXCHG_CASE(w, , , 32, , , , , K) 281 __CMPXCHG_CASE( , , , 64, , , , , L) 282 __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) 283 __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) 284 __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 285 __CMPXCHG_CASE( , , acq_, 64, , a, , "memory", L) 286 __CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory", K) 287 __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) 288 __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) 289 __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) 290 __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) 291 __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) 292 __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 293 __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 294 295 #undef __CMPXCHG_CASE 296 297 union __u128_halves { 298 u128 full; 299 struct { 300 u64 low, high; 301 }; 302 }; 303 304 #define __CMPXCHG128(name, mb, rel, cl...) \ 305 static __always_inline u128 \ 306 __ll_sc__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \ 307 { \ 308 union __u128_halves r, o = { .full = (old) }, \ 309 n = { .full = (new) }; \ 310 unsigned int tmp; \ 311 \ 312 asm volatile("// __cmpxchg128" #name "\n" \ 313 " prfm pstl1strm, %[v]\n" \ 314 "1: ldxp %[rl], %[rh], %[v]\n" \ 315 " cmp %[rl], %[ol]\n" \ 316 " ccmp %[rh], %[oh], 0, eq\n" \ 317 " b.ne 2f\n" \ 318 " st" #rel "xp %w[tmp], %[nl], %[nh], %[v]\n" \ 319 " cbnz %w[tmp], 1b\n" \ 320 " " #mb "\n" \ 321 "2:" \ 322 : [v] "+Q" (*(u128 *)ptr), \ 323 [rl] "=&r" (r.low), [rh] "=&r" (r.high), \ 324 [tmp] "=&r" (tmp) \ 325 : [ol] "r" (o.low), [oh] "r" (o.high), \ 326 [nl] "r" (n.low), [nh] "r" (n.high) \ 327 : "cc", ##cl); \ 328 \ 329 return r.full; \ 330 } 331 332 __CMPXCHG128( , , ) 333 __CMPXCHG128(_mb, dmb ish, l, "memory") 334 335 #undef __CMPXCHG128 336 337 #undef K 338 339 #endif /* __ASM_ATOMIC_LL_SC_H */ 340