1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/cmpxchg.h 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 */ 7 #ifndef __ASM_CMPXCHG_H 8 #define __ASM_CMPXCHG_H 9 10 #include <linux/build_bug.h> 11 #include <linux/compiler.h> 12 13 #include <asm/barrier.h> 14 #include <asm/lse.h> 15 16 /* 17 * We need separate acquire parameters for ll/sc and lse, since the full 18 * barrier case is generated as release+dmb for the former and 19 * acquire+release for the latter. 20 */ 21 #define __XCHG_CASE(w, sfx, name, sz, mb, nop_lse, acq, acq_lse, rel, cl) \ 22 static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr) \ 23 { \ 24 u##sz ret; \ 25 unsigned long tmp; \ 26 \ 27 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 28 /* LL/SC */ \ 29 " prfm pstl1strm, %2\n" \ 30 "1: ld" #acq "xr" #sfx "\t%" #w "0, %2\n" \ 31 " st" #rel "xr" #sfx "\t%w1, %" #w "3, %2\n" \ 32 " cbnz %w1, 1b\n" \ 33 " " #mb, \ 34 /* LSE atomics */ \ 35 " swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n" \ 36 __nops(3) \ 37 " " #nop_lse) \ 38 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u##sz *)ptr) \ 39 : "r" (x) \ 40 : cl); \ 41 \ 42 return ret; \ 43 } 44 45 __XCHG_CASE(w, b, , 8, , , , , , ) 46 __XCHG_CASE(w, h, , 16, , , , , , ) 47 __XCHG_CASE(w, , , 32, , , , , , ) 48 __XCHG_CASE( , , , 64, , , , , , ) 49 __XCHG_CASE(w, b, acq_, 8, , , a, a, , "memory") 50 __XCHG_CASE(w, h, acq_, 16, , , a, a, , "memory") 51 __XCHG_CASE(w, , acq_, 32, , , a, a, , "memory") 52 __XCHG_CASE( , , acq_, 64, , , a, a, , "memory") 53 __XCHG_CASE(w, b, rel_, 8, , , , , l, "memory") 54 __XCHG_CASE(w, h, rel_, 16, , , , , l, "memory") 55 __XCHG_CASE(w, , rel_, 32, , , , , l, "memory") 56 __XCHG_CASE( , , rel_, 64, , , , , l, "memory") 57 __XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory") 58 __XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory") 59 __XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory") 60 __XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory") 61 62 #undef __XCHG_CASE 63 64 #define __XCHG_GEN(sfx) \ 65 static __always_inline unsigned long \ 66 __arch_xchg##sfx(unsigned long x, volatile void *ptr, int size) \ 67 { \ 68 switch (size) { \ 69 case 1: \ 70 return __xchg_case##sfx##_8(x, ptr); \ 71 case 2: \ 72 return __xchg_case##sfx##_16(x, ptr); \ 73 case 4: \ 74 return __xchg_case##sfx##_32(x, ptr); \ 75 case 8: \ 76 return __xchg_case##sfx##_64(x, ptr); \ 77 default: \ 78 BUILD_BUG(); \ 79 } \ 80 \ 81 unreachable(); \ 82 } 83 84 __XCHG_GEN() 85 __XCHG_GEN(_acq) 86 __XCHG_GEN(_rel) 87 __XCHG_GEN(_mb) 88 89 #undef __XCHG_GEN 90 91 #define __xchg_wrapper(sfx, ptr, x) \ 92 ({ \ 93 __typeof__(*(ptr)) __ret; \ 94 __ret = (__typeof__(*(ptr))) \ 95 __arch_xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \ 96 __ret; \ 97 }) 98 99 /* xchg */ 100 #define arch_xchg_relaxed(...) __xchg_wrapper( , __VA_ARGS__) 101 #define arch_xchg_acquire(...) __xchg_wrapper(_acq, __VA_ARGS__) 102 #define arch_xchg_release(...) __xchg_wrapper(_rel, __VA_ARGS__) 103 #define arch_xchg(...) __xchg_wrapper( _mb, __VA_ARGS__) 104 105 #define __CMPXCHG_CASE(name, sz) \ 106 static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr, \ 107 u##sz old, \ 108 u##sz new) \ 109 { \ 110 return __lse_ll_sc_body(_cmpxchg_case_##name##sz, \ 111 ptr, old, new); \ 112 } 113 114 __CMPXCHG_CASE( , 8) 115 __CMPXCHG_CASE( , 16) 116 __CMPXCHG_CASE( , 32) 117 __CMPXCHG_CASE( , 64) 118 __CMPXCHG_CASE(acq_, 8) 119 __CMPXCHG_CASE(acq_, 16) 120 __CMPXCHG_CASE(acq_, 32) 121 __CMPXCHG_CASE(acq_, 64) 122 __CMPXCHG_CASE(rel_, 8) 123 __CMPXCHG_CASE(rel_, 16) 124 __CMPXCHG_CASE(rel_, 32) 125 __CMPXCHG_CASE(rel_, 64) 126 __CMPXCHG_CASE(mb_, 8) 127 __CMPXCHG_CASE(mb_, 16) 128 __CMPXCHG_CASE(mb_, 32) 129 __CMPXCHG_CASE(mb_, 64) 130 131 #undef __CMPXCHG_CASE 132 133 #define __CMPXCHG_DBL(name) \ 134 static inline long __cmpxchg_double##name(unsigned long old1, \ 135 unsigned long old2, \ 136 unsigned long new1, \ 137 unsigned long new2, \ 138 volatile void *ptr) \ 139 { \ 140 return __lse_ll_sc_body(_cmpxchg_double##name, \ 141 old1, old2, new1, new2, ptr); \ 142 } 143 144 __CMPXCHG_DBL( ) 145 __CMPXCHG_DBL(_mb) 146 147 #undef __CMPXCHG_DBL 148 149 #define __CMPXCHG128(name) \ 150 static inline u128 __cmpxchg128##name(volatile u128 *ptr, \ 151 u128 old, u128 new) \ 152 { \ 153 return __lse_ll_sc_body(_cmpxchg128##name, \ 154 ptr, old, new); \ 155 } 156 157 __CMPXCHG128( ) 158 __CMPXCHG128(_mb) 159 160 #undef __CMPXCHG128 161 162 #define __CMPXCHG_GEN(sfx) \ 163 static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr, \ 164 unsigned long old, \ 165 unsigned long new, \ 166 int size) \ 167 { \ 168 switch (size) { \ 169 case 1: \ 170 return __cmpxchg_case##sfx##_8(ptr, old, new); \ 171 case 2: \ 172 return __cmpxchg_case##sfx##_16(ptr, old, new); \ 173 case 4: \ 174 return __cmpxchg_case##sfx##_32(ptr, old, new); \ 175 case 8: \ 176 return __cmpxchg_case##sfx##_64(ptr, old, new); \ 177 default: \ 178 BUILD_BUG(); \ 179 } \ 180 \ 181 unreachable(); \ 182 } 183 184 __CMPXCHG_GEN() 185 __CMPXCHG_GEN(_acq) 186 __CMPXCHG_GEN(_rel) 187 __CMPXCHG_GEN(_mb) 188 189 #undef __CMPXCHG_GEN 190 191 #define __cmpxchg_wrapper(sfx, ptr, o, n) \ 192 ({ \ 193 __typeof__(*(ptr)) __ret; \ 194 __ret = (__typeof__(*(ptr))) \ 195 __cmpxchg##sfx((ptr), (unsigned long)(o), \ 196 (unsigned long)(n), sizeof(*(ptr))); \ 197 __ret; \ 198 }) 199 200 /* cmpxchg */ 201 #define arch_cmpxchg_relaxed(...) __cmpxchg_wrapper( , __VA_ARGS__) 202 #define arch_cmpxchg_acquire(...) __cmpxchg_wrapper(_acq, __VA_ARGS__) 203 #define arch_cmpxchg_release(...) __cmpxchg_wrapper(_rel, __VA_ARGS__) 204 #define arch_cmpxchg(...) __cmpxchg_wrapper( _mb, __VA_ARGS__) 205 #define arch_cmpxchg_local arch_cmpxchg_relaxed 206 207 /* cmpxchg64 */ 208 #define arch_cmpxchg64_relaxed arch_cmpxchg_relaxed 209 #define arch_cmpxchg64_acquire arch_cmpxchg_acquire 210 #define arch_cmpxchg64_release arch_cmpxchg_release 211 #define arch_cmpxchg64 arch_cmpxchg 212 #define arch_cmpxchg64_local arch_cmpxchg_local 213 214 /* cmpxchg_double */ 215 #define system_has_cmpxchg_double() 1 216 217 #define __cmpxchg_double_check(ptr1, ptr2) \ 218 ({ \ 219 if (sizeof(*(ptr1)) != 8) \ 220 BUILD_BUG(); \ 221 VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \ 222 }) 223 224 #define arch_cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \ 225 ({ \ 226 int __ret; \ 227 __cmpxchg_double_check(ptr1, ptr2); \ 228 __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \ 229 (unsigned long)(n1), (unsigned long)(n2), \ 230 ptr1); \ 231 __ret; \ 232 }) 233 234 #define arch_cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \ 235 ({ \ 236 int __ret; \ 237 __cmpxchg_double_check(ptr1, ptr2); \ 238 __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \ 239 (unsigned long)(n1), (unsigned long)(n2), \ 240 ptr1); \ 241 __ret; \ 242 }) 243 244 /* cmpxchg128 */ 245 #define system_has_cmpxchg128() 1 246 247 #define arch_cmpxchg128(ptr, o, n) \ 248 ({ \ 249 __cmpxchg128_mb((ptr), (o), (n)); \ 250 }) 251 252 #define arch_cmpxchg128_local(ptr, o, n) \ 253 ({ \ 254 __cmpxchg128((ptr), (o), (n)); \ 255 }) 256 257 #define __CMPWAIT_CASE(w, sfx, sz) \ 258 static inline void __cmpwait_case_##sz(volatile void *ptr, \ 259 unsigned long val) \ 260 { \ 261 unsigned long tmp; \ 262 \ 263 asm volatile( \ 264 " sevl\n" \ 265 " wfe\n" \ 266 " ldxr" #sfx "\t%" #w "[tmp], %[v]\n" \ 267 " eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \ 268 " cbnz %" #w "[tmp], 1f\n" \ 269 " wfe\n" \ 270 "1:" \ 271 : [tmp] "=&r" (tmp), [v] "+Q" (*(u##sz *)ptr) \ 272 : [val] "r" (val)); \ 273 } 274 275 __CMPWAIT_CASE(w, b, 8); 276 __CMPWAIT_CASE(w, h, 16); 277 __CMPWAIT_CASE(w, , 32); 278 __CMPWAIT_CASE( , , 64); 279 280 #undef __CMPWAIT_CASE 281 282 #define __CMPWAIT_GEN(sfx) \ 283 static __always_inline void __cmpwait##sfx(volatile void *ptr, \ 284 unsigned long val, \ 285 int size) \ 286 { \ 287 switch (size) { \ 288 case 1: \ 289 return __cmpwait_case##sfx##_8(ptr, (u8)val); \ 290 case 2: \ 291 return __cmpwait_case##sfx##_16(ptr, (u16)val); \ 292 case 4: \ 293 return __cmpwait_case##sfx##_32(ptr, val); \ 294 case 8: \ 295 return __cmpwait_case##sfx##_64(ptr, val); \ 296 default: \ 297 BUILD_BUG(); \ 298 } \ 299 \ 300 unreachable(); \ 301 } 302 303 __CMPWAIT_GEN() 304 305 #undef __CMPWAIT_GEN 306 307 #define __cmpwait_relaxed(ptr, val) \ 308 __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr))) 309 310 #endif /* __ASM_CMPXCHG_H */ 311