1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ALPHA_CMPXCHG_H 3 #define _ALPHA_CMPXCHG_H 4 5 /* 6 * Atomic exchange. 7 * Since it can be used to implement critical sections 8 * it must clobber "memory" (also for interrupts in UP). 9 */ 10 11 static inline unsigned long 12 ____xchg_u8(volatile char *m, unsigned long val) 13 { 14 unsigned long ret, tmp, addr64; 15 16 __asm__ __volatile__( 17 " andnot %4,7,%3\n" 18 " insbl %1,%4,%1\n" 19 "1: ldq_l %2,0(%3)\n" 20 " extbl %2,%4,%0\n" 21 " mskbl %2,%4,%2\n" 22 " or %1,%2,%2\n" 23 " stq_c %2,0(%3)\n" 24 " beq %2,2f\n" 25 ".subsection 2\n" 26 "2: br 1b\n" 27 ".previous" 28 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) 29 : "r" ((long)m), "1" (val) : "memory"); 30 31 return ret; 32 } 33 34 static inline unsigned long 35 ____xchg_u16(volatile short *m, unsigned long val) 36 { 37 unsigned long ret, tmp, addr64; 38 39 __asm__ __volatile__( 40 " andnot %4,7,%3\n" 41 " inswl %1,%4,%1\n" 42 "1: ldq_l %2,0(%3)\n" 43 " extwl %2,%4,%0\n" 44 " mskwl %2,%4,%2\n" 45 " or %1,%2,%2\n" 46 " stq_c %2,0(%3)\n" 47 " beq %2,2f\n" 48 ".subsection 2\n" 49 "2: br 1b\n" 50 ".previous" 51 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) 52 : "r" ((long)m), "1" (val) : "memory"); 53 54 return ret; 55 } 56 57 static inline unsigned long 58 ____xchg_u32(volatile int *m, unsigned long val) 59 { 60 unsigned long dummy; 61 62 __asm__ __volatile__( 63 "1: ldl_l %0,%4\n" 64 " bis $31,%3,%1\n" 65 " stl_c %1,%2\n" 66 " beq %1,2f\n" 67 ".subsection 2\n" 68 "2: br 1b\n" 69 ".previous" 70 : "=&r" (val), "=&r" (dummy), "=m" (*m) 71 : "rI" (val), "m" (*m) : "memory"); 72 73 return val; 74 } 75 76 static inline unsigned long 77 ____xchg_u64(volatile long *m, unsigned long val) 78 { 79 unsigned long dummy; 80 81 __asm__ __volatile__( 82 "1: ldq_l %0,%4\n" 83 " bis $31,%3,%1\n" 84 " stq_c %1,%2\n" 85 " beq %1,2f\n" 86 ".subsection 2\n" 87 "2: br 1b\n" 88 ".previous" 89 : "=&r" (val), "=&r" (dummy), "=m" (*m) 90 : "rI" (val), "m" (*m) : "memory"); 91 92 return val; 93 } 94 95 /* This function doesn't exist, so you'll get a linker error 96 if something tries to do an invalid xchg(). */ 97 extern void __xchg_called_with_bad_pointer(void); 98 99 static __always_inline unsigned long 100 ____xchg(volatile void *ptr, unsigned long x, int size) 101 { 102 return 103 size == 1 ? ____xchg_u8(ptr, x) : 104 size == 2 ? ____xchg_u16(ptr, x) : 105 size == 4 ? ____xchg_u32(ptr, x) : 106 size == 8 ? ____xchg_u64(ptr, x) : 107 (__xchg_called_with_bad_pointer(), x); 108 } 109 110 /* 111 * Atomic compare and exchange. Compare OLD with MEM, if identical, 112 * store NEW in MEM. Return the initial value in MEM. Success is 113 * indicated by comparing RETURN with OLD. 114 */ 115 116 static inline unsigned long 117 ____cmpxchg_u8(volatile char *m, unsigned char old, unsigned char new) 118 { 119 unsigned long prev, tmp, cmp, addr64; 120 121 __asm__ __volatile__( 122 " andnot %5,7,%4\n" 123 " insbl %1,%5,%1\n" 124 "1: ldq_l %2,0(%4)\n" 125 " extbl %2,%5,%0\n" 126 " cmpeq %0,%6,%3\n" 127 " beq %3,2f\n" 128 " mskbl %2,%5,%2\n" 129 " or %1,%2,%2\n" 130 " stq_c %2,0(%4)\n" 131 " beq %2,3f\n" 132 "2:\n" 133 ".subsection 2\n" 134 "3: br 1b\n" 135 ".previous" 136 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) 137 : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); 138 139 return prev; 140 } 141 142 static inline unsigned long 143 ____cmpxchg_u16(volatile short *m, unsigned short old, unsigned short new) 144 { 145 unsigned long prev, tmp, cmp, addr64; 146 147 __asm__ __volatile__( 148 " andnot %5,7,%4\n" 149 " inswl %1,%5,%1\n" 150 "1: ldq_l %2,0(%4)\n" 151 " extwl %2,%5,%0\n" 152 " cmpeq %0,%6,%3\n" 153 " beq %3,2f\n" 154 " mskwl %2,%5,%2\n" 155 " or %1,%2,%2\n" 156 " stq_c %2,0(%4)\n" 157 " beq %2,3f\n" 158 "2:\n" 159 ".subsection 2\n" 160 "3: br 1b\n" 161 ".previous" 162 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) 163 : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); 164 165 return prev; 166 } 167 168 static inline unsigned long 169 ____cmpxchg_u32(volatile int *m, int old, int new) 170 { 171 unsigned long prev, cmp; 172 173 __asm__ __volatile__( 174 "1: ldl_l %0,%5\n" 175 " cmpeq %0,%3,%1\n" 176 " beq %1,2f\n" 177 " mov %4,%1\n" 178 " stl_c %1,%2\n" 179 " beq %1,3f\n" 180 "2:\n" 181 ".subsection 2\n" 182 "3: br 1b\n" 183 ".previous" 184 : "=&r"(prev), "=&r"(cmp), "=m"(*m) 185 : "r"((long) old), "r"(new), "m"(*m) : "memory"); 186 187 return prev; 188 } 189 190 static inline unsigned long 191 ____cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new) 192 { 193 unsigned long prev, cmp; 194 195 __asm__ __volatile__( 196 "1: ldq_l %0,%5\n" 197 " cmpeq %0,%3,%1\n" 198 " beq %1,2f\n" 199 " mov %4,%1\n" 200 " stq_c %1,%2\n" 201 " beq %1,3f\n" 202 "2:\n" 203 ".subsection 2\n" 204 "3: br 1b\n" 205 ".previous" 206 : "=&r"(prev), "=&r"(cmp), "=m"(*m) 207 : "r"((long) old), "r"(new), "m"(*m) : "memory"); 208 209 return prev; 210 } 211 212 /* This function doesn't exist, so you'll get a linker error 213 if something tries to do an invalid cmpxchg(). */ 214 extern void __cmpxchg_called_with_bad_pointer(void); 215 216 static __always_inline unsigned long 217 ____cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, 218 int size) 219 { 220 return 221 size == 1 ? ____cmpxchg_u8(ptr, old, new) : 222 size == 2 ? ____cmpxchg_u16(ptr, old, new) : 223 size == 4 ? ____cmpxchg_u32(ptr, old, new) : 224 size == 8 ? ____cmpxchg_u64(ptr, old, new) : 225 (__cmpxchg_called_with_bad_pointer(), old); 226 } 227 228 #define xchg_local(ptr, x) \ 229 ({ \ 230 __typeof__(*(ptr)) _x_ = (x); \ 231 (__typeof__(*(ptr))) ____xchg((ptr), (unsigned long)_x_, \ 232 sizeof(*(ptr))); \ 233 }) 234 235 #define arch_cmpxchg_local(ptr, o, n) \ 236 ({ \ 237 __typeof__(*(ptr)) _o_ = (o); \ 238 __typeof__(*(ptr)) _n_ = (n); \ 239 (__typeof__(*(ptr))) ____cmpxchg((ptr), (unsigned long)_o_, \ 240 (unsigned long)_n_, \ 241 sizeof(*(ptr))); \ 242 }) 243 244 #define arch_cmpxchg64_local(ptr, o, n) \ 245 ({ \ 246 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 247 cmpxchg_local((ptr), (o), (n)); \ 248 }) 249 250 /* 251 * The leading and the trailing memory barriers guarantee that these 252 * operations are fully ordered. 253 */ 254 #define arch_xchg(ptr, x) \ 255 ({ \ 256 __typeof__(*(ptr)) __ret; \ 257 __typeof__(*(ptr)) _x_ = (x); \ 258 smp_mb(); \ 259 __ret = (__typeof__(*(ptr))) \ 260 ____xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \ 261 smp_mb(); \ 262 __ret; \ 263 }) 264 265 #define arch_cmpxchg(ptr, o, n) \ 266 ({ \ 267 __typeof__(*(ptr)) __ret; \ 268 __typeof__(*(ptr)) _o_ = (o); \ 269 __typeof__(*(ptr)) _n_ = (n); \ 270 smp_mb(); \ 271 __ret = (__typeof__(*(ptr))) ____cmpxchg((ptr), \ 272 (unsigned long)_o_, (unsigned long)_n_, sizeof(*(ptr)));\ 273 smp_mb(); \ 274 __ret; \ 275 }) 276 277 #define arch_cmpxchg64(ptr, o, n) \ 278 ({ \ 279 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 280 arch_cmpxchg((ptr), (o), (n)); \ 281 }) 282 283 #endif /* _ALPHA_CMPXCHG_H */ 284