1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ALPHA_CMPXCHG_H
3 #define _ALPHA_CMPXCHG_H
4
5 /*
6 * Atomic exchange.
7 * Since it can be used to implement critical sections
8 * it must clobber "memory" (also for interrupts in UP).
9 */
10
11 static inline unsigned long
____xchg_u8(volatile char * m,unsigned long val)12 ____xchg_u8(volatile char *m, unsigned long val)
13 {
14 unsigned long ret, tmp, addr64;
15
16 __asm__ __volatile__(
17 " andnot %4,7,%3\n"
18 " insbl %1,%4,%1\n"
19 "1: ldq_l %2,0(%3)\n"
20 " extbl %2,%4,%0\n"
21 " mskbl %2,%4,%2\n"
22 " or %1,%2,%2\n"
23 " stq_c %2,0(%3)\n"
24 " beq %2,2f\n"
25 ".subsection 2\n"
26 "2: br 1b\n"
27 ".previous"
28 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
29 : "r" ((long)m), "1" (val) : "memory");
30
31 return ret;
32 }
33
34 static inline unsigned long
____xchg_u16(volatile short * m,unsigned long val)35 ____xchg_u16(volatile short *m, unsigned long val)
36 {
37 unsigned long ret, tmp, addr64;
38
39 __asm__ __volatile__(
40 " andnot %4,7,%3\n"
41 " inswl %1,%4,%1\n"
42 "1: ldq_l %2,0(%3)\n"
43 " extwl %2,%4,%0\n"
44 " mskwl %2,%4,%2\n"
45 " or %1,%2,%2\n"
46 " stq_c %2,0(%3)\n"
47 " beq %2,2f\n"
48 ".subsection 2\n"
49 "2: br 1b\n"
50 ".previous"
51 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
52 : "r" ((long)m), "1" (val) : "memory");
53
54 return ret;
55 }
56
57 static inline unsigned long
____xchg_u32(volatile int * m,unsigned long val)58 ____xchg_u32(volatile int *m, unsigned long val)
59 {
60 unsigned long dummy;
61
62 __asm__ __volatile__(
63 "1: ldl_l %0,%4\n"
64 " bis $31,%3,%1\n"
65 " stl_c %1,%2\n"
66 " beq %1,2f\n"
67 ".subsection 2\n"
68 "2: br 1b\n"
69 ".previous"
70 : "=&r" (val), "=&r" (dummy), "=m" (*m)
71 : "rI" (val), "m" (*m) : "memory");
72
73 return val;
74 }
75
76 static inline unsigned long
____xchg_u64(volatile long * m,unsigned long val)77 ____xchg_u64(volatile long *m, unsigned long val)
78 {
79 unsigned long dummy;
80
81 __asm__ __volatile__(
82 "1: ldq_l %0,%4\n"
83 " bis $31,%3,%1\n"
84 " stq_c %1,%2\n"
85 " beq %1,2f\n"
86 ".subsection 2\n"
87 "2: br 1b\n"
88 ".previous"
89 : "=&r" (val), "=&r" (dummy), "=m" (*m)
90 : "rI" (val), "m" (*m) : "memory");
91
92 return val;
93 }
94
95 /* This function doesn't exist, so you'll get a linker error
96 if something tries to do an invalid xchg(). */
97 extern void __xchg_called_with_bad_pointer(void);
98
99 static __always_inline unsigned long
____xchg(volatile void * ptr,unsigned long x,int size)100 ____xchg(volatile void *ptr, unsigned long x, int size)
101 {
102 return
103 size == 1 ? ____xchg_u8(ptr, x) :
104 size == 2 ? ____xchg_u16(ptr, x) :
105 size == 4 ? ____xchg_u32(ptr, x) :
106 size == 8 ? ____xchg_u64(ptr, x) :
107 (__xchg_called_with_bad_pointer(), x);
108 }
109
110 /*
111 * Atomic compare and exchange. Compare OLD with MEM, if identical,
112 * store NEW in MEM. Return the initial value in MEM. Success is
113 * indicated by comparing RETURN with OLD.
114 */
115
116 static inline unsigned long
____cmpxchg_u8(volatile char * m,unsigned char old,unsigned char new)117 ____cmpxchg_u8(volatile char *m, unsigned char old, unsigned char new)
118 {
119 unsigned long prev, tmp, cmp, addr64;
120
121 __asm__ __volatile__(
122 " andnot %5,7,%4\n"
123 " insbl %1,%5,%1\n"
124 "1: ldq_l %2,0(%4)\n"
125 " extbl %2,%5,%0\n"
126 " cmpeq %0,%6,%3\n"
127 " beq %3,2f\n"
128 " mskbl %2,%5,%2\n"
129 " or %1,%2,%2\n"
130 " stq_c %2,0(%4)\n"
131 " beq %2,3f\n"
132 "2:\n"
133 ".subsection 2\n"
134 "3: br 1b\n"
135 ".previous"
136 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
137 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
138
139 return prev;
140 }
141
142 static inline unsigned long
____cmpxchg_u16(volatile short * m,unsigned short old,unsigned short new)143 ____cmpxchg_u16(volatile short *m, unsigned short old, unsigned short new)
144 {
145 unsigned long prev, tmp, cmp, addr64;
146
147 __asm__ __volatile__(
148 " andnot %5,7,%4\n"
149 " inswl %1,%5,%1\n"
150 "1: ldq_l %2,0(%4)\n"
151 " extwl %2,%5,%0\n"
152 " cmpeq %0,%6,%3\n"
153 " beq %3,2f\n"
154 " mskwl %2,%5,%2\n"
155 " or %1,%2,%2\n"
156 " stq_c %2,0(%4)\n"
157 " beq %2,3f\n"
158 "2:\n"
159 ".subsection 2\n"
160 "3: br 1b\n"
161 ".previous"
162 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
163 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
164
165 return prev;
166 }
167
168 static inline unsigned long
____cmpxchg_u32(volatile int * m,int old,int new)169 ____cmpxchg_u32(volatile int *m, int old, int new)
170 {
171 unsigned long prev, cmp;
172
173 __asm__ __volatile__(
174 "1: ldl_l %0,%5\n"
175 " cmpeq %0,%3,%1\n"
176 " beq %1,2f\n"
177 " mov %4,%1\n"
178 " stl_c %1,%2\n"
179 " beq %1,3f\n"
180 "2:\n"
181 ".subsection 2\n"
182 "3: br 1b\n"
183 ".previous"
184 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
185 : "r"((long) old), "r"(new), "m"(*m) : "memory");
186
187 return prev;
188 }
189
190 static inline unsigned long
____cmpxchg_u64(volatile long * m,unsigned long old,unsigned long new)191 ____cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new)
192 {
193 unsigned long prev, cmp;
194
195 __asm__ __volatile__(
196 "1: ldq_l %0,%5\n"
197 " cmpeq %0,%3,%1\n"
198 " beq %1,2f\n"
199 " mov %4,%1\n"
200 " stq_c %1,%2\n"
201 " beq %1,3f\n"
202 "2:\n"
203 ".subsection 2\n"
204 "3: br 1b\n"
205 ".previous"
206 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
207 : "r"((long) old), "r"(new), "m"(*m) : "memory");
208
209 return prev;
210 }
211
212 /* This function doesn't exist, so you'll get a linker error
213 if something tries to do an invalid cmpxchg(). */
214 extern void __cmpxchg_called_with_bad_pointer(void);
215
216 static __always_inline unsigned long
____cmpxchg(volatile void * ptr,unsigned long old,unsigned long new,int size)217 ____cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
218 int size)
219 {
220 return
221 size == 1 ? ____cmpxchg_u8(ptr, old, new) :
222 size == 2 ? ____cmpxchg_u16(ptr, old, new) :
223 size == 4 ? ____cmpxchg_u32(ptr, old, new) :
224 size == 8 ? ____cmpxchg_u64(ptr, old, new) :
225 (__cmpxchg_called_with_bad_pointer(), old);
226 }
227
228 #define xchg_local(ptr, x) \
229 ({ \
230 __typeof__(*(ptr)) _x_ = (x); \
231 (__typeof__(*(ptr))) ____xchg((ptr), (unsigned long)_x_, \
232 sizeof(*(ptr))); \
233 })
234
235 #define arch_cmpxchg_local(ptr, o, n) \
236 ({ \
237 __typeof__(*(ptr)) _o_ = (o); \
238 __typeof__(*(ptr)) _n_ = (n); \
239 (__typeof__(*(ptr))) ____cmpxchg((ptr), (unsigned long)_o_, \
240 (unsigned long)_n_, \
241 sizeof(*(ptr))); \
242 })
243
244 #define arch_cmpxchg64_local(ptr, o, n) \
245 ({ \
246 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
247 cmpxchg_local((ptr), (o), (n)); \
248 })
249
250 /*
251 * The leading and the trailing memory barriers guarantee that these
252 * operations are fully ordered.
253 */
254 #define arch_xchg(ptr, x) \
255 ({ \
256 __typeof__(*(ptr)) __ret; \
257 __typeof__(*(ptr)) _x_ = (x); \
258 smp_mb(); \
259 __ret = (__typeof__(*(ptr))) \
260 ____xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
261 smp_mb(); \
262 __ret; \
263 })
264
265 #define arch_cmpxchg(ptr, o, n) \
266 ({ \
267 __typeof__(*(ptr)) __ret; \
268 __typeof__(*(ptr)) _o_ = (o); \
269 __typeof__(*(ptr)) _n_ = (n); \
270 smp_mb(); \
271 __ret = (__typeof__(*(ptr))) ____cmpxchg((ptr), \
272 (unsigned long)_o_, (unsigned long)_n_, sizeof(*(ptr)));\
273 smp_mb(); \
274 __ret; \
275 })
276
277 #define arch_cmpxchg64(ptr, o, n) \
278 ({ \
279 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
280 arch_cmpxchg((ptr), (o), (n)); \
281 })
282
283 #endif /* _ALPHA_CMPXCHG_H */
284