xref: /linux/arch/alpha/include/asm/cmpxchg.h (revision 3fd6c59042dbba50391e30862beac979491145fe)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ALPHA_CMPXCHG_H
3 #define _ALPHA_CMPXCHG_H
4 
5 /*
6  * Atomic exchange.
7  * Since it can be used to implement critical sections
8  * it must clobber "memory" (also for interrupts in UP).
9  */
10 
11 static inline unsigned long
____xchg_u8(volatile char * m,unsigned long val)12 ____xchg_u8(volatile char *m, unsigned long val)
13 {
14 	unsigned long ret, tmp, addr64;
15 
16 	__asm__ __volatile__(
17 	"	andnot	%4,7,%3\n"
18 	"	insbl	%1,%4,%1\n"
19 	"1:	ldq_l	%2,0(%3)\n"
20 	"	extbl	%2,%4,%0\n"
21 	"	mskbl	%2,%4,%2\n"
22 	"	or	%1,%2,%2\n"
23 	"	stq_c	%2,0(%3)\n"
24 	"	beq	%2,2f\n"
25 	".subsection 2\n"
26 	"2:	br	1b\n"
27 	".previous"
28 	: "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
29 	: "r" ((long)m), "1" (val) : "memory");
30 
31 	return ret;
32 }
33 
34 static inline unsigned long
____xchg_u16(volatile short * m,unsigned long val)35 ____xchg_u16(volatile short *m, unsigned long val)
36 {
37 	unsigned long ret, tmp, addr64;
38 
39 	__asm__ __volatile__(
40 	"	andnot	%4,7,%3\n"
41 	"	inswl	%1,%4,%1\n"
42 	"1:	ldq_l	%2,0(%3)\n"
43 	"	extwl	%2,%4,%0\n"
44 	"	mskwl	%2,%4,%2\n"
45 	"	or	%1,%2,%2\n"
46 	"	stq_c	%2,0(%3)\n"
47 	"	beq	%2,2f\n"
48 	".subsection 2\n"
49 	"2:	br	1b\n"
50 	".previous"
51 	: "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
52 	: "r" ((long)m), "1" (val) : "memory");
53 
54 	return ret;
55 }
56 
57 static inline unsigned long
____xchg_u32(volatile int * m,unsigned long val)58 ____xchg_u32(volatile int *m, unsigned long val)
59 {
60 	unsigned long dummy;
61 
62 	__asm__ __volatile__(
63 	"1:	ldl_l %0,%4\n"
64 	"	bis $31,%3,%1\n"
65 	"	stl_c %1,%2\n"
66 	"	beq %1,2f\n"
67 	".subsection 2\n"
68 	"2:	br 1b\n"
69 	".previous"
70 	: "=&r" (val), "=&r" (dummy), "=m" (*m)
71 	: "rI" (val), "m" (*m) : "memory");
72 
73 	return val;
74 }
75 
76 static inline unsigned long
____xchg_u64(volatile long * m,unsigned long val)77 ____xchg_u64(volatile long *m, unsigned long val)
78 {
79 	unsigned long dummy;
80 
81 	__asm__ __volatile__(
82 	"1:	ldq_l %0,%4\n"
83 	"	bis $31,%3,%1\n"
84 	"	stq_c %1,%2\n"
85 	"	beq %1,2f\n"
86 	".subsection 2\n"
87 	"2:	br 1b\n"
88 	".previous"
89 	: "=&r" (val), "=&r" (dummy), "=m" (*m)
90 	: "rI" (val), "m" (*m) : "memory");
91 
92 	return val;
93 }
94 
95 /* This function doesn't exist, so you'll get a linker error
96    if something tries to do an invalid xchg().  */
97 extern void __xchg_called_with_bad_pointer(void);
98 
99 static __always_inline unsigned long
____xchg(volatile void * ptr,unsigned long x,int size)100 ____xchg(volatile void *ptr, unsigned long x, int size)
101 {
102 	return
103 		size == 1 ? ____xchg_u8(ptr, x) :
104 		size == 2 ? ____xchg_u16(ptr, x) :
105 		size == 4 ? ____xchg_u32(ptr, x) :
106 		size == 8 ? ____xchg_u64(ptr, x) :
107 			(__xchg_called_with_bad_pointer(), x);
108 }
109 
110 /*
111  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
112  * store NEW in MEM.  Return the initial value in MEM.  Success is
113  * indicated by comparing RETURN with OLD.
114  */
115 
116 static inline unsigned long
____cmpxchg_u8(volatile char * m,unsigned char old,unsigned char new)117 ____cmpxchg_u8(volatile char *m, unsigned char old, unsigned char new)
118 {
119 	unsigned long prev, tmp, cmp, addr64;
120 
121 	__asm__ __volatile__(
122 	"	andnot	%5,7,%4\n"
123 	"	insbl	%1,%5,%1\n"
124 	"1:	ldq_l	%2,0(%4)\n"
125 	"	extbl	%2,%5,%0\n"
126 	"	cmpeq	%0,%6,%3\n"
127 	"	beq	%3,2f\n"
128 	"	mskbl	%2,%5,%2\n"
129 	"	or	%1,%2,%2\n"
130 	"	stq_c	%2,0(%4)\n"
131 	"	beq	%2,3f\n"
132 	"2:\n"
133 	".subsection 2\n"
134 	"3:	br	1b\n"
135 	".previous"
136 	: "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
137 	: "r" ((long)m), "Ir" (old), "1" (new) : "memory");
138 
139 	return prev;
140 }
141 
142 static inline unsigned long
____cmpxchg_u16(volatile short * m,unsigned short old,unsigned short new)143 ____cmpxchg_u16(volatile short *m, unsigned short old, unsigned short new)
144 {
145 	unsigned long prev, tmp, cmp, addr64;
146 
147 	__asm__ __volatile__(
148 	"	andnot	%5,7,%4\n"
149 	"	inswl	%1,%5,%1\n"
150 	"1:	ldq_l	%2,0(%4)\n"
151 	"	extwl	%2,%5,%0\n"
152 	"	cmpeq	%0,%6,%3\n"
153 	"	beq	%3,2f\n"
154 	"	mskwl	%2,%5,%2\n"
155 	"	or	%1,%2,%2\n"
156 	"	stq_c	%2,0(%4)\n"
157 	"	beq	%2,3f\n"
158 	"2:\n"
159 	".subsection 2\n"
160 	"3:	br	1b\n"
161 	".previous"
162 	: "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
163 	: "r" ((long)m), "Ir" (old), "1" (new) : "memory");
164 
165 	return prev;
166 }
167 
168 static inline unsigned long
____cmpxchg_u32(volatile int * m,int old,int new)169 ____cmpxchg_u32(volatile int *m, int old, int new)
170 {
171 	unsigned long prev, cmp;
172 
173 	__asm__ __volatile__(
174 	"1:	ldl_l %0,%5\n"
175 	"	cmpeq %0,%3,%1\n"
176 	"	beq %1,2f\n"
177 	"	mov %4,%1\n"
178 	"	stl_c %1,%2\n"
179 	"	beq %1,3f\n"
180 	"2:\n"
181 	".subsection 2\n"
182 	"3:	br 1b\n"
183 	".previous"
184 	: "=&r"(prev), "=&r"(cmp), "=m"(*m)
185 	: "r"((long) old), "r"(new), "m"(*m) : "memory");
186 
187 	return prev;
188 }
189 
190 static inline unsigned long
____cmpxchg_u64(volatile long * m,unsigned long old,unsigned long new)191 ____cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new)
192 {
193 	unsigned long prev, cmp;
194 
195 	__asm__ __volatile__(
196 	"1:	ldq_l %0,%5\n"
197 	"	cmpeq %0,%3,%1\n"
198 	"	beq %1,2f\n"
199 	"	mov %4,%1\n"
200 	"	stq_c %1,%2\n"
201 	"	beq %1,3f\n"
202 	"2:\n"
203 	".subsection 2\n"
204 	"3:	br 1b\n"
205 	".previous"
206 	: "=&r"(prev), "=&r"(cmp), "=m"(*m)
207 	: "r"((long) old), "r"(new), "m"(*m) : "memory");
208 
209 	return prev;
210 }
211 
212 /* This function doesn't exist, so you'll get a linker error
213    if something tries to do an invalid cmpxchg().  */
214 extern void __cmpxchg_called_with_bad_pointer(void);
215 
216 static __always_inline unsigned long
____cmpxchg(volatile void * ptr,unsigned long old,unsigned long new,int size)217 ____cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
218 	      int size)
219 {
220 	return
221 		size == 1 ? ____cmpxchg_u8(ptr, old, new) :
222 		size == 2 ? ____cmpxchg_u16(ptr, old, new) :
223 		size == 4 ? ____cmpxchg_u32(ptr, old, new) :
224 		size == 8 ? ____cmpxchg_u64(ptr, old, new) :
225 			(__cmpxchg_called_with_bad_pointer(), old);
226 }
227 
228 #define xchg_local(ptr, x)						\
229 ({									\
230 	__typeof__(*(ptr)) _x_ = (x);					\
231 	(__typeof__(*(ptr))) ____xchg((ptr), (unsigned long)_x_,	\
232 					       sizeof(*(ptr)));		\
233 })
234 
235 #define arch_cmpxchg_local(ptr, o, n)					\
236 ({									\
237 	__typeof__(*(ptr)) _o_ = (o);					\
238 	__typeof__(*(ptr)) _n_ = (n);					\
239 	(__typeof__(*(ptr))) ____cmpxchg((ptr), (unsigned long)_o_,	\
240 					  (unsigned long)_n_,		\
241 					  sizeof(*(ptr)));		\
242 })
243 
244 #define arch_cmpxchg64_local(ptr, o, n)					\
245 ({									\
246 	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
247 	cmpxchg_local((ptr), (o), (n));					\
248 })
249 
250 /*
251  * The leading and the trailing memory barriers guarantee that these
252  * operations are fully ordered.
253  */
254 #define arch_xchg(ptr, x)						\
255 ({									\
256 	__typeof__(*(ptr)) __ret;					\
257 	__typeof__(*(ptr)) _x_ = (x);					\
258 	smp_mb();							\
259 	__ret = (__typeof__(*(ptr)))					\
260 		____xchg((ptr), (unsigned long)_x_, sizeof(*(ptr)));	\
261 	smp_mb();							\
262 	__ret;								\
263 })
264 
265 #define arch_cmpxchg(ptr, o, n)						\
266 ({									\
267 	__typeof__(*(ptr)) __ret;					\
268 	__typeof__(*(ptr)) _o_ = (o);					\
269 	__typeof__(*(ptr)) _n_ = (n);					\
270 	smp_mb();							\
271 	__ret = (__typeof__(*(ptr))) ____cmpxchg((ptr),			\
272 		(unsigned long)_o_, (unsigned long)_n_, sizeof(*(ptr)));\
273 	smp_mb();							\
274 	__ret;								\
275 })
276 
277 #define arch_cmpxchg64(ptr, o, n)					\
278 ({									\
279 	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
280 	arch_cmpxchg((ptr), (o), (n));					\
281 })
282 
283 #endif /* _ALPHA_CMPXCHG_H */
284