1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5 #ifndef __ASM_CMPXCHG_H
6 #define __ASM_CMPXCHG_H
7
8 #include <linux/bits.h>
9 #include <linux/build_bug.h>
10 #include <asm/barrier.h>
11 #include <asm/cpu-features.h>
12
13 #define __xchg_amo_asm(amswap_db, m, val) \
14 ({ \
15 __typeof(val) __ret; \
16 \
17 __asm__ __volatile__ ( \
18 " "amswap_db" %1, %z2, %0 \n" \
19 : "+ZB" (*m), "=&r" (__ret) \
20 : "Jr" (val) \
21 : "memory"); \
22 \
23 __ret; \
24 })
25
26 #define __xchg_llsc_asm(ld, st, m, val) \
27 ({ \
28 __typeof(val) __ret, __tmp; \
29 \
30 asm volatile ( \
31 "1: ll.w %0, %3 \n" \
32 " move %1, %z4 \n" \
33 " sc.w %1, %2 \n" \
34 " beqz %1, 1b \n" \
35 : "=&r" (__ret), "=&r" (__tmp), "=ZC" (*m) \
36 : "ZC" (*m), "Jr" (val) \
37 : "memory"); \
38 \
39 __ret; \
40 })
41
__xchg_small(volatile void * ptr,unsigned int val,unsigned int size)42 static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val,
43 unsigned int size)
44 {
45 unsigned int shift;
46 u32 old32, mask, temp;
47 volatile u32 *ptr32;
48
49 /* Mask value to the correct size. */
50 mask = GENMASK((size * BITS_PER_BYTE) - 1, 0);
51 val &= mask;
52
53 /*
54 * Calculate a shift & mask that correspond to the value we wish to
55 * exchange within the naturally aligned 4 byte integerthat includes
56 * it.
57 */
58 shift = (unsigned long)ptr & 0x3;
59 shift *= BITS_PER_BYTE;
60 mask <<= shift;
61
62 /*
63 * Calculate a pointer to the naturally aligned 4 byte integer that
64 * includes our byte of interest, and load its value.
65 */
66 ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3);
67
68 asm volatile (
69 "1: ll.w %0, %3 \n"
70 " andn %1, %0, %z4 \n"
71 " or %1, %1, %z5 \n"
72 " sc.w %1, %2 \n"
73 " beqz %1, 1b \n"
74 : "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32)
75 : "ZC" (*ptr32), "Jr" (mask), "Jr" (val << shift)
76 : "memory");
77
78 return (old32 & mask) >> shift;
79 }
80
81 static __always_inline unsigned long
__arch_xchg(volatile void * ptr,unsigned long x,int size)82 __arch_xchg(volatile void *ptr, unsigned long x, int size)
83 {
84 switch (size) {
85 case 1:
86 case 2:
87 return __xchg_small((volatile void *)ptr, x, size);
88
89 case 4:
90 #ifdef CONFIG_CPU_HAS_AMO
91 return __xchg_amo_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);
92 #else
93 return __xchg_llsc_asm("ll.w", "sc.w", (volatile u32 *)ptr, (u32)x);
94 #endif /* CONFIG_CPU_HAS_AMO */
95
96 #ifdef CONFIG_64BIT
97 case 8:
98 #ifdef CONFIG_CPU_HAS_AMO
99 return __xchg_amo_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);
100 #else
101 return __xchg_llsc_asm("ll.d", "sc.d", (volatile u64 *)ptr, (u64)x);
102 #endif /* CONFIG_CPU_HAS_AMO */
103 #endif /* CONFIG_64BIT */
104
105 default:
106 BUILD_BUG();
107 }
108
109 return 0;
110 }
111
112 #define arch_xchg(ptr, x) \
113 ({ \
114 __typeof__(*(ptr)) __res; \
115 \
116 __res = (__typeof__(*(ptr))) \
117 __arch_xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \
118 \
119 __res; \
120 })
121
122 #define __cmpxchg_asm(ld, st, m, old, new) \
123 ({ \
124 __typeof(old) __ret; \
125 \
126 __asm__ __volatile__( \
127 "1: " ld " %0, %2 # __cmpxchg_asm \n" \
128 " bne %0, %z3, 2f \n" \
129 " move $t0, %z4 \n" \
130 " " st " $t0, %1 \n" \
131 " beqz $t0, 1b \n" \
132 "2: \n" \
133 __WEAK_LLSC_MB \
134 : "=&r" (__ret), "=ZB"(*m) \
135 : "ZB"(*m), "Jr" (old), "Jr" (new) \
136 : "t0", "memory"); \
137 \
138 __ret; \
139 })
140
__cmpxchg_small(volatile void * ptr,unsigned int old,unsigned int new,unsigned int size)141 static inline unsigned int __cmpxchg_small(volatile void *ptr, unsigned int old,
142 unsigned int new, unsigned int size)
143 {
144 unsigned int shift;
145 u32 old32, mask, temp;
146 volatile u32 *ptr32;
147
148 /* Mask inputs to the correct size. */
149 mask = GENMASK((size * BITS_PER_BYTE) - 1, 0);
150 old &= mask;
151 new &= mask;
152
153 /*
154 * Calculate a shift & mask that correspond to the value we wish to
155 * compare & exchange within the naturally aligned 4 byte integer
156 * that includes it.
157 */
158 shift = (unsigned long)ptr & 0x3;
159 shift *= BITS_PER_BYTE;
160 old <<= shift;
161 new <<= shift;
162 mask <<= shift;
163
164 /*
165 * Calculate a pointer to the naturally aligned 4 byte integer that
166 * includes our byte of interest, and load its value.
167 */
168 ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3);
169
170 asm volatile (
171 "1: ll.w %0, %3 \n"
172 " and %1, %0, %z4 \n"
173 " bne %1, %z5, 2f \n"
174 " andn %1, %0, %z4 \n"
175 " or %1, %1, %z6 \n"
176 " sc.w %1, %2 \n"
177 " beqz %1, 1b \n"
178 " b 3f \n"
179 "2: \n"
180 __WEAK_LLSC_MB
181 "3: \n"
182 : "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32)
183 : "ZC" (*ptr32), "Jr" (mask), "Jr" (old), "Jr" (new)
184 : "memory");
185
186 return (old32 & mask) >> shift;
187 }
188
189 static __always_inline unsigned long
__cmpxchg(volatile void * ptr,unsigned long old,unsigned long new,unsigned int size)190 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size)
191 {
192 switch (size) {
193 case 1:
194 case 2:
195 return __cmpxchg_small(ptr, old, new, size);
196
197 case 4:
198 return __cmpxchg_asm("ll.w", "sc.w", (volatile u32 *)ptr,
199 (u32)old, new);
200
201 case 8:
202 return __cmpxchg_asm("ll.d", "sc.d", (volatile u64 *)ptr,
203 (u64)old, new);
204
205 default:
206 BUILD_BUG();
207 }
208
209 return 0;
210 }
211
212 #define arch_cmpxchg_local(ptr, old, new) \
213 ((__typeof__(*(ptr))) \
214 __cmpxchg((ptr), \
215 (unsigned long)(__typeof__(*(ptr)))(old), \
216 (unsigned long)(__typeof__(*(ptr)))(new), \
217 sizeof(*(ptr))))
218
219 #define arch_cmpxchg(ptr, old, new) \
220 ({ \
221 __typeof__(*(ptr)) __res; \
222 \
223 __res = arch_cmpxchg_local((ptr), (old), (new)); \
224 \
225 __res; \
226 })
227
228 #ifdef CONFIG_64BIT
229 #define arch_cmpxchg64_local(ptr, o, n) \
230 ({ \
231 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
232 arch_cmpxchg_local((ptr), (o), (n)); \
233 })
234
235 #define arch_cmpxchg64(ptr, o, n) \
236 ({ \
237 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
238 arch_cmpxchg((ptr), (o), (n)); \
239 })
240
241 #ifdef CONFIG_AS_HAS_SCQ_EXTENSION
242
243 union __u128_halves {
244 u128 full;
245 struct {
246 u64 low;
247 u64 high;
248 };
249 };
250
251 #define system_has_cmpxchg128() cpu_opt(LOONGARCH_CPU_SCQ)
252
253 #define __arch_cmpxchg128(ptr, old, new, llsc_mb) \
254 ({ \
255 union __u128_halves __old, __new, __ret; \
256 volatile u64 *__ptr = (volatile u64 *)(ptr); \
257 \
258 __old.full = (old); \
259 __new.full = (new); \
260 \
261 __asm__ __volatile__( \
262 "1: ll.d %0, %3 # 128-bit cmpxchg low \n" \
263 llsc_mb \
264 " ld.d %1, %4 # 128-bit cmpxchg high \n" \
265 " move $t0, %0 \n" \
266 " move $t1, %1 \n" \
267 " bne %0, %z5, 2f \n" \
268 " bne %1, %z6, 2f \n" \
269 " move $t0, %z7 \n" \
270 " move $t1, %z8 \n" \
271 "2: sc.q $t0, $t1, %2 \n" \
272 " beqz $t0, 1b \n" \
273 llsc_mb \
274 : "=&r" (__ret.low), "=&r" (__ret.high) \
275 : "r" (__ptr), \
276 "ZC" (__ptr[0]), "m" (__ptr[1]), \
277 "Jr" (__old.low), "Jr" (__old.high), \
278 "Jr" (__new.low), "Jr" (__new.high) \
279 : "t0", "t1", "memory"); \
280 \
281 __ret.full; \
282 })
283
284 #define arch_cmpxchg128(ptr, o, n) \
285 ({ \
286 BUILD_BUG_ON(sizeof(*(ptr)) != 16); \
287 __arch_cmpxchg128(ptr, o, n, __WEAK_LLSC_MB); \
288 })
289
290 #define arch_cmpxchg128_local(ptr, o, n) \
291 ({ \
292 BUILD_BUG_ON(sizeof(*(ptr)) != 16); \
293 __arch_cmpxchg128(ptr, o, n, ""); \
294 })
295
296 #endif /* CONFIG_AS_HAS_SCQ_EXTENSION */
297
298 #else
299 #include <asm-generic/cmpxchg-local.h>
300 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
301 #define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n))
302 #endif
303
304 #endif /* __ASM_CMPXCHG_H */
305