1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2014 Regents of the University of California
4 */
5
6 #ifndef _ASM_RISCV_CMPXCHG_H
7 #define _ASM_RISCV_CMPXCHG_H
8
9 #include <linux/bug.h>
10
11 #include <asm/alternative-macros.h>
12 #include <asm/fence.h>
13 #include <asm/hwcap.h>
14 #include <asm/insn-def.h>
15 #include <asm/cpufeature-macros.h>
16 #include <asm/processor.h>
17 #include <asm/errata_list.h>
18
19 #define __arch_xchg_masked(sc_sfx, swap_sfx, prepend, sc_append, \
20 swap_append, r, p, n) \
21 ({ \
22 if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
23 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \
24 __asm__ __volatile__ ( \
25 prepend \
26 " .option push\n" \
27 " .option arch, +zabha\n" \
28 " amoswap" swap_sfx " %0, %z2, %1\n" \
29 " .option pop\n" \
30 swap_append \
31 : "=&r" (r), "+A" (*(p)) \
32 : "rJ" (n) \
33 : "memory"); \
34 } else { \
35 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
36 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
37 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
38 << __s; \
39 ulong __newx = (ulong)(n) << __s; \
40 ulong __retx; \
41 ulong __rc; \
42 \
43 __asm__ __volatile__ ( \
44 prepend \
45 PREFETCHW_ASM(%5) \
46 "0: lr.w %0, %2\n" \
47 " and %1, %0, %z4\n" \
48 " or %1, %1, %z3\n" \
49 " sc.w" sc_sfx " %1, %1, %2\n" \
50 " bnez %1, 0b\n" \
51 sc_append \
52 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
53 : "rJ" (__newx), "rJ" (~__mask), "rJ" (__ptr32b) \
54 : "memory"); \
55 \
56 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
57 } \
58 })
59
60 #define __arch_xchg(sfx, prepend, append, r, p, n) \
61 ({ \
62 __asm__ __volatile__ ( \
63 prepend \
64 " amoswap" sfx " %0, %2, %1\n" \
65 append \
66 : "=r" (r), "+A" (*(p)) \
67 : "r" (n) \
68 : "memory"); \
69 })
70
71 #define _arch_xchg(ptr, new, sc_sfx, swap_sfx, prepend, \
72 sc_append, swap_append) \
73 ({ \
74 __typeof__(ptr) __ptr = (ptr); \
75 __typeof__(*(__ptr)) __new = (new); \
76 __typeof__(*(__ptr)) __ret; \
77 \
78 switch (sizeof(*__ptr)) { \
79 case 1: \
80 __arch_xchg_masked(sc_sfx, ".b" swap_sfx, \
81 prepend, sc_append, swap_append, \
82 __ret, __ptr, __new); \
83 break; \
84 case 2: \
85 __arch_xchg_masked(sc_sfx, ".h" swap_sfx, \
86 prepend, sc_append, swap_append, \
87 __ret, __ptr, __new); \
88 break; \
89 case 4: \
90 __arch_xchg(".w" swap_sfx, prepend, swap_append, \
91 __ret, __ptr, __new); \
92 break; \
93 case 8: \
94 __arch_xchg(".d" swap_sfx, prepend, swap_append, \
95 __ret, __ptr, __new); \
96 break; \
97 default: \
98 BUILD_BUG(); \
99 } \
100 (__typeof__(*(__ptr)))__ret; \
101 })
102
103 #define arch_xchg_relaxed(ptr, x) \
104 _arch_xchg(ptr, x, "", "", "", "", "")
105
106 #define arch_xchg_acquire(ptr, x) \
107 _arch_xchg(ptr, x, "", "", "", \
108 RISCV_ACQUIRE_BARRIER, RISCV_ACQUIRE_BARRIER)
109
110 #define arch_xchg_release(ptr, x) \
111 _arch_xchg(ptr, x, "", "", RISCV_RELEASE_BARRIER, "", "")
112
113 #define arch_xchg(ptr, x) \
114 _arch_xchg(ptr, x, ".rl", ".aqrl", "", RISCV_FULL_BARRIER, "")
115
116 #define xchg32(ptr, x) \
117 ({ \
118 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
119 arch_xchg((ptr), (x)); \
120 })
121
122 #define xchg64(ptr, x) \
123 ({ \
124 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
125 arch_xchg((ptr), (x)); \
126 })
127
128 /*
129 * Atomic compare and exchange. Compare OLD with MEM, if identical,
130 * store NEW in MEM. Return the initial value in MEM. Success is
131 * indicated by comparing RETURN with OLD.
132 */
133 #define __arch_cmpxchg_masked(sc_sfx, cas_sfx, \
134 sc_prepend, sc_append, \
135 cas_prepend, cas_append, \
136 r, p, o, n) \
137 ({ \
138 if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
139 IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
140 IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \
141 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA) && \
142 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
143 r = o; \
144 \
145 __asm__ __volatile__ ( \
146 cas_prepend \
147 " .option push\n" \
148 " .option arch, +zacas, +zabha\n" \
149 " amocas" cas_sfx " %0, %z2, %1\n" \
150 " .option pop\n" \
151 cas_append \
152 : "+&r" (r), "+A" (*(p)) \
153 : "rJ" (n) \
154 : "memory"); \
155 } else { \
156 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
157 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
158 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
159 << __s; \
160 ulong __newx = (ulong)(n) << __s; \
161 ulong __oldx = (ulong)(o) << __s; \
162 ulong __retx; \
163 ulong __rc; \
164 \
165 __asm__ __volatile__ ( \
166 sc_prepend \
167 "0: lr.w %0, %2\n" \
168 " and %1, %0, %z5\n" \
169 " bne %1, %z3, 1f\n" \
170 " and %1, %0, %z6\n" \
171 " or %1, %1, %z4\n" \
172 " sc.w" sc_sfx " %1, %1, %2\n" \
173 " bnez %1, 0b\n" \
174 sc_append \
175 "1:\n" \
176 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
177 : "rJ" ((long)__oldx), "rJ" (__newx), \
178 "rJ" (__mask), "rJ" (~__mask) \
179 : "memory"); \
180 \
181 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
182 } \
183 })
184
185 #define __arch_cmpxchg(lr_sfx, sc_sfx, cas_sfx, \
186 sc_prepend, sc_append, \
187 cas_prepend, cas_append, \
188 r, p, co, o, n) \
189 ({ \
190 if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
191 IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \
192 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
193 r = o; \
194 \
195 __asm__ __volatile__ ( \
196 cas_prepend \
197 " .option push\n" \
198 " .option arch, +zacas\n" \
199 " amocas" cas_sfx " %0, %z2, %1\n" \
200 " .option pop\n" \
201 cas_append \
202 : "+&r" (r), "+A" (*(p)) \
203 : "rJ" (n) \
204 : "memory"); \
205 } else { \
206 register unsigned int __rc; \
207 \
208 __asm__ __volatile__ ( \
209 sc_prepend \
210 "0: lr" lr_sfx " %0, %2\n" \
211 " bne %0, %z3, 1f\n" \
212 " sc" sc_sfx " %1, %z4, %2\n" \
213 " bnez %1, 0b\n" \
214 sc_append \
215 "1:\n" \
216 : "=&r" (r), "=&r" (__rc), "+A" (*(p)) \
217 : "rJ" (co o), "rJ" (n) \
218 : "memory"); \
219 } \
220 })
221
222 #define _arch_cmpxchg(ptr, old, new, sc_sfx, cas_sfx, \
223 sc_prepend, sc_append, \
224 cas_prepend, cas_append) \
225 ({ \
226 __typeof__(ptr) __ptr = (ptr); \
227 __typeof__(*(__ptr)) __old = (old); \
228 __typeof__(*(__ptr)) __new = (new); \
229 __typeof__(*(__ptr)) __ret; \
230 \
231 switch (sizeof(*__ptr)) { \
232 case 1: \
233 __arch_cmpxchg_masked(sc_sfx, ".b" cas_sfx, \
234 sc_prepend, sc_append, \
235 cas_prepend, cas_append, \
236 __ret, __ptr, __old, __new); \
237 break; \
238 case 2: \
239 __arch_cmpxchg_masked(sc_sfx, ".h" cas_sfx, \
240 sc_prepend, sc_append, \
241 cas_prepend, cas_append, \
242 __ret, __ptr, __old, __new); \
243 break; \
244 case 4: \
245 __arch_cmpxchg(".w", ".w" sc_sfx, ".w" cas_sfx, \
246 sc_prepend, sc_append, \
247 cas_prepend, cas_append, \
248 __ret, __ptr, (long)(int)(long), __old, __new); \
249 break; \
250 case 8: \
251 __arch_cmpxchg(".d", ".d" sc_sfx, ".d" cas_sfx, \
252 sc_prepend, sc_append, \
253 cas_prepend, cas_append, \
254 __ret, __ptr, /**/, __old, __new); \
255 break; \
256 default: \
257 BUILD_BUG(); \
258 } \
259 (__typeof__(*(__ptr)))__ret; \
260 })
261
262 /*
263 * These macros are here to improve the readability of the arch_cmpxchg_XXX()
264 * macros.
265 */
266 #define SC_SFX(x) x
267 #define CAS_SFX(x) x
268 #define SC_PREPEND(x) x
269 #define SC_APPEND(x) x
270 #define CAS_PREPEND(x) x
271 #define CAS_APPEND(x) x
272
273 #define arch_cmpxchg_relaxed(ptr, o, n) \
274 _arch_cmpxchg((ptr), (o), (n), \
275 SC_SFX(""), CAS_SFX(""), \
276 SC_PREPEND(""), SC_APPEND(""), \
277 CAS_PREPEND(""), CAS_APPEND(""))
278
279 #define arch_cmpxchg_acquire(ptr, o, n) \
280 _arch_cmpxchg((ptr), (o), (n), \
281 SC_SFX(""), CAS_SFX(""), \
282 SC_PREPEND(""), SC_APPEND(RISCV_ACQUIRE_BARRIER), \
283 CAS_PREPEND(""), CAS_APPEND(RISCV_ACQUIRE_BARRIER))
284
285 #define arch_cmpxchg_release(ptr, o, n) \
286 _arch_cmpxchg((ptr), (o), (n), \
287 SC_SFX(""), CAS_SFX(""), \
288 SC_PREPEND(RISCV_RELEASE_BARRIER), SC_APPEND(""), \
289 CAS_PREPEND(RISCV_RELEASE_BARRIER), CAS_APPEND(""))
290
291 #define arch_cmpxchg(ptr, o, n) \
292 _arch_cmpxchg((ptr), (o), (n), \
293 SC_SFX(".rl"), CAS_SFX(".aqrl"), \
294 SC_PREPEND(""), SC_APPEND(RISCV_FULL_BARRIER), \
295 CAS_PREPEND(""), CAS_APPEND(""))
296
297 #define arch_cmpxchg_local(ptr, o, n) \
298 arch_cmpxchg_relaxed((ptr), (o), (n))
299
300 #define arch_cmpxchg64(ptr, o, n) \
301 ({ \
302 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
303 arch_cmpxchg((ptr), (o), (n)); \
304 })
305
306 #define arch_cmpxchg64_local(ptr, o, n) \
307 ({ \
308 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
309 arch_cmpxchg_relaxed((ptr), (o), (n)); \
310 })
311
312 #define arch_cmpxchg64_relaxed(ptr, o, n) \
313 ({ \
314 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
315 arch_cmpxchg_relaxed((ptr), (o), (n)); \
316 })
317
318 #define arch_cmpxchg64_acquire(ptr, o, n) \
319 ({ \
320 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
321 arch_cmpxchg_acquire((ptr), (o), (n)); \
322 })
323
324 #define arch_cmpxchg64_release(ptr, o, n) \
325 ({ \
326 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
327 arch_cmpxchg_release((ptr), (o), (n)); \
328 })
329
330 #if defined(CONFIG_64BIT) && defined(CONFIG_RISCV_ISA_ZACAS) && defined(CONFIG_TOOLCHAIN_HAS_ZACAS)
331
332 #define system_has_cmpxchg128() riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)
333
334 union __u128_halves {
335 u128 full;
336 struct {
337 u64 low, high;
338 };
339 };
340
341 #define __arch_cmpxchg128(p, o, n, cas_sfx) \
342 ({ \
343 __typeof__(*(p)) __o = (o); \
344 union __u128_halves __hn = { .full = (n) }; \
345 union __u128_halves __ho = { .full = (__o) }; \
346 register unsigned long t1 asm ("t1") = __hn.low; \
347 register unsigned long t2 asm ("t2") = __hn.high; \
348 register unsigned long t3 asm ("t3") = __ho.low; \
349 register unsigned long t4 asm ("t4") = __ho.high; \
350 \
351 __asm__ __volatile__ ( \
352 " .option push\n" \
353 " .option arch, +zacas\n" \
354 " amocas.q" cas_sfx " %0, %z3, %2\n" \
355 " .option pop\n" \
356 : "+&r" (t3), "+&r" (t4), "+A" (*(p)) \
357 : "rJ" (t1), "rJ" (t2) \
358 : "memory"); \
359 \
360 ((u128)t4 << 64) | t3; \
361 })
362
363 #define arch_cmpxchg128(ptr, o, n) \
364 __arch_cmpxchg128((ptr), (o), (n), ".aqrl")
365
366 #define arch_cmpxchg128_local(ptr, o, n) \
367 __arch_cmpxchg128((ptr), (o), (n), "")
368
369 #endif /* CONFIG_64BIT && CONFIG_RISCV_ISA_ZACAS && CONFIG_TOOLCHAIN_HAS_ZACAS */
370
371 #ifdef CONFIG_RISCV_ISA_ZAWRS
372 /*
373 * Despite wrs.nto being "WRS-with-no-timeout", in the absence of changes to
374 * @val we expect it to still terminate within a "reasonable" amount of time
375 * for an implementation-specific other reason, a pending, locally-enabled
376 * interrupt, or because it has been configured to raise an illegal
377 * instruction exception.
378 */
__cmpwait(volatile void * ptr,unsigned long val,int size)379 static __always_inline void __cmpwait(volatile void *ptr,
380 unsigned long val,
381 int size)
382 {
383 unsigned long tmp;
384
385 u32 *__ptr32b;
386 ulong __s, __val, __mask;
387
388 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZAWRS)) {
389 ALT_RISCV_PAUSE();
390 return;
391 }
392
393 switch (size) {
394 case 1:
395 __ptr32b = (u32 *)((ulong)(ptr) & ~0x3);
396 __s = ((ulong)(ptr) & 0x3) * BITS_PER_BYTE;
397 __val = val << __s;
398 __mask = 0xff << __s;
399
400 asm volatile(
401 " lr.w %0, %1\n"
402 " and %0, %0, %3\n"
403 " xor %0, %0, %2\n"
404 " bnez %0, 1f\n"
405 ZAWRS_WRS_NTO "\n"
406 "1:"
407 : "=&r" (tmp), "+A" (*(__ptr32b))
408 : "r" (__val), "r" (__mask)
409 : "memory");
410 break;
411 case 2:
412 __ptr32b = (u32 *)((ulong)(ptr) & ~0x3);
413 __s = ((ulong)(ptr) & 0x2) * BITS_PER_BYTE;
414 __val = val << __s;
415 __mask = 0xffff << __s;
416
417 asm volatile(
418 " lr.w %0, %1\n"
419 " and %0, %0, %3\n"
420 " xor %0, %0, %2\n"
421 " bnez %0, 1f\n"
422 ZAWRS_WRS_NTO "\n"
423 "1:"
424 : "=&r" (tmp), "+A" (*(__ptr32b))
425 : "r" (__val), "r" (__mask)
426 : "memory");
427 break;
428 case 4:
429 asm volatile(
430 " lr.w %0, %1\n"
431 " xor %0, %0, %2\n"
432 " bnez %0, 1f\n"
433 ZAWRS_WRS_NTO "\n"
434 "1:"
435 : "=&r" (tmp), "+A" (*(u32 *)ptr)
436 : "r" (val));
437 break;
438 #if __riscv_xlen == 64
439 case 8:
440 asm volatile(
441 " lr.d %0, %1\n"
442 " xor %0, %0, %2\n"
443 " bnez %0, 1f\n"
444 ZAWRS_WRS_NTO "\n"
445 "1:"
446 : "=&r" (tmp), "+A" (*(u64 *)ptr)
447 : "r" (val));
448 break;
449 #endif
450 default:
451 BUILD_BUG();
452 }
453 }
454
455 #define __cmpwait_relaxed(ptr, val) \
456 __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr)))
457 #endif
458
459 #endif /* _ASM_RISCV_CMPXCHG_H */
460