150acfb2bSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-only */
2fab957c1SPalmer Dabbelt /*
3fab957c1SPalmer Dabbelt * Copyright (C) 2014 Regents of the University of California
4fab957c1SPalmer Dabbelt */
5fab957c1SPalmer Dabbelt
6fab957c1SPalmer Dabbelt #ifndef _ASM_RISCV_CMPXCHG_H
7fab957c1SPalmer Dabbelt #define _ASM_RISCV_CMPXCHG_H
8fab957c1SPalmer Dabbelt
9fab957c1SPalmer Dabbelt #include <linux/bug.h>
10fab957c1SPalmer Dabbelt
11*b8ddb0dfSChristoph Müllner #include <asm/alternative-macros.h>
125ce6c1f3SAndrea Parri #include <asm/fence.h>
13*b8ddb0dfSChristoph Müllner #include <asm/hwcap.h>
14*b8ddb0dfSChristoph Müllner #include <asm/insn-def.h>
15fab957c1SPalmer Dabbelt
161d84afafSAlexandre Ghiti #define __arch_xchg_masked(sc_sfx, prepend, append, r, p, n) \
17a8ed2b7aSLeonardo Bras ({ \
18a8ed2b7aSLeonardo Bras u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
19a8ed2b7aSLeonardo Bras ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
20a8ed2b7aSLeonardo Bras ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
21a8ed2b7aSLeonardo Bras << __s; \
22a8ed2b7aSLeonardo Bras ulong __newx = (ulong)(n) << __s; \
23a8ed2b7aSLeonardo Bras ulong __retx; \
24a8ed2b7aSLeonardo Bras ulong __rc; \
25a8ed2b7aSLeonardo Bras \
26a8ed2b7aSLeonardo Bras __asm__ __volatile__ ( \
27a8ed2b7aSLeonardo Bras prepend \
28a8ed2b7aSLeonardo Bras "0: lr.w %0, %2\n" \
29a8ed2b7aSLeonardo Bras " and %1, %0, %z4\n" \
30a8ed2b7aSLeonardo Bras " or %1, %1, %z3\n" \
311d84afafSAlexandre Ghiti " sc.w" sc_sfx " %1, %1, %2\n" \
32a8ed2b7aSLeonardo Bras " bnez %1, 0b\n" \
33a8ed2b7aSLeonardo Bras append \
34a8ed2b7aSLeonardo Bras : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
35a8ed2b7aSLeonardo Bras : "rJ" (__newx), "rJ" (~__mask) \
36a8ed2b7aSLeonardo Bras : "memory"); \
37a8ed2b7aSLeonardo Bras \
38a8ed2b7aSLeonardo Bras r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
39a8ed2b7aSLeonardo Bras })
40a8ed2b7aSLeonardo Bras
414bfa185fSLeonardo Bras #define __arch_xchg(sfx, prepend, append, r, p, n) \
424bfa185fSLeonardo Bras ({ \
434bfa185fSLeonardo Bras __asm__ __volatile__ ( \
444bfa185fSLeonardo Bras prepend \
454bfa185fSLeonardo Bras " amoswap" sfx " %0, %2, %1\n" \
464bfa185fSLeonardo Bras append \
474bfa185fSLeonardo Bras : "=r" (r), "+A" (*(p)) \
484bfa185fSLeonardo Bras : "r" (n) \
494bfa185fSLeonardo Bras : "memory"); \
504bfa185fSLeonardo Bras })
514bfa185fSLeonardo Bras
521d84afafSAlexandre Ghiti #define _arch_xchg(ptr, new, sc_sfx, swap_sfx, prepend, \
531d84afafSAlexandre Ghiti sc_append, swap_append) \
54fab957c1SPalmer Dabbelt ({ \
55fab957c1SPalmer Dabbelt __typeof__(ptr) __ptr = (ptr); \
564bfa185fSLeonardo Bras __typeof__(*(__ptr)) __new = (new); \
574bfa185fSLeonardo Bras __typeof__(*(__ptr)) __ret; \
58a8ed2b7aSLeonardo Bras \
594bfa185fSLeonardo Bras switch (sizeof(*__ptr)) { \
60a8ed2b7aSLeonardo Bras case 1: \
61a8ed2b7aSLeonardo Bras case 2: \
621d84afafSAlexandre Ghiti __arch_xchg_masked(sc_sfx, prepend, sc_append, \
63a8ed2b7aSLeonardo Bras __ret, __ptr, __new); \
64a8ed2b7aSLeonardo Bras break; \
65fab957c1SPalmer Dabbelt case 4: \
661d84afafSAlexandre Ghiti __arch_xchg(".w" swap_sfx, prepend, swap_append, \
674bfa185fSLeonardo Bras __ret, __ptr, __new); \
68fab957c1SPalmer Dabbelt break; \
69fab957c1SPalmer Dabbelt case 8: \
701d84afafSAlexandre Ghiti __arch_xchg(".d" swap_sfx, prepend, swap_append, \
714bfa185fSLeonardo Bras __ret, __ptr, __new); \
72fab957c1SPalmer Dabbelt break; \
73fab957c1SPalmer Dabbelt default: \
74fab957c1SPalmer Dabbelt BUILD_BUG(); \
75fab957c1SPalmer Dabbelt } \
764bfa185fSLeonardo Bras (__typeof__(*(__ptr)))__ret; \
77fab957c1SPalmer Dabbelt })
78fab957c1SPalmer Dabbelt
799efbb355SMark Rutland #define arch_xchg_relaxed(ptr, x) \
801d84afafSAlexandre Ghiti _arch_xchg(ptr, x, "", "", "", "", "")
815ce6c1f3SAndrea Parri
829efbb355SMark Rutland #define arch_xchg_acquire(ptr, x) \
831d84afafSAlexandre Ghiti _arch_xchg(ptr, x, "", "", "", \
841d84afafSAlexandre Ghiti RISCV_ACQUIRE_BARRIER, RISCV_ACQUIRE_BARRIER)
855ce6c1f3SAndrea Parri
869efbb355SMark Rutland #define arch_xchg_release(ptr, x) \
871d84afafSAlexandre Ghiti _arch_xchg(ptr, x, "", "", RISCV_RELEASE_BARRIER, "", "")
885ce6c1f3SAndrea Parri
899efbb355SMark Rutland #define arch_xchg(ptr, x) \
901d84afafSAlexandre Ghiti _arch_xchg(ptr, x, ".rl", ".aqrl", "", RISCV_FULL_BARRIER, "")
91fab957c1SPalmer Dabbelt
92fab957c1SPalmer Dabbelt #define xchg32(ptr, x) \
93fab957c1SPalmer Dabbelt ({ \
94fab957c1SPalmer Dabbelt BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
959efbb355SMark Rutland arch_xchg((ptr), (x)); \
96fab957c1SPalmer Dabbelt })
97fab957c1SPalmer Dabbelt
98fab957c1SPalmer Dabbelt #define xchg64(ptr, x) \
99fab957c1SPalmer Dabbelt ({ \
100fab957c1SPalmer Dabbelt BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
1019efbb355SMark Rutland arch_xchg((ptr), (x)); \
102fab957c1SPalmer Dabbelt })
103fab957c1SPalmer Dabbelt
104fab957c1SPalmer Dabbelt /*
105fab957c1SPalmer Dabbelt * Atomic compare and exchange. Compare OLD with MEM, if identical,
106fab957c1SPalmer Dabbelt * store NEW in MEM. Return the initial value in MEM. Success is
107fab957c1SPalmer Dabbelt * indicated by comparing RETURN with OLD.
108fab957c1SPalmer Dabbelt */
10907a0a41cSLeonardo Bras
11054280ca6SLeonardo Bras #define __arch_cmpxchg_masked(sc_sfx, prepend, append, r, p, o, n) \
11154280ca6SLeonardo Bras ({ \
11254280ca6SLeonardo Bras u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
11354280ca6SLeonardo Bras ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
11454280ca6SLeonardo Bras ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
11554280ca6SLeonardo Bras << __s; \
11654280ca6SLeonardo Bras ulong __newx = (ulong)(n) << __s; \
11754280ca6SLeonardo Bras ulong __oldx = (ulong)(o) << __s; \
11854280ca6SLeonardo Bras ulong __retx; \
11954280ca6SLeonardo Bras ulong __rc; \
12054280ca6SLeonardo Bras \
12154280ca6SLeonardo Bras __asm__ __volatile__ ( \
12254280ca6SLeonardo Bras prepend \
12354280ca6SLeonardo Bras "0: lr.w %0, %2\n" \
12454280ca6SLeonardo Bras " and %1, %0, %z5\n" \
12554280ca6SLeonardo Bras " bne %1, %z3, 1f\n" \
12654280ca6SLeonardo Bras " and %1, %0, %z6\n" \
12754280ca6SLeonardo Bras " or %1, %1, %z4\n" \
12854280ca6SLeonardo Bras " sc.w" sc_sfx " %1, %1, %2\n" \
12954280ca6SLeonardo Bras " bnez %1, 0b\n" \
13054280ca6SLeonardo Bras append \
13154280ca6SLeonardo Bras "1:\n" \
13254280ca6SLeonardo Bras : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
13354280ca6SLeonardo Bras : "rJ" ((long)__oldx), "rJ" (__newx), \
13454280ca6SLeonardo Bras "rJ" (__mask), "rJ" (~__mask) \
13554280ca6SLeonardo Bras : "memory"); \
13654280ca6SLeonardo Bras \
13754280ca6SLeonardo Bras r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
13854280ca6SLeonardo Bras })
13907a0a41cSLeonardo Bras
14007a0a41cSLeonardo Bras #define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n) \
14107a0a41cSLeonardo Bras ({ \
14207a0a41cSLeonardo Bras register unsigned int __rc; \
14307a0a41cSLeonardo Bras \
14407a0a41cSLeonardo Bras __asm__ __volatile__ ( \
14507a0a41cSLeonardo Bras prepend \
14607a0a41cSLeonardo Bras "0: lr" lr_sfx " %0, %2\n" \
14707a0a41cSLeonardo Bras " bne %0, %z3, 1f\n" \
14807a0a41cSLeonardo Bras " sc" sc_sfx " %1, %z4, %2\n" \
14907a0a41cSLeonardo Bras " bnez %1, 0b\n" \
15007a0a41cSLeonardo Bras append \
15107a0a41cSLeonardo Bras "1:\n" \
15207a0a41cSLeonardo Bras : "=&r" (r), "=&r" (__rc), "+A" (*(p)) \
15307a0a41cSLeonardo Bras : "rJ" (co o), "rJ" (n) \
15407a0a41cSLeonardo Bras : "memory"); \
15507a0a41cSLeonardo Bras })
15607a0a41cSLeonardo Bras
15707a0a41cSLeonardo Bras #define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append) \
158fab957c1SPalmer Dabbelt ({ \
159fab957c1SPalmer Dabbelt __typeof__(ptr) __ptr = (ptr); \
16007a0a41cSLeonardo Bras __typeof__(*(__ptr)) __old = (old); \
16107a0a41cSLeonardo Bras __typeof__(*(__ptr)) __new = (new); \
16207a0a41cSLeonardo Bras __typeof__(*(__ptr)) __ret; \
16307a0a41cSLeonardo Bras \
16407a0a41cSLeonardo Bras switch (sizeof(*__ptr)) { \
16554280ca6SLeonardo Bras case 1: \
16654280ca6SLeonardo Bras case 2: \
16754280ca6SLeonardo Bras __arch_cmpxchg_masked(sc_sfx, prepend, append, \
16854280ca6SLeonardo Bras __ret, __ptr, __old, __new); \
16954280ca6SLeonardo Bras break; \
170fab957c1SPalmer Dabbelt case 4: \
17107a0a41cSLeonardo Bras __arch_cmpxchg(".w", ".w" sc_sfx, prepend, append, \
17207a0a41cSLeonardo Bras __ret, __ptr, (long), __old, __new); \
173fab957c1SPalmer Dabbelt break; \
174fab957c1SPalmer Dabbelt case 8: \
17507a0a41cSLeonardo Bras __arch_cmpxchg(".d", ".d" sc_sfx, prepend, append, \
17607a0a41cSLeonardo Bras __ret, __ptr, /**/, __old, __new); \
1775ce6c1f3SAndrea Parri break; \
1785ce6c1f3SAndrea Parri default: \
1795ce6c1f3SAndrea Parri BUILD_BUG(); \
1805ce6c1f3SAndrea Parri } \
18107a0a41cSLeonardo Bras (__typeof__(*(__ptr)))__ret; \
1825ce6c1f3SAndrea Parri })
1835ce6c1f3SAndrea Parri
1849efbb355SMark Rutland #define arch_cmpxchg_relaxed(ptr, o, n) \
18507a0a41cSLeonardo Bras _arch_cmpxchg((ptr), (o), (n), "", "", "")
1865ce6c1f3SAndrea Parri
1879efbb355SMark Rutland #define arch_cmpxchg_acquire(ptr, o, n) \
18807a0a41cSLeonardo Bras _arch_cmpxchg((ptr), (o), (n), "", "", RISCV_ACQUIRE_BARRIER)
1895ce6c1f3SAndrea Parri
1909efbb355SMark Rutland #define arch_cmpxchg_release(ptr, o, n) \
19107a0a41cSLeonardo Bras _arch_cmpxchg((ptr), (o), (n), "", RISCV_RELEASE_BARRIER, "")
192fab957c1SPalmer Dabbelt
1939efbb355SMark Rutland #define arch_cmpxchg(ptr, o, n) \
19407a0a41cSLeonardo Bras _arch_cmpxchg((ptr), (o), (n), ".rl", "", " fence rw, rw\n")
195fab957c1SPalmer Dabbelt
1969efbb355SMark Rutland #define arch_cmpxchg_local(ptr, o, n) \
19707a0a41cSLeonardo Bras arch_cmpxchg_relaxed((ptr), (o), (n))
198fab957c1SPalmer Dabbelt
1999efbb355SMark Rutland #define arch_cmpxchg64(ptr, o, n) \
200fab957c1SPalmer Dabbelt ({ \
201fab957c1SPalmer Dabbelt BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
2029efbb355SMark Rutland arch_cmpxchg((ptr), (o), (n)); \
203fab957c1SPalmer Dabbelt })
204fab957c1SPalmer Dabbelt
2059efbb355SMark Rutland #define arch_cmpxchg64_local(ptr, o, n) \
206fab957c1SPalmer Dabbelt ({ \
207fab957c1SPalmer Dabbelt BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
2089efbb355SMark Rutland arch_cmpxchg_relaxed((ptr), (o), (n)); \
209fab957c1SPalmer Dabbelt })
210fab957c1SPalmer Dabbelt
21179d6e4eaSJisheng Zhang #define arch_cmpxchg64_relaxed(ptr, o, n) \
21279d6e4eaSJisheng Zhang ({ \
21379d6e4eaSJisheng Zhang BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
21479d6e4eaSJisheng Zhang arch_cmpxchg_relaxed((ptr), (o), (n)); \
21579d6e4eaSJisheng Zhang })
21679d6e4eaSJisheng Zhang
21779d6e4eaSJisheng Zhang #define arch_cmpxchg64_acquire(ptr, o, n) \
21879d6e4eaSJisheng Zhang ({ \
21979d6e4eaSJisheng Zhang BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
22079d6e4eaSJisheng Zhang arch_cmpxchg_acquire((ptr), (o), (n)); \
22179d6e4eaSJisheng Zhang })
22279d6e4eaSJisheng Zhang
22379d6e4eaSJisheng Zhang #define arch_cmpxchg64_release(ptr, o, n) \
22479d6e4eaSJisheng Zhang ({ \
22579d6e4eaSJisheng Zhang BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
22679d6e4eaSJisheng Zhang arch_cmpxchg_release((ptr), (o), (n)); \
22779d6e4eaSJisheng Zhang })
22879d6e4eaSJisheng Zhang
229*b8ddb0dfSChristoph Müllner #ifdef CONFIG_RISCV_ISA_ZAWRS
230*b8ddb0dfSChristoph Müllner /*
231*b8ddb0dfSChristoph Müllner * Despite wrs.nto being "WRS-with-no-timeout", in the absence of changes to
232*b8ddb0dfSChristoph Müllner * @val we expect it to still terminate within a "reasonable" amount of time
233*b8ddb0dfSChristoph Müllner * for an implementation-specific other reason, a pending, locally-enabled
234*b8ddb0dfSChristoph Müllner * interrupt, or because it has been configured to raise an illegal
235*b8ddb0dfSChristoph Müllner * instruction exception.
236*b8ddb0dfSChristoph Müllner */
__cmpwait(volatile void * ptr,unsigned long val,int size)237*b8ddb0dfSChristoph Müllner static __always_inline void __cmpwait(volatile void *ptr,
238*b8ddb0dfSChristoph Müllner unsigned long val,
239*b8ddb0dfSChristoph Müllner int size)
240*b8ddb0dfSChristoph Müllner {
241*b8ddb0dfSChristoph Müllner unsigned long tmp;
242*b8ddb0dfSChristoph Müllner
243*b8ddb0dfSChristoph Müllner asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop",
244*b8ddb0dfSChristoph Müllner 0, RISCV_ISA_EXT_ZAWRS, 1)
245*b8ddb0dfSChristoph Müllner : : : : no_zawrs);
246*b8ddb0dfSChristoph Müllner
247*b8ddb0dfSChristoph Müllner switch (size) {
248*b8ddb0dfSChristoph Müllner case 4:
249*b8ddb0dfSChristoph Müllner asm volatile(
250*b8ddb0dfSChristoph Müllner " lr.w %0, %1\n"
251*b8ddb0dfSChristoph Müllner " xor %0, %0, %2\n"
252*b8ddb0dfSChristoph Müllner " bnez %0, 1f\n"
253*b8ddb0dfSChristoph Müllner ZAWRS_WRS_NTO "\n"
254*b8ddb0dfSChristoph Müllner "1:"
255*b8ddb0dfSChristoph Müllner : "=&r" (tmp), "+A" (*(u32 *)ptr)
256*b8ddb0dfSChristoph Müllner : "r" (val));
257*b8ddb0dfSChristoph Müllner break;
258*b8ddb0dfSChristoph Müllner #if __riscv_xlen == 64
259*b8ddb0dfSChristoph Müllner case 8:
260*b8ddb0dfSChristoph Müllner asm volatile(
261*b8ddb0dfSChristoph Müllner " lr.d %0, %1\n"
262*b8ddb0dfSChristoph Müllner " xor %0, %0, %2\n"
263*b8ddb0dfSChristoph Müllner " bnez %0, 1f\n"
264*b8ddb0dfSChristoph Müllner ZAWRS_WRS_NTO "\n"
265*b8ddb0dfSChristoph Müllner "1:"
266*b8ddb0dfSChristoph Müllner : "=&r" (tmp), "+A" (*(u64 *)ptr)
267*b8ddb0dfSChristoph Müllner : "r" (val));
268*b8ddb0dfSChristoph Müllner break;
269*b8ddb0dfSChristoph Müllner #endif
270*b8ddb0dfSChristoph Müllner default:
271*b8ddb0dfSChristoph Müllner BUILD_BUG();
272*b8ddb0dfSChristoph Müllner }
273*b8ddb0dfSChristoph Müllner
274*b8ddb0dfSChristoph Müllner return;
275*b8ddb0dfSChristoph Müllner
276*b8ddb0dfSChristoph Müllner no_zawrs:
277*b8ddb0dfSChristoph Müllner asm volatile(RISCV_PAUSE : : : "memory");
278*b8ddb0dfSChristoph Müllner }
279*b8ddb0dfSChristoph Müllner
280*b8ddb0dfSChristoph Müllner #define __cmpwait_relaxed(ptr, val) \
281*b8ddb0dfSChristoph Müllner __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr)))
282*b8ddb0dfSChristoph Müllner #endif
283*b8ddb0dfSChristoph Müllner
284fab957c1SPalmer Dabbelt #endif /* _ASM_RISCV_CMPXCHG_H */
285