1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2014 Regents of the University of California 4 */ 5 6 #ifndef _ASM_RISCV_CMPXCHG_H 7 #define _ASM_RISCV_CMPXCHG_H 8 9 #include <linux/bug.h> 10 11 #include <asm/alternative-macros.h> 12 #include <asm/fence.h> 13 #include <asm/hwcap.h> 14 #include <asm/insn-def.h> 15 #include <asm/cpufeature-macros.h> 16 #include <asm/processor.h> 17 #include <asm/errata_list.h> 18 19 #define __arch_xchg_masked(sc_sfx, swap_sfx, prepend, sc_append, \ 20 swap_append, r, p, n) \ 21 ({ \ 22 if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \ 23 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \ 24 __asm__ __volatile__ ( \ 25 prepend \ 26 " .option push\n" \ 27 " .option arch, +zabha\n" \ 28 " amoswap" swap_sfx " %0, %z2, %1\n" \ 29 " .option pop\n" \ 30 swap_append \ 31 : "=&r" (r), "+A" (*(p)) \ 32 : "rJ" (n) \ 33 : "memory"); \ 34 } else { \ 35 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \ 36 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \ 37 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \ 38 << __s; \ 39 ulong __newx = (ulong)(n) << __s; \ 40 ulong __retx; \ 41 ulong __rc; \ 42 \ 43 __asm__ __volatile__ ( \ 44 prepend \ 45 PREFETCHW_ASM(%5) \ 46 "0: lr.w %0, %2\n" \ 47 " and %1, %0, %z4\n" \ 48 " or %1, %1, %z3\n" \ 49 " sc.w" sc_sfx " %1, %1, %2\n" \ 50 " bnez %1, 0b\n" \ 51 sc_append \ 52 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \ 53 : "rJ" (__newx), "rJ" (~__mask), "rJ" (__ptr32b) \ 54 : "memory"); \ 55 \ 56 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \ 57 } \ 58 }) 59 60 #define __arch_xchg(sfx, prepend, append, r, p, n) \ 61 ({ \ 62 __asm__ __volatile__ ( \ 63 prepend \ 64 " amoswap" sfx " %0, %2, %1\n" \ 65 append \ 66 : "=r" (r), "+A" (*(p)) \ 67 : "r" (n) \ 68 : "memory"); \ 69 }) 70 71 #define _arch_xchg(ptr, new, sc_sfx, swap_sfx, prepend, \ 72 sc_append, swap_append) \ 73 ({ \ 74 __typeof__(ptr) __ptr = (ptr); \ 75 __typeof__(*(__ptr)) __new = (new); \ 76 __typeof__(*(__ptr)) __ret; \ 77 \ 78 switch (sizeof(*__ptr)) { \ 79 case 1: \ 80 __arch_xchg_masked(sc_sfx, ".b" swap_sfx, \ 81 prepend, sc_append, swap_append, \ 82 __ret, __ptr, __new); \ 83 break; \ 84 case 2: \ 85 __arch_xchg_masked(sc_sfx, ".h" swap_sfx, \ 86 prepend, sc_append, swap_append, \ 87 __ret, __ptr, __new); \ 88 break; \ 89 case 4: \ 90 __arch_xchg(".w" swap_sfx, prepend, swap_append, \ 91 __ret, __ptr, __new); \ 92 break; \ 93 case 8: \ 94 __arch_xchg(".d" swap_sfx, prepend, swap_append, \ 95 __ret, __ptr, __new); \ 96 break; \ 97 default: \ 98 BUILD_BUG(); \ 99 } \ 100 (__typeof__(*(__ptr)))__ret; \ 101 }) 102 103 #define arch_xchg_relaxed(ptr, x) \ 104 _arch_xchg(ptr, x, "", "", "", "", "") 105 106 #define arch_xchg_acquire(ptr, x) \ 107 _arch_xchg(ptr, x, "", "", "", \ 108 RISCV_ACQUIRE_BARRIER, RISCV_ACQUIRE_BARRIER) 109 110 #define arch_xchg_release(ptr, x) \ 111 _arch_xchg(ptr, x, "", "", RISCV_RELEASE_BARRIER, "", "") 112 113 #define arch_xchg(ptr, x) \ 114 _arch_xchg(ptr, x, ".rl", ".aqrl", "", RISCV_FULL_BARRIER, "") 115 116 #define xchg32(ptr, x) \ 117 ({ \ 118 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 119 arch_xchg((ptr), (x)); \ 120 }) 121 122 #define xchg64(ptr, x) \ 123 ({ \ 124 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 125 arch_xchg((ptr), (x)); \ 126 }) 127 128 /* 129 * Atomic compare and exchange. Compare OLD with MEM, if identical, 130 * store NEW in MEM. Return the initial value in MEM. Success is 131 * indicated by comparing RETURN with OLD. 132 */ 133 #define __arch_cmpxchg_masked(sc_sfx, cas_sfx, \ 134 sc_prepend, sc_append, \ 135 cas_prepend, cas_append, \ 136 r, p, o, n) \ 137 ({ \ 138 if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \ 139 IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \ 140 IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \ 141 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA) && \ 142 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \ 143 r = o; \ 144 \ 145 __asm__ __volatile__ ( \ 146 cas_prepend \ 147 " .option push\n" \ 148 " .option arch, +zacas, +zabha\n" \ 149 " amocas" cas_sfx " %0, %z2, %1\n" \ 150 " .option pop\n" \ 151 cas_append \ 152 : "+&r" (r), "+A" (*(p)) \ 153 : "rJ" (n) \ 154 : "memory"); \ 155 } else { \ 156 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \ 157 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \ 158 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \ 159 << __s; \ 160 ulong __newx = (ulong)(n) << __s; \ 161 ulong __oldx = (ulong)(o) << __s; \ 162 ulong __retx; \ 163 ulong __rc; \ 164 \ 165 __asm__ __volatile__ ( \ 166 sc_prepend \ 167 "0: lr.w %0, %2\n" \ 168 " and %1, %0, %z5\n" \ 169 " bne %1, %z3, 1f\n" \ 170 " and %1, %0, %z6\n" \ 171 " or %1, %1, %z4\n" \ 172 " sc.w" sc_sfx " %1, %1, %2\n" \ 173 " bnez %1, 0b\n" \ 174 sc_append \ 175 "1:\n" \ 176 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \ 177 : "rJ" ((long)__oldx), "rJ" (__newx), \ 178 "rJ" (__mask), "rJ" (~__mask) \ 179 : "memory"); \ 180 \ 181 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \ 182 } \ 183 }) 184 185 #define __arch_cmpxchg(lr_sfx, sc_sfx, cas_sfx, \ 186 sc_prepend, sc_append, \ 187 cas_prepend, cas_append, \ 188 r, p, co, o, n) \ 189 ({ \ 190 if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \ 191 IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \ 192 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \ 193 r = o; \ 194 \ 195 __asm__ __volatile__ ( \ 196 cas_prepend \ 197 " .option push\n" \ 198 " .option arch, +zacas\n" \ 199 " amocas" cas_sfx " %0, %z2, %1\n" \ 200 " .option pop\n" \ 201 cas_append \ 202 : "+&r" (r), "+A" (*(p)) \ 203 : "rJ" (n) \ 204 : "memory"); \ 205 } else { \ 206 register unsigned int __rc; \ 207 \ 208 __asm__ __volatile__ ( \ 209 sc_prepend \ 210 "0: lr" lr_sfx " %0, %2\n" \ 211 " bne %0, %z3, 1f\n" \ 212 " sc" sc_sfx " %1, %z4, %2\n" \ 213 " bnez %1, 0b\n" \ 214 sc_append \ 215 "1:\n" \ 216 : "=&r" (r), "=&r" (__rc), "+A" (*(p)) \ 217 : "rJ" (co o), "rJ" (n) \ 218 : "memory"); \ 219 } \ 220 }) 221 222 #define _arch_cmpxchg(ptr, old, new, sc_sfx, cas_sfx, \ 223 sc_prepend, sc_append, \ 224 cas_prepend, cas_append) \ 225 ({ \ 226 __typeof__(ptr) __ptr = (ptr); \ 227 __typeof__(*(__ptr)) __old = (old); \ 228 __typeof__(*(__ptr)) __new = (new); \ 229 __typeof__(*(__ptr)) __ret; \ 230 \ 231 switch (sizeof(*__ptr)) { \ 232 case 1: \ 233 __arch_cmpxchg_masked(sc_sfx, ".b" cas_sfx, \ 234 sc_prepend, sc_append, \ 235 cas_prepend, cas_append, \ 236 __ret, __ptr, __old, __new); \ 237 break; \ 238 case 2: \ 239 __arch_cmpxchg_masked(sc_sfx, ".h" cas_sfx, \ 240 sc_prepend, sc_append, \ 241 cas_prepend, cas_append, \ 242 __ret, __ptr, __old, __new); \ 243 break; \ 244 case 4: \ 245 __arch_cmpxchg(".w", ".w" sc_sfx, ".w" cas_sfx, \ 246 sc_prepend, sc_append, \ 247 cas_prepend, cas_append, \ 248 __ret, __ptr, (long)(int)(long), __old, __new); \ 249 break; \ 250 case 8: \ 251 __arch_cmpxchg(".d", ".d" sc_sfx, ".d" cas_sfx, \ 252 sc_prepend, sc_append, \ 253 cas_prepend, cas_append, \ 254 __ret, __ptr, /**/, __old, __new); \ 255 break; \ 256 default: \ 257 BUILD_BUG(); \ 258 } \ 259 (__typeof__(*(__ptr)))__ret; \ 260 }) 261 262 /* 263 * These macros are here to improve the readability of the arch_cmpxchg_XXX() 264 * macros. 265 */ 266 #define SC_SFX(x) x 267 #define CAS_SFX(x) x 268 #define SC_PREPEND(x) x 269 #define SC_APPEND(x) x 270 #define CAS_PREPEND(x) x 271 #define CAS_APPEND(x) x 272 273 #define arch_cmpxchg_relaxed(ptr, o, n) \ 274 _arch_cmpxchg((ptr), (o), (n), \ 275 SC_SFX(""), CAS_SFX(""), \ 276 SC_PREPEND(""), SC_APPEND(""), \ 277 CAS_PREPEND(""), CAS_APPEND("")) 278 279 #define arch_cmpxchg_acquire(ptr, o, n) \ 280 _arch_cmpxchg((ptr), (o), (n), \ 281 SC_SFX(""), CAS_SFX(""), \ 282 SC_PREPEND(""), SC_APPEND(RISCV_ACQUIRE_BARRIER), \ 283 CAS_PREPEND(""), CAS_APPEND(RISCV_ACQUIRE_BARRIER)) 284 285 #define arch_cmpxchg_release(ptr, o, n) \ 286 _arch_cmpxchg((ptr), (o), (n), \ 287 SC_SFX(""), CAS_SFX(""), \ 288 SC_PREPEND(RISCV_RELEASE_BARRIER), SC_APPEND(""), \ 289 CAS_PREPEND(RISCV_RELEASE_BARRIER), CAS_APPEND("")) 290 291 #define arch_cmpxchg(ptr, o, n) \ 292 _arch_cmpxchg((ptr), (o), (n), \ 293 SC_SFX(".rl"), CAS_SFX(".aqrl"), \ 294 SC_PREPEND(""), SC_APPEND(RISCV_FULL_BARRIER), \ 295 CAS_PREPEND(""), CAS_APPEND("")) 296 297 #define arch_cmpxchg_local(ptr, o, n) \ 298 arch_cmpxchg_relaxed((ptr), (o), (n)) 299 300 #define arch_cmpxchg64(ptr, o, n) \ 301 ({ \ 302 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 303 arch_cmpxchg((ptr), (o), (n)); \ 304 }) 305 306 #define arch_cmpxchg64_local(ptr, o, n) \ 307 ({ \ 308 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 309 arch_cmpxchg_relaxed((ptr), (o), (n)); \ 310 }) 311 312 #define arch_cmpxchg64_relaxed(ptr, o, n) \ 313 ({ \ 314 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 315 arch_cmpxchg_relaxed((ptr), (o), (n)); \ 316 }) 317 318 #define arch_cmpxchg64_acquire(ptr, o, n) \ 319 ({ \ 320 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 321 arch_cmpxchg_acquire((ptr), (o), (n)); \ 322 }) 323 324 #define arch_cmpxchg64_release(ptr, o, n) \ 325 ({ \ 326 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 327 arch_cmpxchg_release((ptr), (o), (n)); \ 328 }) 329 330 #if defined(CONFIG_64BIT) && defined(CONFIG_RISCV_ISA_ZACAS) && defined(CONFIG_TOOLCHAIN_HAS_ZACAS) 331 332 #define system_has_cmpxchg128() riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS) 333 334 union __u128_halves { 335 u128 full; 336 struct { 337 u64 low, high; 338 }; 339 }; 340 341 #define __arch_cmpxchg128(p, o, n, cas_sfx) \ 342 ({ \ 343 __typeof__(*(p)) __o = (o); \ 344 union __u128_halves __hn = { .full = (n) }; \ 345 union __u128_halves __ho = { .full = (__o) }; \ 346 register unsigned long t1 asm ("t1") = __hn.low; \ 347 register unsigned long t2 asm ("t2") = __hn.high; \ 348 register unsigned long t3 asm ("t3") = __ho.low; \ 349 register unsigned long t4 asm ("t4") = __ho.high; \ 350 \ 351 __asm__ __volatile__ ( \ 352 " .option push\n" \ 353 " .option arch, +zacas\n" \ 354 " amocas.q" cas_sfx " %0, %z3, %2\n" \ 355 " .option pop\n" \ 356 : "+&r" (t3), "+&r" (t4), "+A" (*(p)) \ 357 : "rJ" (t1), "rJ" (t2) \ 358 : "memory"); \ 359 \ 360 ((u128)t4 << 64) | t3; \ 361 }) 362 363 #define arch_cmpxchg128(ptr, o, n) \ 364 __arch_cmpxchg128((ptr), (o), (n), ".aqrl") 365 366 #define arch_cmpxchg128_local(ptr, o, n) \ 367 __arch_cmpxchg128((ptr), (o), (n), "") 368 369 #endif /* CONFIG_64BIT && CONFIG_RISCV_ISA_ZACAS && CONFIG_TOOLCHAIN_HAS_ZACAS */ 370 371 #ifdef CONFIG_RISCV_ISA_ZAWRS 372 /* 373 * Despite wrs.nto being "WRS-with-no-timeout", in the absence of changes to 374 * @val we expect it to still terminate within a "reasonable" amount of time 375 * for an implementation-specific other reason, a pending, locally-enabled 376 * interrupt, or because it has been configured to raise an illegal 377 * instruction exception. 378 */ 379 static __always_inline void __cmpwait(volatile void *ptr, 380 unsigned long val, 381 int size) 382 { 383 unsigned long tmp; 384 385 u32 *__ptr32b; 386 ulong __s, __val, __mask; 387 388 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZAWRS)) { 389 ALT_RISCV_PAUSE(); 390 return; 391 } 392 393 switch (size) { 394 case 1: 395 __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); 396 __s = ((ulong)(ptr) & 0x3) * BITS_PER_BYTE; 397 __val = val << __s; 398 __mask = 0xff << __s; 399 400 asm volatile( 401 " lr.w %0, %1\n" 402 " and %0, %0, %3\n" 403 " xor %0, %0, %2\n" 404 " bnez %0, 1f\n" 405 ZAWRS_WRS_NTO "\n" 406 "1:" 407 : "=&r" (tmp), "+A" (*(__ptr32b)) 408 : "r" (__val), "r" (__mask) 409 : "memory"); 410 break; 411 case 2: 412 __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); 413 __s = ((ulong)(ptr) & 0x2) * BITS_PER_BYTE; 414 __val = val << __s; 415 __mask = 0xffff << __s; 416 417 asm volatile( 418 " lr.w %0, %1\n" 419 " and %0, %0, %3\n" 420 " xor %0, %0, %2\n" 421 " bnez %0, 1f\n" 422 ZAWRS_WRS_NTO "\n" 423 "1:" 424 : "=&r" (tmp), "+A" (*(__ptr32b)) 425 : "r" (__val), "r" (__mask) 426 : "memory"); 427 break; 428 case 4: 429 asm volatile( 430 " lr.w %0, %1\n" 431 " xor %0, %0, %2\n" 432 " bnez %0, 1f\n" 433 ZAWRS_WRS_NTO "\n" 434 "1:" 435 : "=&r" (tmp), "+A" (*(u32 *)ptr) 436 : "r" (val)); 437 break; 438 #if __riscv_xlen == 64 439 case 8: 440 asm volatile( 441 " lr.d %0, %1\n" 442 " xor %0, %0, %2\n" 443 " bnez %0, 1f\n" 444 ZAWRS_WRS_NTO "\n" 445 "1:" 446 : "=&r" (tmp), "+A" (*(u64 *)ptr) 447 : "r" (val)); 448 break; 449 #endif 450 default: 451 BUILD_BUG(); 452 } 453 } 454 455 #define __cmpwait_relaxed(ptr, val) \ 456 __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr))) 457 #endif 458 459 #endif /* _ASM_RISCV_CMPXCHG_H */ 460