1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. 4 * Copyright (C) 2012 Regents of the University of California 5 * Copyright (C) 2017 SiFive 6 */ 7 8 #ifndef _ASM_RISCV_ATOMIC_H 9 #define _ASM_RISCV_ATOMIC_H 10 11 #ifdef CONFIG_GENERIC_ATOMIC64 12 # include <asm-generic/atomic64.h> 13 #else 14 # if (__riscv_xlen < 64) 15 # error "64-bit atomics require XLEN to be at least 64" 16 # endif 17 #endif 18 19 #include <asm/cmpxchg.h> 20 21 #define __atomic_acquire_fence() \ 22 __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory") 23 24 #define __atomic_release_fence() \ 25 __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); 26 27 static __always_inline int arch_atomic_read(const atomic_t *v) 28 { 29 return READ_ONCE(v->counter); 30 } 31 static __always_inline void arch_atomic_set(atomic_t *v, int i) 32 { 33 WRITE_ONCE(v->counter, i); 34 } 35 36 #ifndef CONFIG_GENERIC_ATOMIC64 37 #define ATOMIC64_INIT(i) { (i) } 38 static __always_inline s64 arch_atomic64_read(const atomic64_t *v) 39 { 40 return READ_ONCE(v->counter); 41 } 42 static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) 43 { 44 WRITE_ONCE(v->counter, i); 45 } 46 #endif 47 48 /* 49 * First, the atomic ops that have no ordering constraints and therefor don't 50 * have the AQ or RL bits set. These don't return anything, so there's only 51 * one version to worry about. 52 */ 53 #define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \ 54 static __always_inline \ 55 void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ 56 { \ 57 __asm__ __volatile__ ( \ 58 " amo" #asm_op "." #asm_type " zero, %1, %0" \ 59 : "+A" (v->counter) \ 60 : "r" (I) \ 61 : "memory"); \ 62 } \ 63 64 #ifdef CONFIG_GENERIC_ATOMIC64 65 #define ATOMIC_OPS(op, asm_op, I) \ 66 ATOMIC_OP (op, asm_op, I, w, int, ) 67 #else 68 #define ATOMIC_OPS(op, asm_op, I) \ 69 ATOMIC_OP (op, asm_op, I, w, int, ) \ 70 ATOMIC_OP (op, asm_op, I, d, s64, 64) 71 #endif 72 73 ATOMIC_OPS(add, add, i) 74 ATOMIC_OPS(sub, add, -i) 75 ATOMIC_OPS(and, and, i) 76 ATOMIC_OPS( or, or, i) 77 ATOMIC_OPS(xor, xor, i) 78 79 #undef ATOMIC_OP 80 #undef ATOMIC_OPS 81 82 /* 83 * Atomic ops that have ordered, relaxed, acquire, and release variants. 84 * There's two flavors of these: the arithmatic ops have both fetch and return 85 * versions, while the logical ops only have fetch versions. 86 */ 87 #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \ 88 static __always_inline \ 89 c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \ 90 atomic##prefix##_t *v) \ 91 { \ 92 register c_type ret; \ 93 __asm__ __volatile__ ( \ 94 " amo" #asm_op "." #asm_type " %1, %2, %0" \ 95 : "+A" (v->counter), "=r" (ret) \ 96 : "r" (I) \ 97 : "memory"); \ 98 return ret; \ 99 } \ 100 static __always_inline \ 101 c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ 102 { \ 103 register c_type ret; \ 104 __asm__ __volatile__ ( \ 105 " amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \ 106 : "+A" (v->counter), "=r" (ret) \ 107 : "r" (I) \ 108 : "memory"); \ 109 return ret; \ 110 } 111 112 #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \ 113 static __always_inline \ 114 c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \ 115 atomic##prefix##_t *v) \ 116 { \ 117 return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ 118 } \ 119 static __always_inline \ 120 c_type arch_atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ 121 { \ 122 return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \ 123 } 124 125 #ifdef CONFIG_GENERIC_ATOMIC64 126 #define ATOMIC_OPS(op, asm_op, c_op, I) \ 127 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \ 128 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) 129 #else 130 #define ATOMIC_OPS(op, asm_op, c_op, I) \ 131 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \ 132 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \ 133 ATOMIC_FETCH_OP( op, asm_op, I, d, s64, 64) \ 134 ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, s64, 64) 135 #endif 136 137 ATOMIC_OPS(add, add, +, i) 138 ATOMIC_OPS(sub, add, +, -i) 139 140 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 141 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 142 #define arch_atomic_add_return arch_atomic_add_return 143 #define arch_atomic_sub_return arch_atomic_sub_return 144 145 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 146 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 147 #define arch_atomic_fetch_add arch_atomic_fetch_add 148 #define arch_atomic_fetch_sub arch_atomic_fetch_sub 149 150 #ifndef CONFIG_GENERIC_ATOMIC64 151 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 152 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 153 #define arch_atomic64_add_return arch_atomic64_add_return 154 #define arch_atomic64_sub_return arch_atomic64_sub_return 155 156 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 157 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 158 #define arch_atomic64_fetch_add arch_atomic64_fetch_add 159 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub 160 #endif 161 162 #undef ATOMIC_OPS 163 164 #ifdef CONFIG_GENERIC_ATOMIC64 165 #define ATOMIC_OPS(op, asm_op, I) \ 166 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) 167 #else 168 #define ATOMIC_OPS(op, asm_op, I) \ 169 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \ 170 ATOMIC_FETCH_OP(op, asm_op, I, d, s64, 64) 171 #endif 172 173 ATOMIC_OPS(and, and, i) 174 ATOMIC_OPS( or, or, i) 175 ATOMIC_OPS(xor, xor, i) 176 177 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 178 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 179 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 180 #define arch_atomic_fetch_and arch_atomic_fetch_and 181 #define arch_atomic_fetch_or arch_atomic_fetch_or 182 #define arch_atomic_fetch_xor arch_atomic_fetch_xor 183 184 #ifndef CONFIG_GENERIC_ATOMIC64 185 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 186 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 187 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 188 #define arch_atomic64_fetch_and arch_atomic64_fetch_and 189 #define arch_atomic64_fetch_or arch_atomic64_fetch_or 190 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor 191 #endif 192 193 #undef ATOMIC_OPS 194 195 #undef ATOMIC_FETCH_OP 196 #undef ATOMIC_OP_RETURN 197 198 /* This is required to provide a full barrier on success. */ 199 static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 200 { 201 int prev, rc; 202 203 __asm__ __volatile__ ( 204 "0: lr.w %[p], %[c]\n" 205 " beq %[p], %[u], 1f\n" 206 " add %[rc], %[p], %[a]\n" 207 " sc.w.rl %[rc], %[rc], %[c]\n" 208 " bnez %[rc], 0b\n" 209 RISCV_FULL_BARRIER 210 "1:\n" 211 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 212 : [a]"r" (a), [u]"r" (u) 213 : "memory"); 214 return prev; 215 } 216 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 217 218 #ifndef CONFIG_GENERIC_ATOMIC64 219 static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 220 { 221 s64 prev; 222 long rc; 223 224 __asm__ __volatile__ ( 225 "0: lr.d %[p], %[c]\n" 226 " beq %[p], %[u], 1f\n" 227 " add %[rc], %[p], %[a]\n" 228 " sc.d.rl %[rc], %[rc], %[c]\n" 229 " bnez %[rc], 0b\n" 230 RISCV_FULL_BARRIER 231 "1:\n" 232 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 233 : [a]"r" (a), [u]"r" (u) 234 : "memory"); 235 return prev; 236 } 237 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 238 #endif 239 240 static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v) 241 { 242 int prev, rc; 243 244 __asm__ __volatile__ ( 245 "0: lr.w %[p], %[c]\n" 246 " bltz %[p], 1f\n" 247 " addi %[rc], %[p], 1\n" 248 " sc.w.rl %[rc], %[rc], %[c]\n" 249 " bnez %[rc], 0b\n" 250 RISCV_FULL_BARRIER 251 "1:\n" 252 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 253 : 254 : "memory"); 255 return !(prev < 0); 256 } 257 258 #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative 259 260 static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v) 261 { 262 int prev, rc; 263 264 __asm__ __volatile__ ( 265 "0: lr.w %[p], %[c]\n" 266 " bgtz %[p], 1f\n" 267 " addi %[rc], %[p], -1\n" 268 " sc.w.rl %[rc], %[rc], %[c]\n" 269 " bnez %[rc], 0b\n" 270 RISCV_FULL_BARRIER 271 "1:\n" 272 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 273 : 274 : "memory"); 275 return !(prev > 0); 276 } 277 278 #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive 279 280 static __always_inline int arch_atomic_dec_if_positive(atomic_t *v) 281 { 282 int prev, rc; 283 284 __asm__ __volatile__ ( 285 "0: lr.w %[p], %[c]\n" 286 " addi %[rc], %[p], -1\n" 287 " bltz %[rc], 1f\n" 288 " sc.w.rl %[rc], %[rc], %[c]\n" 289 " bnez %[rc], 0b\n" 290 RISCV_FULL_BARRIER 291 "1:\n" 292 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 293 : 294 : "memory"); 295 return prev - 1; 296 } 297 298 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive 299 300 #ifndef CONFIG_GENERIC_ATOMIC64 301 static __always_inline bool arch_atomic64_inc_unless_negative(atomic64_t *v) 302 { 303 s64 prev; 304 long rc; 305 306 __asm__ __volatile__ ( 307 "0: lr.d %[p], %[c]\n" 308 " bltz %[p], 1f\n" 309 " addi %[rc], %[p], 1\n" 310 " sc.d.rl %[rc], %[rc], %[c]\n" 311 " bnez %[rc], 0b\n" 312 RISCV_FULL_BARRIER 313 "1:\n" 314 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 315 : 316 : "memory"); 317 return !(prev < 0); 318 } 319 320 #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative 321 322 static __always_inline bool arch_atomic64_dec_unless_positive(atomic64_t *v) 323 { 324 s64 prev; 325 long rc; 326 327 __asm__ __volatile__ ( 328 "0: lr.d %[p], %[c]\n" 329 " bgtz %[p], 1f\n" 330 " addi %[rc], %[p], -1\n" 331 " sc.d.rl %[rc], %[rc], %[c]\n" 332 " bnez %[rc], 0b\n" 333 RISCV_FULL_BARRIER 334 "1:\n" 335 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 336 : 337 : "memory"); 338 return !(prev > 0); 339 } 340 341 #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive 342 343 static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 344 { 345 s64 prev; 346 long rc; 347 348 __asm__ __volatile__ ( 349 "0: lr.d %[p], %[c]\n" 350 " addi %[rc], %[p], -1\n" 351 " bltz %[rc], 1f\n" 352 " sc.d.rl %[rc], %[rc], %[c]\n" 353 " bnez %[rc], 0b\n" 354 RISCV_FULL_BARRIER 355 "1:\n" 356 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 357 : 358 : "memory"); 359 return prev - 1; 360 } 361 362 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 363 #endif 364 365 #endif /* _ASM_RISCV_ATOMIC_H */ 366