1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. 4 * Copyright (C) 2012 Regents of the University of California 5 * Copyright (C) 2017 SiFive 6 */ 7 8 #ifndef _ASM_RISCV_ATOMIC_H 9 #define _ASM_RISCV_ATOMIC_H 10 11 #ifdef CONFIG_GENERIC_ATOMIC64 12 # include <asm-generic/atomic64.h> 13 #else 14 # if (__riscv_xlen < 64) 15 # error "64-bit atomics require XLEN to be at least 64" 16 # endif 17 #endif 18 19 #include <asm/cmpxchg.h> 20 21 #define __atomic_acquire_fence() \ 22 __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory") 23 24 #define __atomic_release_fence() \ 25 __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); 26 27 static __always_inline int arch_atomic_read(const atomic_t *v) 28 { 29 return READ_ONCE(v->counter); 30 } 31 static __always_inline void arch_atomic_set(atomic_t *v, int i) 32 { 33 WRITE_ONCE(v->counter, i); 34 } 35 36 #ifndef CONFIG_GENERIC_ATOMIC64 37 #define ATOMIC64_INIT(i) { (i) } 38 static __always_inline s64 arch_atomic64_read(const atomic64_t *v) 39 { 40 return READ_ONCE(v->counter); 41 } 42 static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) 43 { 44 WRITE_ONCE(v->counter, i); 45 } 46 #endif 47 48 /* 49 * First, the atomic ops that have no ordering constraints and therefor don't 50 * have the AQ or RL bits set. These don't return anything, so there's only 51 * one version to worry about. 52 */ 53 #define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \ 54 static __always_inline \ 55 void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ 56 { \ 57 __asm__ __volatile__ ( \ 58 " amo" #asm_op "." #asm_type " zero, %1, %0" \ 59 : "+A" (v->counter) \ 60 : "r" (I) \ 61 : "memory"); \ 62 } \ 63 64 #ifdef CONFIG_GENERIC_ATOMIC64 65 #define ATOMIC_OPS(op, asm_op, I) \ 66 ATOMIC_OP (op, asm_op, I, w, int, ) 67 #else 68 #define ATOMIC_OPS(op, asm_op, I) \ 69 ATOMIC_OP (op, asm_op, I, w, int, ) \ 70 ATOMIC_OP (op, asm_op, I, d, s64, 64) 71 #endif 72 73 ATOMIC_OPS(add, add, i) 74 ATOMIC_OPS(sub, add, -i) 75 ATOMIC_OPS(and, and, i) 76 ATOMIC_OPS( or, or, i) 77 ATOMIC_OPS(xor, xor, i) 78 79 #undef ATOMIC_OP 80 #undef ATOMIC_OPS 81 82 /* 83 * Atomic ops that have ordered, relaxed, acquire, and release variants. 84 * There's two flavors of these: the arithmatic ops have both fetch and return 85 * versions, while the logical ops only have fetch versions. 86 */ 87 #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \ 88 static __always_inline \ 89 c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \ 90 atomic##prefix##_t *v) \ 91 { \ 92 register c_type ret; \ 93 __asm__ __volatile__ ( \ 94 " amo" #asm_op "." #asm_type " %1, %2, %0" \ 95 : "+A" (v->counter), "=r" (ret) \ 96 : "r" (I) \ 97 : "memory"); \ 98 return ret; \ 99 } \ 100 static __always_inline \ 101 c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ 102 { \ 103 register c_type ret; \ 104 __asm__ __volatile__ ( \ 105 " amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \ 106 : "+A" (v->counter), "=r" (ret) \ 107 : "r" (I) \ 108 : "memory"); \ 109 return ret; \ 110 } 111 112 #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \ 113 static __always_inline \ 114 c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \ 115 atomic##prefix##_t *v) \ 116 { \ 117 return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ 118 } \ 119 static __always_inline \ 120 c_type arch_atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ 121 { \ 122 return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \ 123 } 124 125 #ifdef CONFIG_GENERIC_ATOMIC64 126 #define ATOMIC_OPS(op, asm_op, c_op, I) \ 127 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \ 128 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) 129 #else 130 #define ATOMIC_OPS(op, asm_op, c_op, I) \ 131 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \ 132 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \ 133 ATOMIC_FETCH_OP( op, asm_op, I, d, s64, 64) \ 134 ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, s64, 64) 135 #endif 136 137 ATOMIC_OPS(add, add, +, i) 138 ATOMIC_OPS(sub, add, +, -i) 139 140 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 141 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 142 #define arch_atomic_add_return arch_atomic_add_return 143 #define arch_atomic_sub_return arch_atomic_sub_return 144 145 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 146 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 147 #define arch_atomic_fetch_add arch_atomic_fetch_add 148 #define arch_atomic_fetch_sub arch_atomic_fetch_sub 149 150 #ifndef CONFIG_GENERIC_ATOMIC64 151 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 152 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 153 #define arch_atomic64_add_return arch_atomic64_add_return 154 #define arch_atomic64_sub_return arch_atomic64_sub_return 155 156 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 157 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 158 #define arch_atomic64_fetch_add arch_atomic64_fetch_add 159 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub 160 #endif 161 162 #undef ATOMIC_OPS 163 164 #ifdef CONFIG_GENERIC_ATOMIC64 165 #define ATOMIC_OPS(op, asm_op, I) \ 166 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) 167 #else 168 #define ATOMIC_OPS(op, asm_op, I) \ 169 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \ 170 ATOMIC_FETCH_OP(op, asm_op, I, d, s64, 64) 171 #endif 172 173 ATOMIC_OPS(and, and, i) 174 ATOMIC_OPS( or, or, i) 175 ATOMIC_OPS(xor, xor, i) 176 177 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 178 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 179 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 180 #define arch_atomic_fetch_and arch_atomic_fetch_and 181 #define arch_atomic_fetch_or arch_atomic_fetch_or 182 #define arch_atomic_fetch_xor arch_atomic_fetch_xor 183 184 #ifndef CONFIG_GENERIC_ATOMIC64 185 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 186 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 187 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 188 #define arch_atomic64_fetch_and arch_atomic64_fetch_and 189 #define arch_atomic64_fetch_or arch_atomic64_fetch_or 190 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor 191 #endif 192 193 #undef ATOMIC_OPS 194 195 #undef ATOMIC_FETCH_OP 196 #undef ATOMIC_OP_RETURN 197 198 #define _arch_atomic_fetch_add_unless(_prev, _rc, counter, _a, _u, sfx) \ 199 ({ \ 200 __asm__ __volatile__ ( \ 201 "0: lr." sfx " %[p], %[c]\n" \ 202 " beq %[p], %[u], 1f\n" \ 203 " add %[rc], %[p], %[a]\n" \ 204 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \ 205 " bnez %[rc], 0b\n" \ 206 " fence rw, rw\n" \ 207 "1:\n" \ 208 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \ 209 : [a]"r" (_a), [u]"r" (_u) \ 210 : "memory"); \ 211 }) 212 213 /* This is required to provide a full barrier on success. */ 214 static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 215 { 216 int prev, rc; 217 218 _arch_atomic_fetch_add_unless(prev, rc, v->counter, a, u, "w"); 219 220 return prev; 221 } 222 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 223 224 #ifndef CONFIG_GENERIC_ATOMIC64 225 static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 226 { 227 s64 prev; 228 long rc; 229 230 _arch_atomic_fetch_add_unless(prev, rc, v->counter, a, u, "d"); 231 232 return prev; 233 } 234 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 235 #endif 236 237 #define _arch_atomic_inc_unless_negative(_prev, _rc, counter, sfx) \ 238 ({ \ 239 __asm__ __volatile__ ( \ 240 "0: lr." sfx " %[p], %[c]\n" \ 241 " bltz %[p], 1f\n" \ 242 " addi %[rc], %[p], 1\n" \ 243 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \ 244 " bnez %[rc], 0b\n" \ 245 " fence rw, rw\n" \ 246 "1:\n" \ 247 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \ 248 : \ 249 : "memory"); \ 250 }) 251 252 static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v) 253 { 254 int prev, rc; 255 256 _arch_atomic_inc_unless_negative(prev, rc, v->counter, "w"); 257 258 return !(prev < 0); 259 } 260 261 #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative 262 263 #define _arch_atomic_dec_unless_positive(_prev, _rc, counter, sfx) \ 264 ({ \ 265 __asm__ __volatile__ ( \ 266 "0: lr." sfx " %[p], %[c]\n" \ 267 " bgtz %[p], 1f\n" \ 268 " addi %[rc], %[p], -1\n" \ 269 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \ 270 " bnez %[rc], 0b\n" \ 271 " fence rw, rw\n" \ 272 "1:\n" \ 273 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \ 274 : \ 275 : "memory"); \ 276 }) 277 278 static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v) 279 { 280 int prev, rc; 281 282 _arch_atomic_dec_unless_positive(prev, rc, v->counter, "w"); 283 284 return !(prev > 0); 285 } 286 287 #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive 288 289 #define _arch_atomic_dec_if_positive(_prev, _rc, counter, sfx) \ 290 ({ \ 291 __asm__ __volatile__ ( \ 292 "0: lr." sfx " %[p], %[c]\n" \ 293 " addi %[rc], %[p], -1\n" \ 294 " bltz %[rc], 1f\n" \ 295 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \ 296 " bnez %[rc], 0b\n" \ 297 " fence rw, rw\n" \ 298 "1:\n" \ 299 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \ 300 : \ 301 : "memory"); \ 302 }) 303 304 static __always_inline int arch_atomic_dec_if_positive(atomic_t *v) 305 { 306 int prev, rc; 307 308 _arch_atomic_dec_if_positive(prev, rc, v->counter, "w"); 309 310 return prev - 1; 311 } 312 313 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive 314 315 #ifndef CONFIG_GENERIC_ATOMIC64 316 static __always_inline bool arch_atomic64_inc_unless_negative(atomic64_t *v) 317 { 318 s64 prev; 319 long rc; 320 321 _arch_atomic_inc_unless_negative(prev, rc, v->counter, "d"); 322 323 return !(prev < 0); 324 } 325 326 #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative 327 328 static __always_inline bool arch_atomic64_dec_unless_positive(atomic64_t *v) 329 { 330 s64 prev; 331 long rc; 332 333 _arch_atomic_dec_unless_positive(prev, rc, v->counter, "d"); 334 335 return !(prev > 0); 336 } 337 338 #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive 339 340 static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 341 { 342 s64 prev; 343 long rc; 344 345 _arch_atomic_dec_if_positive(prev, rc, v->counter, "d"); 346 347 return prev - 1; 348 } 349 350 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 351 #endif 352 353 #endif /* _ASM_RISCV_ATOMIC_H */ 354