1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Atomic operations. 4 * 5 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 6 */ 7 #ifndef _ASM_ATOMIC_H 8 #define _ASM_ATOMIC_H 9 10 #include <linux/types.h> 11 #include <asm/barrier.h> 12 #include <asm/cmpxchg.h> 13 14 #if __SIZEOF_LONG__ == 4 15 #define __LL "ll.w " 16 #define __SC "sc.w " 17 #define __AMADD "amadd.w " 18 #define __AMAND_DB "amand_db.w " 19 #define __AMOR_DB "amor_db.w " 20 #define __AMXOR_DB "amxor_db.w " 21 #elif __SIZEOF_LONG__ == 8 22 #define __LL "ll.d " 23 #define __SC "sc.d " 24 #define __AMADD "amadd.d " 25 #define __AMAND_DB "amand_db.d " 26 #define __AMOR_DB "amor_db.d " 27 #define __AMXOR_DB "amxor_db.d " 28 #endif 29 30 #define ATOMIC_INIT(i) { (i) } 31 32 /* 33 * arch_atomic_read - read atomic variable 34 * @v: pointer of type atomic_t 35 * 36 * Atomically reads the value of @v. 37 */ 38 #define arch_atomic_read(v) READ_ONCE((v)->counter) 39 40 /* 41 * arch_atomic_set - set atomic variable 42 * @v: pointer of type atomic_t 43 * @i: required value 44 * 45 * Atomically sets the value of @v to @i. 46 */ 47 #define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i)) 48 49 #define ATOMIC_OP(op, I, asm_op) \ 50 static inline void arch_atomic_##op(int i, atomic_t *v) \ 51 { \ 52 __asm__ __volatile__( \ 53 "am"#asm_op"_db.w" " $zero, %1, %0 \n" \ 54 : "+ZB" (v->counter) \ 55 : "r" (I) \ 56 : "memory"); \ 57 } 58 59 #define ATOMIC_OP_RETURN(op, I, asm_op, c_op) \ 60 static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ 61 { \ 62 int result; \ 63 \ 64 __asm__ __volatile__( \ 65 "am"#asm_op"_db.w" " %1, %2, %0 \n" \ 66 : "+ZB" (v->counter), "=&r" (result) \ 67 : "r" (I) \ 68 : "memory"); \ 69 \ 70 return result c_op I; \ 71 } 72 73 #define ATOMIC_FETCH_OP(op, I, asm_op) \ 74 static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ 75 { \ 76 int result; \ 77 \ 78 __asm__ __volatile__( \ 79 "am"#asm_op"_db.w" " %1, %2, %0 \n" \ 80 : "+ZB" (v->counter), "=&r" (result) \ 81 : "r" (I) \ 82 : "memory"); \ 83 \ 84 return result; \ 85 } 86 87 #define ATOMIC_OPS(op, I, asm_op, c_op) \ 88 ATOMIC_OP(op, I, asm_op) \ 89 ATOMIC_OP_RETURN(op, I, asm_op, c_op) \ 90 ATOMIC_FETCH_OP(op, I, asm_op) 91 92 ATOMIC_OPS(add, i, add, +) 93 ATOMIC_OPS(sub, -i, add, +) 94 95 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 96 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 97 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 98 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 99 100 #undef ATOMIC_OPS 101 102 #define ATOMIC_OPS(op, I, asm_op) \ 103 ATOMIC_OP(op, I, asm_op) \ 104 ATOMIC_FETCH_OP(op, I, asm_op) 105 106 ATOMIC_OPS(and, i, and) 107 ATOMIC_OPS(or, i, or) 108 ATOMIC_OPS(xor, i, xor) 109 110 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 111 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 112 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 113 114 #undef ATOMIC_OPS 115 #undef ATOMIC_FETCH_OP 116 #undef ATOMIC_OP_RETURN 117 #undef ATOMIC_OP 118 119 static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 120 { 121 int prev, rc; 122 123 __asm__ __volatile__ ( 124 "0: ll.w %[p], %[c]\n" 125 " beq %[p], %[u], 1f\n" 126 " add.w %[rc], %[p], %[a]\n" 127 " sc.w %[rc], %[c]\n" 128 " beqz %[rc], 0b\n" 129 " b 2f\n" 130 "1:\n" 131 __WEAK_LLSC_MB 132 "2:\n" 133 : [p]"=&r" (prev), [rc]"=&r" (rc), 134 [c]"=ZB" (v->counter) 135 : [a]"r" (a), [u]"r" (u) 136 : "memory"); 137 138 return prev; 139 } 140 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 141 142 /* 143 * arch_atomic_sub_if_positive - conditionally subtract integer from atomic variable 144 * @i: integer value to subtract 145 * @v: pointer of type atomic_t 146 * 147 * Atomically test @v and subtract @i if @v is greater or equal than @i. 148 * The function returns the old value of @v minus @i. 149 */ 150 static inline int arch_atomic_sub_if_positive(int i, atomic_t *v) 151 { 152 int result; 153 int temp; 154 155 if (__builtin_constant_p(i)) { 156 __asm__ __volatile__( 157 "1: ll.w %1, %2 # atomic_sub_if_positive\n" 158 " addi.w %0, %1, %3 \n" 159 " move %1, %0 \n" 160 " bltz %0, 2f \n" 161 " sc.w %1, %2 \n" 162 " beqz %1, 1b \n" 163 "2: \n" 164 __WEAK_LLSC_MB 165 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) 166 : "I" (-i)); 167 } else { 168 __asm__ __volatile__( 169 "1: ll.w %1, %2 # atomic_sub_if_positive\n" 170 " sub.w %0, %1, %3 \n" 171 " move %1, %0 \n" 172 " bltz %0, 2f \n" 173 " sc.w %1, %2 \n" 174 " beqz %1, 1b \n" 175 "2: \n" 176 __WEAK_LLSC_MB 177 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) 178 : "r" (i)); 179 } 180 181 return result; 182 } 183 184 #define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n))) 185 #define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new))) 186 187 /* 188 * arch_atomic_dec_if_positive - decrement by 1 if old value positive 189 * @v: pointer of type atomic_t 190 */ 191 #define arch_atomic_dec_if_positive(v) arch_atomic_sub_if_positive(1, v) 192 193 #ifdef CONFIG_64BIT 194 195 #define ATOMIC64_INIT(i) { (i) } 196 197 /* 198 * arch_atomic64_read - read atomic variable 199 * @v: pointer of type atomic64_t 200 * 201 */ 202 #define arch_atomic64_read(v) READ_ONCE((v)->counter) 203 204 /* 205 * arch_atomic64_set - set atomic variable 206 * @v: pointer of type atomic64_t 207 * @i: required value 208 */ 209 #define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i)) 210 211 #define ATOMIC64_OP(op, I, asm_op) \ 212 static inline void arch_atomic64_##op(long i, atomic64_t *v) \ 213 { \ 214 __asm__ __volatile__( \ 215 "am"#asm_op"_db.d " " $zero, %1, %0 \n" \ 216 : "+ZB" (v->counter) \ 217 : "r" (I) \ 218 : "memory"); \ 219 } 220 221 #define ATOMIC64_OP_RETURN(op, I, asm_op, c_op) \ 222 static inline long arch_atomic64_##op##_return_relaxed(long i, atomic64_t *v) \ 223 { \ 224 long result; \ 225 __asm__ __volatile__( \ 226 "am"#asm_op"_db.d " " %1, %2, %0 \n" \ 227 : "+ZB" (v->counter), "=&r" (result) \ 228 : "r" (I) \ 229 : "memory"); \ 230 \ 231 return result c_op I; \ 232 } 233 234 #define ATOMIC64_FETCH_OP(op, I, asm_op) \ 235 static inline long arch_atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ 236 { \ 237 long result; \ 238 \ 239 __asm__ __volatile__( \ 240 "am"#asm_op"_db.d " " %1, %2, %0 \n" \ 241 : "+ZB" (v->counter), "=&r" (result) \ 242 : "r" (I) \ 243 : "memory"); \ 244 \ 245 return result; \ 246 } 247 248 #define ATOMIC64_OPS(op, I, asm_op, c_op) \ 249 ATOMIC64_OP(op, I, asm_op) \ 250 ATOMIC64_OP_RETURN(op, I, asm_op, c_op) \ 251 ATOMIC64_FETCH_OP(op, I, asm_op) 252 253 ATOMIC64_OPS(add, i, add, +) 254 ATOMIC64_OPS(sub, -i, add, +) 255 256 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 257 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 258 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 259 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 260 261 #undef ATOMIC64_OPS 262 263 #define ATOMIC64_OPS(op, I, asm_op) \ 264 ATOMIC64_OP(op, I, asm_op) \ 265 ATOMIC64_FETCH_OP(op, I, asm_op) 266 267 ATOMIC64_OPS(and, i, and) 268 ATOMIC64_OPS(or, i, or) 269 ATOMIC64_OPS(xor, i, xor) 270 271 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 272 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 273 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 274 275 #undef ATOMIC64_OPS 276 #undef ATOMIC64_FETCH_OP 277 #undef ATOMIC64_OP_RETURN 278 #undef ATOMIC64_OP 279 280 static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u) 281 { 282 long prev, rc; 283 284 __asm__ __volatile__ ( 285 "0: ll.d %[p], %[c]\n" 286 " beq %[p], %[u], 1f\n" 287 " add.d %[rc], %[p], %[a]\n" 288 " sc.d %[rc], %[c]\n" 289 " beqz %[rc], 0b\n" 290 " b 2f\n" 291 "1:\n" 292 __WEAK_LLSC_MB 293 "2:\n" 294 : [p]"=&r" (prev), [rc]"=&r" (rc), 295 [c] "=ZB" (v->counter) 296 : [a]"r" (a), [u]"r" (u) 297 : "memory"); 298 299 return prev; 300 } 301 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 302 303 /* 304 * arch_atomic64_sub_if_positive - conditionally subtract integer from atomic variable 305 * @i: integer value to subtract 306 * @v: pointer of type atomic64_t 307 * 308 * Atomically test @v and subtract @i if @v is greater or equal than @i. 309 * The function returns the old value of @v minus @i. 310 */ 311 static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v) 312 { 313 long result; 314 long temp; 315 316 if (__builtin_constant_p(i)) { 317 __asm__ __volatile__( 318 "1: ll.d %1, %2 # atomic64_sub_if_positive \n" 319 " addi.d %0, %1, %3 \n" 320 " move %1, %0 \n" 321 " bltz %0, 2f \n" 322 " sc.d %1, %2 \n" 323 " beqz %1, 1b \n" 324 "2: \n" 325 __WEAK_LLSC_MB 326 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) 327 : "I" (-i)); 328 } else { 329 __asm__ __volatile__( 330 "1: ll.d %1, %2 # atomic64_sub_if_positive \n" 331 " sub.d %0, %1, %3 \n" 332 " move %1, %0 \n" 333 " bltz %0, 2f \n" 334 " sc.d %1, %2 \n" 335 " beqz %1, 1b \n" 336 "2: \n" 337 __WEAK_LLSC_MB 338 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) 339 : "r" (i)); 340 } 341 342 return result; 343 } 344 345 #define arch_atomic64_cmpxchg(v, o, n) \ 346 ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n))) 347 #define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), (new))) 348 349 /* 350 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive 351 * @v: pointer of type atomic64_t 352 */ 353 #define arch_atomic64_dec_if_positive(v) arch_atomic64_sub_if_positive(1, v) 354 355 #endif /* CONFIG_64BIT */ 356 357 #endif /* _ASM_ATOMIC_H */ 358