1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_POWERPC_ATOMIC_H_ 3 #define _ASM_POWERPC_ATOMIC_H_ 4 5 /* 6 * PowerPC atomic operations 7 */ 8 9 #ifdef __KERNEL__ 10 #include <linux/types.h> 11 #include <asm/cmpxchg.h> 12 #include <asm/barrier.h> 13 #include <asm/asm-const.h> 14 #include <asm/asm-compat.h> 15 16 /* 17 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with 18 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier 19 * on the platform without lwsync. 20 */ 21 #define __atomic_acquire_fence() \ 22 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") 23 24 #define __atomic_release_fence() \ 25 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") 26 27 static __inline__ int arch_atomic_read(const atomic_t *v) 28 { 29 int t; 30 31 /* -mprefixed can generate offsets beyond range, fall back hack */ 32 if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED)) 33 __asm__ __volatile__("lwz %0,0(%1)" : "=r"(t) : "b"(&v->counter)); 34 else 35 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m<>"(v->counter)); 36 37 return t; 38 } 39 40 static __inline__ void arch_atomic_set(atomic_t *v, int i) 41 { 42 /* -mprefixed can generate offsets beyond range, fall back hack */ 43 if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED)) 44 __asm__ __volatile__("stw %1,0(%2)" : "=m"(v->counter) : "r"(i), "b"(&v->counter)); 45 else 46 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m<>"(v->counter) : "r"(i)); 47 } 48 49 #define ATOMIC_OP(op, asm_op, suffix, sign, ...) \ 50 static __inline__ void arch_atomic_##op(int a, atomic_t *v) \ 51 { \ 52 int t; \ 53 \ 54 __asm__ __volatile__( \ 55 "1: lwarx %0,0,%3 # atomic_" #op "\n" \ 56 #asm_op "%I2" suffix " %0,%0,%2\n" \ 57 " stwcx. %0,0,%3 \n" \ 58 " bne- 1b\n" \ 59 : "=&r" (t), "+m" (v->counter) \ 60 : "r"#sign (a), "r" (&v->counter) \ 61 : "cc", ##__VA_ARGS__); \ 62 } \ 63 64 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ...) \ 65 static inline int arch_atomic_##op##_return_relaxed(int a, atomic_t *v) \ 66 { \ 67 int t; \ 68 \ 69 __asm__ __volatile__( \ 70 "1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ 71 #asm_op "%I2" suffix " %0,%0,%2\n" \ 72 " stwcx. %0,0,%3\n" \ 73 " bne- 1b\n" \ 74 : "=&r" (t), "+m" (v->counter) \ 75 : "r"#sign (a), "r" (&v->counter) \ 76 : "cc", ##__VA_ARGS__); \ 77 \ 78 return t; \ 79 } 80 81 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ...) \ 82 static inline int arch_atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ 83 { \ 84 int res, t; \ 85 \ 86 __asm__ __volatile__( \ 87 "1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ 88 #asm_op "%I3" suffix " %1,%0,%3\n" \ 89 " stwcx. %1,0,%4\n" \ 90 " bne- 1b\n" \ 91 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 92 : "r"#sign (a), "r" (&v->counter) \ 93 : "cc", ##__VA_ARGS__); \ 94 \ 95 return res; \ 96 } 97 98 #define ATOMIC_OPS(op, asm_op, suffix, sign, ...) \ 99 ATOMIC_OP(op, asm_op, suffix, sign, ##__VA_ARGS__) \ 100 ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__)\ 101 ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__) 102 103 ATOMIC_OPS(add, add, "c", I, "xer") 104 ATOMIC_OPS(sub, sub, "c", I, "xer") 105 106 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 107 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 108 109 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 110 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 111 112 #undef ATOMIC_OPS 113 #define ATOMIC_OPS(op, asm_op, suffix, sign) \ 114 ATOMIC_OP(op, asm_op, suffix, sign) \ 115 ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign) 116 117 ATOMIC_OPS(and, and, ".", K) 118 ATOMIC_OPS(or, or, "", K) 119 ATOMIC_OPS(xor, xor, "", K) 120 121 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 122 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 123 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 124 125 #undef ATOMIC_OPS 126 #undef ATOMIC_FETCH_OP_RELAXED 127 #undef ATOMIC_OP_RETURN_RELAXED 128 #undef ATOMIC_OP 129 130 /** 131 * atomic_fetch_add_unless - add unless the number is a given value 132 * @v: pointer of type atomic_t 133 * @a: the amount to add to v... 134 * @u: ...unless v is equal to u. 135 * 136 * Atomically adds @a to @v, so long as it was not @u. 137 * Returns the old value of @v. 138 */ 139 static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 140 { 141 int t; 142 143 __asm__ __volatile__ ( 144 PPC_ATOMIC_ENTRY_BARRIER 145 "1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ 146 cmpw 0,%0,%3 \n\ 147 beq 2f \n\ 148 add%I2c %0,%0,%2 \n" 149 " stwcx. %0,0,%1 \n\ 150 bne- 1b \n" 151 PPC_ATOMIC_EXIT_BARRIER 152 " sub%I2c %0,%0,%2 \n\ 153 2:" 154 : "=&r" (t) 155 : "r" (&v->counter), "rI" (a), "r" (u) 156 : "cc", "memory", "xer"); 157 158 return t; 159 } 160 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 161 162 /* 163 * Atomically test *v and decrement if it is greater than 0. 164 * The function returns the old value of *v minus 1, even if 165 * the atomic variable, v, was not decremented. 166 */ 167 static __inline__ int arch_atomic_dec_if_positive(atomic_t *v) 168 { 169 int t; 170 171 __asm__ __volatile__( 172 PPC_ATOMIC_ENTRY_BARRIER 173 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 174 cmpwi %0,1\n\ 175 addi %0,%0,-1\n\ 176 blt- 2f\n" 177 " stwcx. %0,0,%1\n\ 178 bne- 1b" 179 PPC_ATOMIC_EXIT_BARRIER 180 "\n\ 181 2:" : "=&b" (t) 182 : "r" (&v->counter) 183 : "cc", "memory"); 184 185 return t; 186 } 187 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive 188 189 #ifdef __powerpc64__ 190 191 #define ATOMIC64_INIT(i) { (i) } 192 193 static __inline__ s64 arch_atomic64_read(const atomic64_t *v) 194 { 195 s64 t; 196 197 /* -mprefixed can generate offsets beyond range, fall back hack */ 198 if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED)) 199 __asm__ __volatile__("ld %0,0(%1)" : "=r"(t) : "b"(&v->counter)); 200 else 201 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : DS_FORM_CONSTRAINT (v->counter)); 202 203 return t; 204 } 205 206 static __inline__ void arch_atomic64_set(atomic64_t *v, s64 i) 207 { 208 /* -mprefixed can generate offsets beyond range, fall back hack */ 209 if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED)) 210 __asm__ __volatile__("std %1,0(%2)" : "=m"(v->counter) : "r"(i), "b"(&v->counter)); 211 else 212 __asm__ __volatile__("std%U0%X0 %1,%0" : "=" DS_FORM_CONSTRAINT (v->counter) : "r"(i)); 213 } 214 215 #define ATOMIC64_OP(op, asm_op) \ 216 static __inline__ void arch_atomic64_##op(s64 a, atomic64_t *v) \ 217 { \ 218 s64 t; \ 219 \ 220 __asm__ __volatile__( \ 221 "1: ldarx %0,0,%3 # atomic64_" #op "\n" \ 222 #asm_op " %0,%2,%0\n" \ 223 " stdcx. %0,0,%3 \n" \ 224 " bne- 1b\n" \ 225 : "=&r" (t), "+m" (v->counter) \ 226 : "r" (a), "r" (&v->counter) \ 227 : "cc"); \ 228 } 229 230 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 231 static inline s64 \ 232 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 233 { \ 234 s64 t; \ 235 \ 236 __asm__ __volatile__( \ 237 "1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ 238 #asm_op " %0,%2,%0\n" \ 239 " stdcx. %0,0,%3\n" \ 240 " bne- 1b\n" \ 241 : "=&r" (t), "+m" (v->counter) \ 242 : "r" (a), "r" (&v->counter) \ 243 : "cc"); \ 244 \ 245 return t; \ 246 } 247 248 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ 249 static inline s64 \ 250 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 251 { \ 252 s64 res, t; \ 253 \ 254 __asm__ __volatile__( \ 255 "1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ 256 #asm_op " %1,%3,%0\n" \ 257 " stdcx. %1,0,%4\n" \ 258 " bne- 1b\n" \ 259 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 260 : "r" (a), "r" (&v->counter) \ 261 : "cc"); \ 262 \ 263 return res; \ 264 } 265 266 #define ATOMIC64_OPS(op, asm_op) \ 267 ATOMIC64_OP(op, asm_op) \ 268 ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 269 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 270 271 ATOMIC64_OPS(add, add) 272 ATOMIC64_OPS(sub, subf) 273 274 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 275 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 276 277 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 278 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 279 280 #undef ATOMIC64_OPS 281 #define ATOMIC64_OPS(op, asm_op) \ 282 ATOMIC64_OP(op, asm_op) \ 283 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 284 285 ATOMIC64_OPS(and, and) 286 ATOMIC64_OPS(or, or) 287 ATOMIC64_OPS(xor, xor) 288 289 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 290 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 291 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 292 293 #undef ATOPIC64_OPS 294 #undef ATOMIC64_FETCH_OP_RELAXED 295 #undef ATOMIC64_OP_RETURN_RELAXED 296 #undef ATOMIC64_OP 297 298 static __inline__ void arch_atomic64_inc(atomic64_t *v) 299 { 300 s64 t; 301 302 __asm__ __volatile__( 303 "1: ldarx %0,0,%2 # atomic64_inc\n\ 304 addic %0,%0,1\n\ 305 stdcx. %0,0,%2 \n\ 306 bne- 1b" 307 : "=&r" (t), "+m" (v->counter) 308 : "r" (&v->counter) 309 : "cc", "xer"); 310 } 311 #define arch_atomic64_inc arch_atomic64_inc 312 313 static __inline__ s64 arch_atomic64_inc_return_relaxed(atomic64_t *v) 314 { 315 s64 t; 316 317 __asm__ __volatile__( 318 "1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" 319 " addic %0,%0,1\n" 320 " stdcx. %0,0,%2\n" 321 " bne- 1b" 322 : "=&r" (t), "+m" (v->counter) 323 : "r" (&v->counter) 324 : "cc", "xer"); 325 326 return t; 327 } 328 329 static __inline__ void arch_atomic64_dec(atomic64_t *v) 330 { 331 s64 t; 332 333 __asm__ __volatile__( 334 "1: ldarx %0,0,%2 # atomic64_dec\n\ 335 addic %0,%0,-1\n\ 336 stdcx. %0,0,%2\n\ 337 bne- 1b" 338 : "=&r" (t), "+m" (v->counter) 339 : "r" (&v->counter) 340 : "cc", "xer"); 341 } 342 #define arch_atomic64_dec arch_atomic64_dec 343 344 static __inline__ s64 arch_atomic64_dec_return_relaxed(atomic64_t *v) 345 { 346 s64 t; 347 348 __asm__ __volatile__( 349 "1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" 350 " addic %0,%0,-1\n" 351 " stdcx. %0,0,%2\n" 352 " bne- 1b" 353 : "=&r" (t), "+m" (v->counter) 354 : "r" (&v->counter) 355 : "cc", "xer"); 356 357 return t; 358 } 359 360 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed 361 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed 362 363 /* 364 * Atomically test *v and decrement if it is greater than 0. 365 * The function returns the old value of *v minus 1. 366 */ 367 static __inline__ s64 arch_atomic64_dec_if_positive(atomic64_t *v) 368 { 369 s64 t; 370 371 __asm__ __volatile__( 372 PPC_ATOMIC_ENTRY_BARRIER 373 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 374 addic. %0,%0,-1\n\ 375 blt- 2f\n\ 376 stdcx. %0,0,%1\n\ 377 bne- 1b" 378 PPC_ATOMIC_EXIT_BARRIER 379 "\n\ 380 2:" : "=&r" (t) 381 : "r" (&v->counter) 382 : "cc", "xer", "memory"); 383 384 return t; 385 } 386 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 387 388 /** 389 * atomic64_fetch_add_unless - add unless the number is a given value 390 * @v: pointer of type atomic64_t 391 * @a: the amount to add to v... 392 * @u: ...unless v is equal to u. 393 * 394 * Atomically adds @a to @v, so long as it was not @u. 395 * Returns the old value of @v. 396 */ 397 static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 398 { 399 s64 t; 400 401 __asm__ __volatile__ ( 402 PPC_ATOMIC_ENTRY_BARRIER 403 "1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ 404 cmpd 0,%0,%3 \n\ 405 beq 2f \n\ 406 add %0,%2,%0 \n" 407 " stdcx. %0,0,%1 \n\ 408 bne- 1b \n" 409 PPC_ATOMIC_EXIT_BARRIER 410 " subf %0,%2,%0 \n\ 411 2:" 412 : "=&r" (t) 413 : "r" (&v->counter), "r" (a), "r" (u) 414 : "cc", "memory"); 415 416 return t; 417 } 418 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 419 420 /** 421 * atomic_inc64_not_zero - increment unless the number is zero 422 * @v: pointer of type atomic64_t 423 * 424 * Atomically increments @v by 1, so long as @v is non-zero. 425 * Returns non-zero if @v was non-zero, and zero otherwise. 426 */ 427 static __inline__ int arch_atomic64_inc_not_zero(atomic64_t *v) 428 { 429 s64 t1, t2; 430 431 __asm__ __volatile__ ( 432 PPC_ATOMIC_ENTRY_BARRIER 433 "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ 434 cmpdi 0,%0,0\n\ 435 beq- 2f\n\ 436 addic %1,%0,1\n\ 437 stdcx. %1,0,%2\n\ 438 bne- 1b\n" 439 PPC_ATOMIC_EXIT_BARRIER 440 "\n\ 441 2:" 442 : "=&r" (t1), "=&r" (t2) 443 : "r" (&v->counter) 444 : "cc", "xer", "memory"); 445 446 return t1 != 0; 447 } 448 #define arch_atomic64_inc_not_zero(v) arch_atomic64_inc_not_zero((v)) 449 450 #endif /* __powerpc64__ */ 451 452 #endif /* __KERNEL__ */ 453 #endif /* _ASM_POWERPC_ATOMIC_H_ */ 454