1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_POWERPC_ATOMIC_H_ 3 #define _ASM_POWERPC_ATOMIC_H_ 4 5 /* 6 * PowerPC atomic operations 7 */ 8 9 #ifdef __KERNEL__ 10 #include <linux/types.h> 11 #include <asm/cmpxchg.h> 12 #include <asm/barrier.h> 13 14 /* 15 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with 16 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier 17 * on the platform without lwsync. 18 */ 19 #define __atomic_acquire_fence() \ 20 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") 21 22 #define __atomic_release_fence() \ 23 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") 24 25 static __inline__ int atomic_read(const atomic_t *v) 26 { 27 int t; 28 29 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 30 31 return t; 32 } 33 34 static __inline__ void atomic_set(atomic_t *v, int i) 35 { 36 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 37 } 38 39 #define ATOMIC_OP(op, asm_op) \ 40 static __inline__ void atomic_##op(int a, atomic_t *v) \ 41 { \ 42 int t; \ 43 \ 44 __asm__ __volatile__( \ 45 "1: lwarx %0,0,%3 # atomic_" #op "\n" \ 46 #asm_op " %0,%2,%0\n" \ 47 " stwcx. %0,0,%3 \n" \ 48 " bne- 1b\n" \ 49 : "=&r" (t), "+m" (v->counter) \ 50 : "r" (a), "r" (&v->counter) \ 51 : "cc"); \ 52 } \ 53 54 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 55 static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ 56 { \ 57 int t; \ 58 \ 59 __asm__ __volatile__( \ 60 "1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ 61 #asm_op " %0,%2,%0\n" \ 62 " stwcx. %0,0,%3\n" \ 63 " bne- 1b\n" \ 64 : "=&r" (t), "+m" (v->counter) \ 65 : "r" (a), "r" (&v->counter) \ 66 : "cc"); \ 67 \ 68 return t; \ 69 } 70 71 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \ 72 static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ 73 { \ 74 int res, t; \ 75 \ 76 __asm__ __volatile__( \ 77 "1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ 78 #asm_op " %1,%3,%0\n" \ 79 " stwcx. %1,0,%4\n" \ 80 " bne- 1b\n" \ 81 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 82 : "r" (a), "r" (&v->counter) \ 83 : "cc"); \ 84 \ 85 return res; \ 86 } 87 88 #define ATOMIC_OPS(op, asm_op) \ 89 ATOMIC_OP(op, asm_op) \ 90 ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 91 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 92 93 ATOMIC_OPS(add, add) 94 ATOMIC_OPS(sub, subf) 95 96 #define atomic_add_return_relaxed atomic_add_return_relaxed 97 #define atomic_sub_return_relaxed atomic_sub_return_relaxed 98 99 #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed 100 #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed 101 102 #undef ATOMIC_OPS 103 #define ATOMIC_OPS(op, asm_op) \ 104 ATOMIC_OP(op, asm_op) \ 105 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 106 107 ATOMIC_OPS(and, and) 108 ATOMIC_OPS(or, or) 109 ATOMIC_OPS(xor, xor) 110 111 #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed 112 #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed 113 #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed 114 115 #undef ATOMIC_OPS 116 #undef ATOMIC_FETCH_OP_RELAXED 117 #undef ATOMIC_OP_RETURN_RELAXED 118 #undef ATOMIC_OP 119 120 static __inline__ void atomic_inc(atomic_t *v) 121 { 122 int t; 123 124 __asm__ __volatile__( 125 "1: lwarx %0,0,%2 # atomic_inc\n\ 126 addic %0,%0,1\n" 127 " stwcx. %0,0,%2 \n\ 128 bne- 1b" 129 : "=&r" (t), "+m" (v->counter) 130 : "r" (&v->counter) 131 : "cc", "xer"); 132 } 133 #define atomic_inc atomic_inc 134 135 static __inline__ int atomic_inc_return_relaxed(atomic_t *v) 136 { 137 int t; 138 139 __asm__ __volatile__( 140 "1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n" 141 " addic %0,%0,1\n" 142 " stwcx. %0,0,%2\n" 143 " bne- 1b" 144 : "=&r" (t), "+m" (v->counter) 145 : "r" (&v->counter) 146 : "cc", "xer"); 147 148 return t; 149 } 150 151 static __inline__ void atomic_dec(atomic_t *v) 152 { 153 int t; 154 155 __asm__ __volatile__( 156 "1: lwarx %0,0,%2 # atomic_dec\n\ 157 addic %0,%0,-1\n" 158 " stwcx. %0,0,%2\n\ 159 bne- 1b" 160 : "=&r" (t), "+m" (v->counter) 161 : "r" (&v->counter) 162 : "cc", "xer"); 163 } 164 #define atomic_dec atomic_dec 165 166 static __inline__ int atomic_dec_return_relaxed(atomic_t *v) 167 { 168 int t; 169 170 __asm__ __volatile__( 171 "1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n" 172 " addic %0,%0,-1\n" 173 " stwcx. %0,0,%2\n" 174 " bne- 1b" 175 : "=&r" (t), "+m" (v->counter) 176 : "r" (&v->counter) 177 : "cc", "xer"); 178 179 return t; 180 } 181 182 #define atomic_inc_return_relaxed atomic_inc_return_relaxed 183 #define atomic_dec_return_relaxed atomic_dec_return_relaxed 184 185 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 186 #define atomic_cmpxchg_relaxed(v, o, n) \ 187 cmpxchg_relaxed(&((v)->counter), (o), (n)) 188 #define atomic_cmpxchg_acquire(v, o, n) \ 189 cmpxchg_acquire(&((v)->counter), (o), (n)) 190 191 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 192 #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 193 194 /** 195 * atomic_fetch_add_unless - add unless the number is a given value 196 * @v: pointer of type atomic_t 197 * @a: the amount to add to v... 198 * @u: ...unless v is equal to u. 199 * 200 * Atomically adds @a to @v, so long as it was not @u. 201 * Returns the old value of @v. 202 */ 203 static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) 204 { 205 int t; 206 207 __asm__ __volatile__ ( 208 PPC_ATOMIC_ENTRY_BARRIER 209 "1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ 210 cmpw 0,%0,%3 \n\ 211 beq 2f \n\ 212 add %0,%2,%0 \n" 213 " stwcx. %0,0,%1 \n\ 214 bne- 1b \n" 215 PPC_ATOMIC_EXIT_BARRIER 216 " subf %0,%2,%0 \n\ 217 2:" 218 : "=&r" (t) 219 : "r" (&v->counter), "r" (a), "r" (u) 220 : "cc", "memory"); 221 222 return t; 223 } 224 #define atomic_fetch_add_unless atomic_fetch_add_unless 225 226 /** 227 * atomic_inc_not_zero - increment unless the number is zero 228 * @v: pointer of type atomic_t 229 * 230 * Atomically increments @v by 1, so long as @v is non-zero. 231 * Returns non-zero if @v was non-zero, and zero otherwise. 232 */ 233 static __inline__ int atomic_inc_not_zero(atomic_t *v) 234 { 235 int t1, t2; 236 237 __asm__ __volatile__ ( 238 PPC_ATOMIC_ENTRY_BARRIER 239 "1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ 240 cmpwi 0,%0,0\n\ 241 beq- 2f\n\ 242 addic %1,%0,1\n" 243 " stwcx. %1,0,%2\n\ 244 bne- 1b\n" 245 PPC_ATOMIC_EXIT_BARRIER 246 "\n\ 247 2:" 248 : "=&r" (t1), "=&r" (t2) 249 : "r" (&v->counter) 250 : "cc", "xer", "memory"); 251 252 return t1; 253 } 254 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) 255 256 /* 257 * Atomically test *v and decrement if it is greater than 0. 258 * The function returns the old value of *v minus 1, even if 259 * the atomic variable, v, was not decremented. 260 */ 261 static __inline__ int atomic_dec_if_positive(atomic_t *v) 262 { 263 int t; 264 265 __asm__ __volatile__( 266 PPC_ATOMIC_ENTRY_BARRIER 267 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 268 cmpwi %0,1\n\ 269 addi %0,%0,-1\n\ 270 blt- 2f\n" 271 " stwcx. %0,0,%1\n\ 272 bne- 1b" 273 PPC_ATOMIC_EXIT_BARRIER 274 "\n\ 275 2:" : "=&b" (t) 276 : "r" (&v->counter) 277 : "cc", "memory"); 278 279 return t; 280 } 281 #define atomic_dec_if_positive atomic_dec_if_positive 282 283 #ifdef __powerpc64__ 284 285 #define ATOMIC64_INIT(i) { (i) } 286 287 static __inline__ s64 atomic64_read(const atomic64_t *v) 288 { 289 s64 t; 290 291 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 292 293 return t; 294 } 295 296 static __inline__ void atomic64_set(atomic64_t *v, s64 i) 297 { 298 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 299 } 300 301 #define ATOMIC64_OP(op, asm_op) \ 302 static __inline__ void atomic64_##op(s64 a, atomic64_t *v) \ 303 { \ 304 s64 t; \ 305 \ 306 __asm__ __volatile__( \ 307 "1: ldarx %0,0,%3 # atomic64_" #op "\n" \ 308 #asm_op " %0,%2,%0\n" \ 309 " stdcx. %0,0,%3 \n" \ 310 " bne- 1b\n" \ 311 : "=&r" (t), "+m" (v->counter) \ 312 : "r" (a), "r" (&v->counter) \ 313 : "cc"); \ 314 } 315 316 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 317 static inline s64 \ 318 atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 319 { \ 320 s64 t; \ 321 \ 322 __asm__ __volatile__( \ 323 "1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ 324 #asm_op " %0,%2,%0\n" \ 325 " stdcx. %0,0,%3\n" \ 326 " bne- 1b\n" \ 327 : "=&r" (t), "+m" (v->counter) \ 328 : "r" (a), "r" (&v->counter) \ 329 : "cc"); \ 330 \ 331 return t; \ 332 } 333 334 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ 335 static inline s64 \ 336 atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 337 { \ 338 s64 res, t; \ 339 \ 340 __asm__ __volatile__( \ 341 "1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ 342 #asm_op " %1,%3,%0\n" \ 343 " stdcx. %1,0,%4\n" \ 344 " bne- 1b\n" \ 345 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 346 : "r" (a), "r" (&v->counter) \ 347 : "cc"); \ 348 \ 349 return res; \ 350 } 351 352 #define ATOMIC64_OPS(op, asm_op) \ 353 ATOMIC64_OP(op, asm_op) \ 354 ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 355 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 356 357 ATOMIC64_OPS(add, add) 358 ATOMIC64_OPS(sub, subf) 359 360 #define atomic64_add_return_relaxed atomic64_add_return_relaxed 361 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed 362 363 #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed 364 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed 365 366 #undef ATOMIC64_OPS 367 #define ATOMIC64_OPS(op, asm_op) \ 368 ATOMIC64_OP(op, asm_op) \ 369 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 370 371 ATOMIC64_OPS(and, and) 372 ATOMIC64_OPS(or, or) 373 ATOMIC64_OPS(xor, xor) 374 375 #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed 376 #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed 377 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed 378 379 #undef ATOPIC64_OPS 380 #undef ATOMIC64_FETCH_OP_RELAXED 381 #undef ATOMIC64_OP_RETURN_RELAXED 382 #undef ATOMIC64_OP 383 384 static __inline__ void atomic64_inc(atomic64_t *v) 385 { 386 s64 t; 387 388 __asm__ __volatile__( 389 "1: ldarx %0,0,%2 # atomic64_inc\n\ 390 addic %0,%0,1\n\ 391 stdcx. %0,0,%2 \n\ 392 bne- 1b" 393 : "=&r" (t), "+m" (v->counter) 394 : "r" (&v->counter) 395 : "cc", "xer"); 396 } 397 #define atomic64_inc atomic64_inc 398 399 static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v) 400 { 401 s64 t; 402 403 __asm__ __volatile__( 404 "1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" 405 " addic %0,%0,1\n" 406 " stdcx. %0,0,%2\n" 407 " bne- 1b" 408 : "=&r" (t), "+m" (v->counter) 409 : "r" (&v->counter) 410 : "cc", "xer"); 411 412 return t; 413 } 414 415 static __inline__ void atomic64_dec(atomic64_t *v) 416 { 417 s64 t; 418 419 __asm__ __volatile__( 420 "1: ldarx %0,0,%2 # atomic64_dec\n\ 421 addic %0,%0,-1\n\ 422 stdcx. %0,0,%2\n\ 423 bne- 1b" 424 : "=&r" (t), "+m" (v->counter) 425 : "r" (&v->counter) 426 : "cc", "xer"); 427 } 428 #define atomic64_dec atomic64_dec 429 430 static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v) 431 { 432 s64 t; 433 434 __asm__ __volatile__( 435 "1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" 436 " addic %0,%0,-1\n" 437 " stdcx. %0,0,%2\n" 438 " bne- 1b" 439 : "=&r" (t), "+m" (v->counter) 440 : "r" (&v->counter) 441 : "cc", "xer"); 442 443 return t; 444 } 445 446 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed 447 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed 448 449 /* 450 * Atomically test *v and decrement if it is greater than 0. 451 * The function returns the old value of *v minus 1. 452 */ 453 static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v) 454 { 455 s64 t; 456 457 __asm__ __volatile__( 458 PPC_ATOMIC_ENTRY_BARRIER 459 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 460 addic. %0,%0,-1\n\ 461 blt- 2f\n\ 462 stdcx. %0,0,%1\n\ 463 bne- 1b" 464 PPC_ATOMIC_EXIT_BARRIER 465 "\n\ 466 2:" : "=&r" (t) 467 : "r" (&v->counter) 468 : "cc", "xer", "memory"); 469 470 return t; 471 } 472 #define atomic64_dec_if_positive atomic64_dec_if_positive 473 474 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 475 #define atomic64_cmpxchg_relaxed(v, o, n) \ 476 cmpxchg_relaxed(&((v)->counter), (o), (n)) 477 #define atomic64_cmpxchg_acquire(v, o, n) \ 478 cmpxchg_acquire(&((v)->counter), (o), (n)) 479 480 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 481 #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 482 483 /** 484 * atomic64_fetch_add_unless - add unless the number is a given value 485 * @v: pointer of type atomic64_t 486 * @a: the amount to add to v... 487 * @u: ...unless v is equal to u. 488 * 489 * Atomically adds @a to @v, so long as it was not @u. 490 * Returns the old value of @v. 491 */ 492 static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 493 { 494 s64 t; 495 496 __asm__ __volatile__ ( 497 PPC_ATOMIC_ENTRY_BARRIER 498 "1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ 499 cmpd 0,%0,%3 \n\ 500 beq 2f \n\ 501 add %0,%2,%0 \n" 502 " stdcx. %0,0,%1 \n\ 503 bne- 1b \n" 504 PPC_ATOMIC_EXIT_BARRIER 505 " subf %0,%2,%0 \n\ 506 2:" 507 : "=&r" (t) 508 : "r" (&v->counter), "r" (a), "r" (u) 509 : "cc", "memory"); 510 511 return t; 512 } 513 #define atomic64_fetch_add_unless atomic64_fetch_add_unless 514 515 /** 516 * atomic_inc64_not_zero - increment unless the number is zero 517 * @v: pointer of type atomic64_t 518 * 519 * Atomically increments @v by 1, so long as @v is non-zero. 520 * Returns non-zero if @v was non-zero, and zero otherwise. 521 */ 522 static __inline__ int atomic64_inc_not_zero(atomic64_t *v) 523 { 524 s64 t1, t2; 525 526 __asm__ __volatile__ ( 527 PPC_ATOMIC_ENTRY_BARRIER 528 "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ 529 cmpdi 0,%0,0\n\ 530 beq- 2f\n\ 531 addic %1,%0,1\n\ 532 stdcx. %1,0,%2\n\ 533 bne- 1b\n" 534 PPC_ATOMIC_EXIT_BARRIER 535 "\n\ 536 2:" 537 : "=&r" (t1), "=&r" (t2) 538 : "r" (&v->counter) 539 : "cc", "xer", "memory"); 540 541 return t1 != 0; 542 } 543 #define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v)) 544 545 #endif /* __powerpc64__ */ 546 547 #endif /* __KERNEL__ */ 548 #endif /* _ASM_POWERPC_ATOMIC_H_ */ 549