1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2008 Marcel Moolenaar 5 * Copyright (c) 2001 Benno Rice 6 * Copyright (c) 2001 David E. O'Brien 7 * Copyright (c) 1998 Doug Rabson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 #ifndef _MACHINE_ATOMIC_H_ 35 #define _MACHINE_ATOMIC_H_ 36 37 #include <sys/atomic_common.h> 38 39 #ifndef __powerpc64__ 40 #include <sys/_atomic64e.h> 41 #endif 42 43 /* 44 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 45 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 46 * of this file. See also Appendix B.2 of Book II of the architecture manual. 47 * 48 * Note that not all Book-E processors accept the light-weight sync variant. 49 * In particular, early models of E500 cores are known to wedge. Bank on all 50 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 51 * to use the heavier-weight sync. 52 */ 53 54 #ifdef __powerpc64__ 55 #define mb() __asm __volatile("sync" : : : "memory") 56 #define rmb() __asm __volatile("lwsync" : : : "memory") 57 #define wmb() __asm __volatile("lwsync" : : : "memory") 58 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 59 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 60 #else 61 #define mb() __asm __volatile("sync" : : : "memory") 62 #define rmb() __asm __volatile("sync" : : : "memory") 63 #define wmb() __asm __volatile("sync" : : : "memory") 64 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 65 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 66 #endif 67 68 static __inline void 69 powerpc_lwsync(void) 70 { 71 72 #ifdef __powerpc64__ 73 __asm __volatile("lwsync" : : : "memory"); 74 #else 75 __asm __volatile("sync" : : : "memory"); 76 #endif 77 } 78 79 /* 80 * atomic_add(p, v) 81 * { *p += v; } 82 */ 83 84 #define __atomic_add_int(p, v, t) \ 85 __asm __volatile( \ 86 "1: lwarx %0, 0, %2\n" \ 87 " add %0, %3, %0\n" \ 88 " stwcx. %0, 0, %2\n" \ 89 " bne- 1b\n" \ 90 : "=&r" (t), "=m" (*p) \ 91 : "r" (p), "r" (v), "m" (*p) \ 92 : "cr0", "memory") \ 93 /* __atomic_add_int */ 94 95 #ifdef __powerpc64__ 96 #define __atomic_add_long(p, v, t) \ 97 __asm __volatile( \ 98 "1: ldarx %0, 0, %2\n" \ 99 " add %0, %3, %0\n" \ 100 " stdcx. %0, 0, %2\n" \ 101 " bne- 1b\n" \ 102 : "=&r" (t), "=m" (*p) \ 103 : "r" (p), "r" (v), "m" (*p) \ 104 : "cr0", "memory") \ 105 /* __atomic_add_long */ 106 #else 107 #define __atomic_add_long(p, v, t) \ 108 __asm __volatile( \ 109 "1: lwarx %0, 0, %2\n" \ 110 " add %0, %3, %0\n" \ 111 " stwcx. %0, 0, %2\n" \ 112 " bne- 1b\n" \ 113 : "=&r" (t), "=m" (*p) \ 114 : "r" (p), "r" (v), "m" (*p) \ 115 : "cr0", "memory") \ 116 /* __atomic_add_long */ 117 #endif 118 119 #define _ATOMIC_ADD(type) \ 120 static __inline void \ 121 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 122 u_##type t; \ 123 __atomic_add_##type(p, v, t); \ 124 } \ 125 \ 126 static __inline void \ 127 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 128 u_##type t; \ 129 __atomic_add_##type(p, v, t); \ 130 __ATOMIC_ACQ(); \ 131 } \ 132 \ 133 static __inline void \ 134 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 135 u_##type t; \ 136 __ATOMIC_REL(); \ 137 __atomic_add_##type(p, v, t); \ 138 } \ 139 /* _ATOMIC_ADD */ 140 141 _ATOMIC_ADD(int) 142 _ATOMIC_ADD(long) 143 144 #define atomic_add_32 atomic_add_int 145 #define atomic_add_acq_32 atomic_add_acq_int 146 #define atomic_add_rel_32 atomic_add_rel_int 147 148 #ifdef __powerpc64__ 149 #define atomic_add_64 atomic_add_long 150 #define atomic_add_acq_64 atomic_add_acq_long 151 #define atomic_add_rel_64 atomic_add_rel_long 152 153 #define atomic_add_ptr atomic_add_long 154 #define atomic_add_acq_ptr atomic_add_acq_long 155 #define atomic_add_rel_ptr atomic_add_rel_long 156 #else 157 #define atomic_add_ptr atomic_add_int 158 #define atomic_add_acq_ptr atomic_add_acq_int 159 #define atomic_add_rel_ptr atomic_add_rel_int 160 #endif 161 #undef _ATOMIC_ADD 162 #undef __atomic_add_long 163 #undef __atomic_add_int 164 165 /* 166 * atomic_clear(p, v) 167 * { *p &= ~v; } 168 */ 169 170 #define __atomic_clear_int(p, v, t) \ 171 __asm __volatile( \ 172 "1: lwarx %0, 0, %2\n" \ 173 " andc %0, %0, %3\n" \ 174 " stwcx. %0, 0, %2\n" \ 175 " bne- 1b\n" \ 176 : "=&r" (t), "=m" (*p) \ 177 : "r" (p), "r" (v), "m" (*p) \ 178 : "cr0", "memory") \ 179 /* __atomic_clear_int */ 180 181 #ifdef __powerpc64__ 182 #define __atomic_clear_long(p, v, t) \ 183 __asm __volatile( \ 184 "1: ldarx %0, 0, %2\n" \ 185 " andc %0, %0, %3\n" \ 186 " stdcx. %0, 0, %2\n" \ 187 " bne- 1b\n" \ 188 : "=&r" (t), "=m" (*p) \ 189 : "r" (p), "r" (v), "m" (*p) \ 190 : "cr0", "memory") \ 191 /* __atomic_clear_long */ 192 #else 193 #define __atomic_clear_long(p, v, t) \ 194 __asm __volatile( \ 195 "1: lwarx %0, 0, %2\n" \ 196 " andc %0, %0, %3\n" \ 197 " stwcx. %0, 0, %2\n" \ 198 " bne- 1b\n" \ 199 : "=&r" (t), "=m" (*p) \ 200 : "r" (p), "r" (v), "m" (*p) \ 201 : "cr0", "memory") \ 202 /* __atomic_clear_long */ 203 #endif 204 205 #define _ATOMIC_CLEAR(type) \ 206 static __inline void \ 207 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 208 u_##type t; \ 209 __atomic_clear_##type(p, v, t); \ 210 } \ 211 \ 212 static __inline void \ 213 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 214 u_##type t; \ 215 __atomic_clear_##type(p, v, t); \ 216 __ATOMIC_ACQ(); \ 217 } \ 218 \ 219 static __inline void \ 220 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 221 u_##type t; \ 222 __ATOMIC_REL(); \ 223 __atomic_clear_##type(p, v, t); \ 224 } \ 225 /* _ATOMIC_CLEAR */ 226 227 _ATOMIC_CLEAR(int) 228 _ATOMIC_CLEAR(long) 229 230 #define atomic_clear_32 atomic_clear_int 231 #define atomic_clear_acq_32 atomic_clear_acq_int 232 #define atomic_clear_rel_32 atomic_clear_rel_int 233 234 #ifdef __powerpc64__ 235 #define atomic_clear_64 atomic_clear_long 236 #define atomic_clear_acq_64 atomic_clear_acq_long 237 #define atomic_clear_rel_64 atomic_clear_rel_long 238 239 #define atomic_clear_ptr atomic_clear_long 240 #define atomic_clear_acq_ptr atomic_clear_acq_long 241 #define atomic_clear_rel_ptr atomic_clear_rel_long 242 #else 243 #define atomic_clear_ptr atomic_clear_int 244 #define atomic_clear_acq_ptr atomic_clear_acq_int 245 #define atomic_clear_rel_ptr atomic_clear_rel_int 246 #endif 247 #undef _ATOMIC_CLEAR 248 #undef __atomic_clear_long 249 #undef __atomic_clear_int 250 251 /* 252 * atomic_cmpset(p, o, n) 253 */ 254 /* TODO -- see below */ 255 256 /* 257 * atomic_load_acq(p) 258 */ 259 /* TODO -- see below */ 260 261 /* 262 * atomic_readandclear(p) 263 */ 264 /* TODO -- see below */ 265 266 /* 267 * atomic_set(p, v) 268 * { *p |= v; } 269 */ 270 271 #define __atomic_set_int(p, v, t) \ 272 __asm __volatile( \ 273 "1: lwarx %0, 0, %2\n" \ 274 " or %0, %3, %0\n" \ 275 " stwcx. %0, 0, %2\n" \ 276 " bne- 1b\n" \ 277 : "=&r" (t), "=m" (*p) \ 278 : "r" (p), "r" (v), "m" (*p) \ 279 : "cr0", "memory") \ 280 /* __atomic_set_int */ 281 282 #ifdef __powerpc64__ 283 #define __atomic_set_long(p, v, t) \ 284 __asm __volatile( \ 285 "1: ldarx %0, 0, %2\n" \ 286 " or %0, %3, %0\n" \ 287 " stdcx. %0, 0, %2\n" \ 288 " bne- 1b\n" \ 289 : "=&r" (t), "=m" (*p) \ 290 : "r" (p), "r" (v), "m" (*p) \ 291 : "cr0", "memory") \ 292 /* __atomic_set_long */ 293 #else 294 #define __atomic_set_long(p, v, t) \ 295 __asm __volatile( \ 296 "1: lwarx %0, 0, %2\n" \ 297 " or %0, %3, %0\n" \ 298 " stwcx. %0, 0, %2\n" \ 299 " bne- 1b\n" \ 300 : "=&r" (t), "=m" (*p) \ 301 : "r" (p), "r" (v), "m" (*p) \ 302 : "cr0", "memory") \ 303 /* __atomic_set_long */ 304 #endif 305 306 #define _ATOMIC_SET(type) \ 307 static __inline void \ 308 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 309 u_##type t; \ 310 __atomic_set_##type(p, v, t); \ 311 } \ 312 \ 313 static __inline void \ 314 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 315 u_##type t; \ 316 __atomic_set_##type(p, v, t); \ 317 __ATOMIC_ACQ(); \ 318 } \ 319 \ 320 static __inline void \ 321 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 322 u_##type t; \ 323 __ATOMIC_REL(); \ 324 __atomic_set_##type(p, v, t); \ 325 } \ 326 /* _ATOMIC_SET */ 327 328 _ATOMIC_SET(int) 329 _ATOMIC_SET(long) 330 331 #define atomic_set_32 atomic_set_int 332 #define atomic_set_acq_32 atomic_set_acq_int 333 #define atomic_set_rel_32 atomic_set_rel_int 334 335 #ifdef __powerpc64__ 336 #define atomic_set_64 atomic_set_long 337 #define atomic_set_acq_64 atomic_set_acq_long 338 #define atomic_set_rel_64 atomic_set_rel_long 339 340 #define atomic_set_ptr atomic_set_long 341 #define atomic_set_acq_ptr atomic_set_acq_long 342 #define atomic_set_rel_ptr atomic_set_rel_long 343 #else 344 #define atomic_set_ptr atomic_set_int 345 #define atomic_set_acq_ptr atomic_set_acq_int 346 #define atomic_set_rel_ptr atomic_set_rel_int 347 #endif 348 #undef _ATOMIC_SET 349 #undef __atomic_set_long 350 #undef __atomic_set_int 351 352 /* 353 * atomic_subtract(p, v) 354 * { *p -= v; } 355 */ 356 357 #define __atomic_subtract_int(p, v, t) \ 358 __asm __volatile( \ 359 "1: lwarx %0, 0, %2\n" \ 360 " subf %0, %3, %0\n" \ 361 " stwcx. %0, 0, %2\n" \ 362 " bne- 1b\n" \ 363 : "=&r" (t), "=m" (*p) \ 364 : "r" (p), "r" (v), "m" (*p) \ 365 : "cr0", "memory") \ 366 /* __atomic_subtract_int */ 367 368 #ifdef __powerpc64__ 369 #define __atomic_subtract_long(p, v, t) \ 370 __asm __volatile( \ 371 "1: ldarx %0, 0, %2\n" \ 372 " subf %0, %3, %0\n" \ 373 " stdcx. %0, 0, %2\n" \ 374 " bne- 1b\n" \ 375 : "=&r" (t), "=m" (*p) \ 376 : "r" (p), "r" (v), "m" (*p) \ 377 : "cr0", "memory") \ 378 /* __atomic_subtract_long */ 379 #else 380 #define __atomic_subtract_long(p, v, t) \ 381 __asm __volatile( \ 382 "1: lwarx %0, 0, %2\n" \ 383 " subf %0, %3, %0\n" \ 384 " stwcx. %0, 0, %2\n" \ 385 " bne- 1b\n" \ 386 : "=&r" (t), "=m" (*p) \ 387 : "r" (p), "r" (v), "m" (*p) \ 388 : "cr0", "memory") \ 389 /* __atomic_subtract_long */ 390 #endif 391 392 #define _ATOMIC_SUBTRACT(type) \ 393 static __inline void \ 394 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 395 u_##type t; \ 396 __atomic_subtract_##type(p, v, t); \ 397 } \ 398 \ 399 static __inline void \ 400 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 401 u_##type t; \ 402 __atomic_subtract_##type(p, v, t); \ 403 __ATOMIC_ACQ(); \ 404 } \ 405 \ 406 static __inline void \ 407 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 408 u_##type t; \ 409 __ATOMIC_REL(); \ 410 __atomic_subtract_##type(p, v, t); \ 411 } \ 412 /* _ATOMIC_SUBTRACT */ 413 414 _ATOMIC_SUBTRACT(int) 415 _ATOMIC_SUBTRACT(long) 416 417 #define atomic_subtract_32 atomic_subtract_int 418 #define atomic_subtract_acq_32 atomic_subtract_acq_int 419 #define atomic_subtract_rel_32 atomic_subtract_rel_int 420 421 #ifdef __powerpc64__ 422 #define atomic_subtract_64 atomic_subtract_long 423 #define atomic_subtract_acq_64 atomic_subract_acq_long 424 #define atomic_subtract_rel_64 atomic_subtract_rel_long 425 426 #define atomic_subtract_ptr atomic_subtract_long 427 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 428 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 429 #else 430 #define atomic_subtract_ptr atomic_subtract_int 431 #define atomic_subtract_acq_ptr atomic_subtract_acq_int 432 #define atomic_subtract_rel_ptr atomic_subtract_rel_int 433 #endif 434 #undef _ATOMIC_SUBTRACT 435 #undef __atomic_subtract_long 436 #undef __atomic_subtract_int 437 438 /* 439 * atomic_store_rel(p, v) 440 */ 441 /* TODO -- see below */ 442 443 /* 444 * Old/original implementations that still need revisiting. 445 */ 446 447 static __inline u_int 448 atomic_readandclear_int(volatile u_int *addr) 449 { 450 u_int result,temp; 451 452 __asm __volatile ( 453 "\tsync\n" /* drain writes */ 454 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 455 "li %1, 0\n\t" /* load new value */ 456 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 457 "bne- 1b\n\t" /* spin if failed */ 458 : "=&r"(result), "=&r"(temp), "=m" (*addr) 459 : "r" (addr), "m" (*addr) 460 : "cr0", "memory"); 461 462 return (result); 463 } 464 465 #ifdef __powerpc64__ 466 static __inline u_long 467 atomic_readandclear_long(volatile u_long *addr) 468 { 469 u_long result,temp; 470 471 __asm __volatile ( 472 "\tsync\n" /* drain writes */ 473 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 474 "li %1, 0\n\t" /* load new value */ 475 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 476 "bne- 1b\n\t" /* spin if failed */ 477 : "=&r"(result), "=&r"(temp), "=m" (*addr) 478 : "r" (addr), "m" (*addr) 479 : "cr0", "memory"); 480 481 return (result); 482 } 483 #endif 484 485 #define atomic_readandclear_32 atomic_readandclear_int 486 487 #ifdef __powerpc64__ 488 #define atomic_readandclear_64 atomic_readandclear_long 489 490 #define atomic_readandclear_ptr atomic_readandclear_long 491 #else 492 static __inline u_long 493 atomic_readandclear_long(volatile u_long *addr) 494 { 495 496 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 497 } 498 499 #define atomic_readandclear_ptr atomic_readandclear_int 500 #endif 501 502 /* 503 * We assume that a = b will do atomic loads and stores. 504 */ 505 #define ATOMIC_STORE_LOAD(TYPE) \ 506 static __inline u_##TYPE \ 507 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 508 { \ 509 u_##TYPE v; \ 510 \ 511 v = *p; \ 512 powerpc_lwsync(); \ 513 return (v); \ 514 } \ 515 \ 516 static __inline void \ 517 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 518 { \ 519 \ 520 powerpc_lwsync(); \ 521 *p = v; \ 522 } 523 524 ATOMIC_STORE_LOAD(int) 525 526 #define atomic_load_acq_32 atomic_load_acq_int 527 #define atomic_store_rel_32 atomic_store_rel_int 528 529 #ifdef __powerpc64__ 530 ATOMIC_STORE_LOAD(long) 531 532 #define atomic_load_acq_64 atomic_load_acq_long 533 #define atomic_store_rel_64 atomic_store_rel_long 534 535 #define atomic_load_acq_ptr atomic_load_acq_long 536 #define atomic_store_rel_ptr atomic_store_rel_long 537 #else 538 static __inline u_long 539 atomic_load_acq_long(volatile u_long *addr) 540 { 541 542 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 543 } 544 545 static __inline void 546 atomic_store_rel_long(volatile u_long *addr, u_long val) 547 { 548 549 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 550 } 551 552 #define atomic_load_acq_ptr atomic_load_acq_int 553 #define atomic_store_rel_ptr atomic_store_rel_int 554 #endif 555 #undef ATOMIC_STORE_LOAD 556 557 /* 558 * Atomically compare the value stored at *p with cmpval and if the 559 * two values are equal, update the value of *p with newval. Returns 560 * zero if the compare failed, nonzero otherwise. 561 */ 562 #ifdef ISA_206_ATOMICS 563 static __inline int 564 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) 565 { 566 int ret; 567 568 __asm __volatile ( 569 "1:\tlbarx %0, 0, %2\n\t" /* load old value */ 570 "cmplw %3, %0\n\t" /* compare */ 571 "bne- 2f\n\t" /* exit if not equal */ 572 "stbcx. %4, 0, %2\n\t" /* attempt to store */ 573 "bne- 1b\n\t" /* spin if failed */ 574 "li %0, 1\n\t" /* success - retval = 1 */ 575 "b 3f\n\t" /* we've succeeded */ 576 "2:\n\t" 577 "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 578 "li %0, 0\n\t" /* failure - retval = 0 */ 579 "3:\n\t" 580 : "=&r" (ret), "=m" (*p) 581 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 582 : "cr0", "memory"); 583 584 return (ret); 585 } 586 587 static __inline int 588 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) 589 { 590 int ret; 591 592 __asm __volatile ( 593 "1:\tlharx %0, 0, %2\n\t" /* load old value */ 594 "cmplw %3, %0\n\t" /* compare */ 595 "bne- 2f\n\t" /* exit if not equal */ 596 "sthcx. %4, 0, %2\n\t" /* attempt to store */ 597 "bne- 1b\n\t" /* spin if failed */ 598 "li %0, 1\n\t" /* success - retval = 1 */ 599 "b 3f\n\t" /* we've succeeded */ 600 "2:\n\t" 601 "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 602 "li %0, 0\n\t" /* failure - retval = 0 */ 603 "3:\n\t" 604 : "=&r" (ret), "=m" (*p) 605 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 606 : "cr0", "memory"); 607 608 return (ret); 609 } 610 #else 611 static __inline int 612 atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval, 613 uint32_t mask) 614 { 615 int ret; 616 uint32_t tmp; 617 618 __asm __volatile ( 619 "1:\tlwarx %2, 0, %3\n\t" /* load old value */ 620 "and %0, %2, %7\n\t" 621 "cmplw %4, %0\n\t" /* compare */ 622 "bne- 2f\n\t" /* exit if not equal */ 623 "andc %2, %2, %7\n\t" 624 "or %2, %2, %5\n\t" 625 "stwcx. %2, 0, %3\n\t" /* attempt to store */ 626 "bne- 1b\n\t" /* spin if failed */ 627 "li %0, 1\n\t" /* success - retval = 1 */ 628 "b 3f\n\t" /* we've succeeded */ 629 "2:\n\t" 630 "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */ 631 "li %0, 0\n\t" /* failure - retval = 0 */ 632 "3:\n\t" 633 : "=&r" (ret), "=m" (*p), "+&r" (tmp) 634 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p), 635 "r" (mask) 636 : "cr0", "memory"); 637 638 return (ret); 639 } 640 641 #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m) 642 #endif 643 644 static __inline int 645 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 646 { 647 int ret; 648 649 __asm __volatile ( 650 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 651 "cmplw %3, %0\n\t" /* compare */ 652 "bne- 2f\n\t" /* exit if not equal */ 653 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 654 "bne- 1b\n\t" /* spin if failed */ 655 "li %0, 1\n\t" /* success - retval = 1 */ 656 "b 3f\n\t" /* we've succeeded */ 657 "2:\n\t" 658 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 659 "li %0, 0\n\t" /* failure - retval = 0 */ 660 "3:\n\t" 661 : "=&r" (ret), "=m" (*p) 662 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 663 : "cr0", "memory"); 664 665 return (ret); 666 } 667 static __inline int 668 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 669 { 670 int ret; 671 672 __asm __volatile ( 673 #ifdef __powerpc64__ 674 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 675 "cmpld %3, %0\n\t" /* compare */ 676 "bne- 2f\n\t" /* exit if not equal */ 677 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 678 #else 679 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 680 "cmplw %3, %0\n\t" /* compare */ 681 "bne- 2f\n\t" /* exit if not equal */ 682 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 683 #endif 684 "bne- 1b\n\t" /* spin if failed */ 685 "li %0, 1\n\t" /* success - retval = 1 */ 686 "b 3f\n\t" /* we've succeeded */ 687 "2:\n\t" 688 #ifdef __powerpc64__ 689 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 690 #else 691 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 692 #endif 693 "li %0, 0\n\t" /* failure - retval = 0 */ 694 "3:\n\t" 695 : "=&r" (ret), "=m" (*p) 696 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 697 : "cr0", "memory"); 698 699 return (ret); 700 } 701 702 #define ATOMIC_CMPSET_ACQ_REL(type) \ 703 static __inline int \ 704 atomic_cmpset_acq_##type(volatile u_##type *p, \ 705 u_##type cmpval, u_##type newval)\ 706 {\ 707 u_##type retval; \ 708 retval = atomic_cmpset_##type(p, cmpval, newval);\ 709 __ATOMIC_ACQ();\ 710 return (retval);\ 711 }\ 712 static __inline int \ 713 atomic_cmpset_rel_##type(volatile u_##type *p, \ 714 u_##type cmpval, u_##type newval)\ 715 {\ 716 __ATOMIC_REL();\ 717 return (atomic_cmpset_##type(p, cmpval, newval));\ 718 }\ 719 struct hack 720 721 ATOMIC_CMPSET_ACQ_REL(int); 722 ATOMIC_CMPSET_ACQ_REL(long); 723 724 #ifdef ISA_206_ATOMICS 725 #define atomic_cmpset_8 atomic_cmpset_char 726 #endif 727 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char 728 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char 729 730 #ifdef ISA_206_ATOMICS 731 #define atomic_cmpset_16 atomic_cmpset_short 732 #endif 733 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short 734 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short 735 736 #define atomic_cmpset_32 atomic_cmpset_int 737 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 738 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 739 740 #ifdef __powerpc64__ 741 #define atomic_cmpset_64 atomic_cmpset_long 742 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 743 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 744 745 #define atomic_cmpset_ptr atomic_cmpset_long 746 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 747 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 748 #else 749 #define atomic_cmpset_ptr atomic_cmpset_int 750 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 751 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 752 #endif 753 754 /* 755 * Atomically compare the value stored at *p with *cmpval and if the 756 * two values are equal, update the value of *p with newval. Returns 757 * zero if the compare failed and sets *cmpval to the read value from *p, 758 * nonzero otherwise. 759 */ 760 #ifdef ISA_206_ATOMICS 761 static __inline int 762 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) 763 { 764 int ret; 765 766 __asm __volatile ( 767 "lbarx %0, 0, %3\n\t" /* load old value */ 768 "cmplw %4, %0\n\t" /* compare */ 769 "bne- 1f\n\t" /* exit if not equal */ 770 "stbcx. %5, 0, %3\n\t" /* attempt to store */ 771 "bne- 1f\n\t" /* exit if failed */ 772 "li %0, 1\n\t" /* success - retval = 1 */ 773 "b 2f\n\t" /* we've succeeded */ 774 "1:\n\t" 775 "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 776 "stbx %0, 0, %7\n\t" 777 "li %0, 0\n\t" /* failure - retval = 0 */ 778 "2:\n\t" 779 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 780 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 781 : "cr0", "memory"); 782 783 return (ret); 784 } 785 786 static __inline int 787 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) 788 { 789 int ret; 790 791 __asm __volatile ( 792 "lharx %0, 0, %3\n\t" /* load old value */ 793 "cmplw %4, %0\n\t" /* compare */ 794 "bne- 1f\n\t" /* exit if not equal */ 795 "sthcx. %5, 0, %3\n\t" /* attempt to store */ 796 "bne- 1f\n\t" /* exit if failed */ 797 "li %0, 1\n\t" /* success - retval = 1 */ 798 "b 2f\n\t" /* we've succeeded */ 799 "1:\n\t" 800 "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 801 "sthx %0, 0, %7\n\t" 802 "li %0, 0\n\t" /* failure - retval = 0 */ 803 "2:\n\t" 804 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 805 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 806 : "cr0", "memory"); 807 808 return (ret); 809 } 810 #endif /* ISA_206_ATOMICS */ 811 812 static __inline int 813 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) 814 { 815 int ret; 816 817 __asm __volatile ( 818 "lwarx %0, 0, %3\n\t" /* load old value */ 819 "cmplw %4, %0\n\t" /* compare */ 820 "bne- 1f\n\t" /* exit if not equal */ 821 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 822 "bne- 1f\n\t" /* exit if failed */ 823 "li %0, 1\n\t" /* success - retval = 1 */ 824 "b 2f\n\t" /* we've succeeded */ 825 "1:\n\t" 826 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 827 "stwx %0, 0, %7\n\t" 828 "li %0, 0\n\t" /* failure - retval = 0 */ 829 "2:\n\t" 830 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 831 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 832 : "cr0", "memory"); 833 834 return (ret); 835 } 836 static __inline int 837 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) 838 { 839 int ret; 840 841 __asm __volatile ( 842 #ifdef __powerpc64__ 843 "ldarx %0, 0, %3\n\t" /* load old value */ 844 "cmpld %4, %0\n\t" /* compare */ 845 "bne- 1f\n\t" /* exit if not equal */ 846 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 847 #else 848 "lwarx %0, 0, %3\n\t" /* load old value */ 849 "cmplw %4, %0\n\t" /* compare */ 850 "bne- 1f\n\t" /* exit if not equal */ 851 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 852 #endif 853 "bne- 1f\n\t" /* exit if failed */ 854 "li %0, 1\n\t" /* success - retval = 1 */ 855 "b 2f\n\t" /* we've succeeded */ 856 "1:\n\t" 857 #ifdef __powerpc64__ 858 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 859 "stdx %0, 0, %7\n\t" 860 #else 861 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 862 "stwx %0, 0, %7\n\t" 863 #endif 864 "li %0, 0\n\t" /* failure - retval = 0 */ 865 "2:\n\t" 866 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 867 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 868 : "cr0", "memory"); 869 870 return (ret); 871 } 872 873 #define ATOMIC_FCMPSET_ACQ_REL(type) \ 874 static __inline int \ 875 atomic_fcmpset_acq_##type(volatile u_##type *p, \ 876 u_##type *cmpval, u_##type newval)\ 877 {\ 878 u_##type retval; \ 879 retval = atomic_fcmpset_##type(p, cmpval, newval);\ 880 __ATOMIC_ACQ();\ 881 return (retval);\ 882 }\ 883 static __inline int \ 884 atomic_fcmpset_rel_##type(volatile u_##type *p, \ 885 u_##type *cmpval, u_##type newval)\ 886 {\ 887 __ATOMIC_REL();\ 888 return (atomic_fcmpset_##type(p, cmpval, newval));\ 889 }\ 890 struct hack 891 892 ATOMIC_FCMPSET_ACQ_REL(int); 893 ATOMIC_FCMPSET_ACQ_REL(long); 894 895 #ifdef ISA_206_ATOMICS 896 #define atomic_fcmpset_8 atomic_fcmpset_char 897 #endif 898 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char 899 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char 900 901 #ifdef ISA_206_ATOMICS 902 #define atomic_fcmpset_16 atomic_fcmpset_short 903 #endif 904 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short 905 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short 906 907 #define atomic_fcmpset_32 atomic_fcmpset_int 908 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 909 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 910 911 #ifdef __powerpc64__ 912 #define atomic_fcmpset_64 atomic_fcmpset_long 913 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long 914 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long 915 916 #define atomic_fcmpset_ptr atomic_fcmpset_long 917 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 918 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 919 #else 920 #define atomic_fcmpset_ptr atomic_fcmpset_int 921 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int 922 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int 923 #endif 924 925 static __inline u_int 926 atomic_fetchadd_int(volatile u_int *p, u_int v) 927 { 928 u_int value; 929 930 do { 931 value = *p; 932 } while (!atomic_cmpset_int(p, value, value + v)); 933 return (value); 934 } 935 936 static __inline u_long 937 atomic_fetchadd_long(volatile u_long *p, u_long v) 938 { 939 u_long value; 940 941 do { 942 value = *p; 943 } while (!atomic_cmpset_long(p, value, value + v)); 944 return (value); 945 } 946 947 static __inline u_int 948 atomic_swap_32(volatile u_int *p, u_int v) 949 { 950 u_int prev; 951 952 __asm __volatile( 953 "1: lwarx %0,0,%2\n" 954 " stwcx. %3,0,%2\n" 955 " bne- 1b\n" 956 : "=&r" (prev), "+m" (*(volatile u_int *)p) 957 : "r" (p), "r" (v) 958 : "cr0", "memory"); 959 960 return (prev); 961 } 962 963 #ifdef __powerpc64__ 964 static __inline u_long 965 atomic_swap_64(volatile u_long *p, u_long v) 966 { 967 u_long prev; 968 969 __asm __volatile( 970 "1: ldarx %0,0,%2\n" 971 " stdcx. %3,0,%2\n" 972 " bne- 1b\n" 973 : "=&r" (prev), "+m" (*(volatile u_long *)p) 974 : "r" (p), "r" (v) 975 : "cr0", "memory"); 976 977 return (prev); 978 } 979 #endif 980 981 #define atomic_fetchadd_32 atomic_fetchadd_int 982 #define atomic_swap_int atomic_swap_32 983 984 #ifdef __powerpc64__ 985 #define atomic_fetchadd_64 atomic_fetchadd_long 986 #define atomic_swap_long atomic_swap_64 987 #define atomic_swap_ptr atomic_swap_64 988 #else 989 #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) 990 #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) 991 #endif 992 993 static __inline int 994 atomic_testandset_int(volatile u_int *p, u_int v) 995 { 996 u_int m = (1u << (v & 0x1f)); 997 u_int res; 998 u_int tmp; 999 1000 __asm __volatile( 1001 "1: lwarx %0,0,%3\n" 1002 " and %1,%0,%4\n" 1003 " or %0,%0,%4\n" 1004 " stwcx. %0,0,%3\n" 1005 " bne- 1b\n" 1006 : "=&r"(tmp), "=&r"(res), "+m"(*p) 1007 : "r"(p), "r"(m) 1008 : "cr0", "memory"); 1009 1010 return (res != 0); 1011 } 1012 1013 static __inline int 1014 atomic_testandclear_int(volatile u_int *p, u_int v) 1015 { 1016 u_int m = (1u << (v & 0x1f)); 1017 u_int res; 1018 u_int tmp; 1019 1020 __asm __volatile( 1021 "1: lwarx %0,0,%3\n" 1022 " and %1,%0,%4\n" 1023 " andc %0,%0,%4\n" 1024 " stwcx. %0,0,%3\n" 1025 " bne- 1b\n" 1026 : "=&r"(tmp), "=&r"(res), "+m"(*p) 1027 : "r"(p), "r"(m) 1028 : "cr0", "memory"); 1029 1030 return (res != 0); 1031 } 1032 1033 #ifdef __powerpc64__ 1034 static __inline int 1035 atomic_testandset_long(volatile u_long *p, u_int v) 1036 { 1037 u_long m = (1ul << (v & 0x3f)); 1038 u_long res; 1039 u_long tmp; 1040 1041 __asm __volatile( 1042 "1: ldarx %0,0,%3\n" 1043 " and %1,%0,%4\n" 1044 " or %0,%0,%4\n" 1045 " stdcx. %0,0,%3\n" 1046 " bne- 1b\n" 1047 : "=&r"(tmp), "=&r"(res), "+m"(*(volatile u_long *)p) 1048 : "r"(p), "r"(m) 1049 : "cr0", "memory"); 1050 1051 return (res != 0); 1052 } 1053 1054 static __inline int 1055 atomic_testandclear_long(volatile u_long *p, u_int v) 1056 { 1057 u_long m = (1ul << (v & 0x3f)); 1058 u_long res; 1059 u_long tmp; 1060 1061 __asm __volatile( 1062 "1: ldarx %0,0,%3\n" 1063 " and %1,%0,%4\n" 1064 " andc %0,%0,%4\n" 1065 " stdcx. %0,0,%3\n" 1066 " bne- 1b\n" 1067 : "=&r"(tmp), "=&r"(res), "+m"(*p) 1068 : "r"(p), "r"(m) 1069 : "cr0", "memory"); 1070 1071 return (res != 0); 1072 } 1073 #else 1074 static __inline int 1075 atomic_testandset_long(volatile u_long *p, u_int v) 1076 { 1077 return (atomic_testandset_int((volatile u_int *)p, v)); 1078 } 1079 1080 static __inline int 1081 atomic_testandclear_long(volatile u_long *p, u_int v) 1082 { 1083 return (atomic_testandclear_int((volatile u_int *)p, v)); 1084 } 1085 #endif 1086 1087 #define atomic_testandclear_32 atomic_testandclear_int 1088 #define atomic_testandset_32 atomic_testandset_int 1089 1090 static __inline int 1091 atomic_testandset_acq_long(volatile u_long *p, u_int v) 1092 { 1093 u_int a = atomic_testandset_long(p, v); 1094 __ATOMIC_ACQ(); 1095 return (a); 1096 } 1097 1098 #define atomic_testandclear_int atomic_testandclear_int 1099 #define atomic_testandset_int atomic_testandset_int 1100 #define atomic_testandclear_long atomic_testandclear_long 1101 #define atomic_testandset_long atomic_testandset_long 1102 #define atomic_testandset_acq_long atomic_testandset_acq_long 1103 1104 static __inline void 1105 atomic_thread_fence_acq(void) 1106 { 1107 1108 powerpc_lwsync(); 1109 } 1110 1111 static __inline void 1112 atomic_thread_fence_rel(void) 1113 { 1114 1115 powerpc_lwsync(); 1116 } 1117 1118 static __inline void 1119 atomic_thread_fence_acq_rel(void) 1120 { 1121 1122 powerpc_lwsync(); 1123 } 1124 1125 static __inline void 1126 atomic_thread_fence_seq_cst(void) 1127 { 1128 1129 __asm __volatile("sync" : : : "memory"); 1130 } 1131 1132 #ifndef ISA_206_ATOMICS 1133 #include <sys/_atomic_subword.h> 1134 #define atomic_cmpset_char atomic_cmpset_8 1135 #define atomic_cmpset_short atomic_cmpset_16 1136 #define atomic_fcmpset_char atomic_fcmpset_8 1137 #define atomic_fcmpset_short atomic_fcmpset_16 1138 #endif 1139 1140 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ 1141 ATOMIC_CMPSET_ACQ_REL(char); 1142 ATOMIC_CMPSET_ACQ_REL(short); 1143 1144 ATOMIC_FCMPSET_ACQ_REL(char); 1145 ATOMIC_FCMPSET_ACQ_REL(short); 1146 1147 #undef __ATOMIC_REL 1148 #undef __ATOMIC_ACQ 1149 1150 #endif /* ! _MACHINE_ATOMIC_H_ */ 1151