1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2008 Marcel Moolenaar 5 * Copyright (c) 2001 Benno Rice 6 * Copyright (c) 2001 David E. O'Brien 7 * Copyright (c) 1998 Doug Rabson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 #ifndef _MACHINE_ATOMIC_H_ 35 #define _MACHINE_ATOMIC_H_ 36 37 #ifndef _SYS_CDEFS_H_ 38 #error this file needs sys/cdefs.h as a prerequisite 39 #endif 40 41 #include <sys/atomic_common.h> 42 43 /* 44 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 45 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 46 * of this file. See also Appendix B.2 of Book II of the architecture manual. 47 * 48 * Note that not all Book-E processors accept the light-weight sync variant. 49 * In particular, early models of E500 cores are known to wedge. Bank on all 50 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 51 * to use the heavier-weight sync. 52 */ 53 54 #ifdef __powerpc64__ 55 #define mb() __asm __volatile("sync" : : : "memory") 56 #define rmb() __asm __volatile("lwsync" : : : "memory") 57 #define wmb() __asm __volatile("lwsync" : : : "memory") 58 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 59 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 60 #else 61 #define mb() __asm __volatile("sync" : : : "memory") 62 #define rmb() __asm __volatile("sync" : : : "memory") 63 #define wmb() __asm __volatile("sync" : : : "memory") 64 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 65 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 66 #endif 67 68 static __inline void 69 powerpc_lwsync(void) 70 { 71 72 #ifdef __powerpc64__ 73 __asm __volatile("lwsync" : : : "memory"); 74 #else 75 __asm __volatile("sync" : : : "memory"); 76 #endif 77 } 78 79 /* 80 * atomic_add(p, v) 81 * { *p += v; } 82 */ 83 84 #define __atomic_add_int(p, v, t) \ 85 __asm __volatile( \ 86 "1: lwarx %0, 0, %2\n" \ 87 " add %0, %3, %0\n" \ 88 " stwcx. %0, 0, %2\n" \ 89 " bne- 1b\n" \ 90 : "=&r" (t), "=m" (*p) \ 91 : "r" (p), "r" (v), "m" (*p) \ 92 : "cr0", "memory") \ 93 /* __atomic_add_int */ 94 95 #ifdef __powerpc64__ 96 #define __atomic_add_long(p, v, t) \ 97 __asm __volatile( \ 98 "1: ldarx %0, 0, %2\n" \ 99 " add %0, %3, %0\n" \ 100 " stdcx. %0, 0, %2\n" \ 101 " bne- 1b\n" \ 102 : "=&r" (t), "=m" (*p) \ 103 : "r" (p), "r" (v), "m" (*p) \ 104 : "cr0", "memory") \ 105 /* __atomic_add_long */ 106 #else 107 #define __atomic_add_long(p, v, t) \ 108 __asm __volatile( \ 109 "1: lwarx %0, 0, %2\n" \ 110 " add %0, %3, %0\n" \ 111 " stwcx. %0, 0, %2\n" \ 112 " bne- 1b\n" \ 113 : "=&r" (t), "=m" (*p) \ 114 : "r" (p), "r" (v), "m" (*p) \ 115 : "cr0", "memory") \ 116 /* __atomic_add_long */ 117 #endif 118 119 #define _ATOMIC_ADD(type) \ 120 static __inline void \ 121 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 122 u_##type t; \ 123 __atomic_add_##type(p, v, t); \ 124 } \ 125 \ 126 static __inline void \ 127 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 128 u_##type t; \ 129 __atomic_add_##type(p, v, t); \ 130 __ATOMIC_ACQ(); \ 131 } \ 132 \ 133 static __inline void \ 134 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 135 u_##type t; \ 136 __ATOMIC_REL(); \ 137 __atomic_add_##type(p, v, t); \ 138 } \ 139 /* _ATOMIC_ADD */ 140 141 _ATOMIC_ADD(int) 142 _ATOMIC_ADD(long) 143 144 #define atomic_add_32 atomic_add_int 145 #define atomic_add_acq_32 atomic_add_acq_int 146 #define atomic_add_rel_32 atomic_add_rel_int 147 148 #ifdef __powerpc64__ 149 #define atomic_add_64 atomic_add_long 150 #define atomic_add_acq_64 atomic_add_acq_long 151 #define atomic_add_rel_64 atomic_add_rel_long 152 153 #define atomic_add_ptr atomic_add_long 154 #define atomic_add_acq_ptr atomic_add_acq_long 155 #define atomic_add_rel_ptr atomic_add_rel_long 156 #else 157 #define atomic_add_ptr atomic_add_int 158 #define atomic_add_acq_ptr atomic_add_acq_int 159 #define atomic_add_rel_ptr atomic_add_rel_int 160 #endif 161 #undef _ATOMIC_ADD 162 #undef __atomic_add_long 163 #undef __atomic_add_int 164 165 /* 166 * atomic_clear(p, v) 167 * { *p &= ~v; } 168 */ 169 170 #define __atomic_clear_int(p, v, t) \ 171 __asm __volatile( \ 172 "1: lwarx %0, 0, %2\n" \ 173 " andc %0, %0, %3\n" \ 174 " stwcx. %0, 0, %2\n" \ 175 " bne- 1b\n" \ 176 : "=&r" (t), "=m" (*p) \ 177 : "r" (p), "r" (v), "m" (*p) \ 178 : "cr0", "memory") \ 179 /* __atomic_clear_int */ 180 181 #ifdef __powerpc64__ 182 #define __atomic_clear_long(p, v, t) \ 183 __asm __volatile( \ 184 "1: ldarx %0, 0, %2\n" \ 185 " andc %0, %0, %3\n" \ 186 " stdcx. %0, 0, %2\n" \ 187 " bne- 1b\n" \ 188 : "=&r" (t), "=m" (*p) \ 189 : "r" (p), "r" (v), "m" (*p) \ 190 : "cr0", "memory") \ 191 /* __atomic_clear_long */ 192 #else 193 #define __atomic_clear_long(p, v, t) \ 194 __asm __volatile( \ 195 "1: lwarx %0, 0, %2\n" \ 196 " andc %0, %0, %3\n" \ 197 " stwcx. %0, 0, %2\n" \ 198 " bne- 1b\n" \ 199 : "=&r" (t), "=m" (*p) \ 200 : "r" (p), "r" (v), "m" (*p) \ 201 : "cr0", "memory") \ 202 /* __atomic_clear_long */ 203 #endif 204 205 #define _ATOMIC_CLEAR(type) \ 206 static __inline void \ 207 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 208 u_##type t; \ 209 __atomic_clear_##type(p, v, t); \ 210 } \ 211 \ 212 static __inline void \ 213 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 214 u_##type t; \ 215 __atomic_clear_##type(p, v, t); \ 216 __ATOMIC_ACQ(); \ 217 } \ 218 \ 219 static __inline void \ 220 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 221 u_##type t; \ 222 __ATOMIC_REL(); \ 223 __atomic_clear_##type(p, v, t); \ 224 } \ 225 /* _ATOMIC_CLEAR */ 226 227 228 _ATOMIC_CLEAR(int) 229 _ATOMIC_CLEAR(long) 230 231 #define atomic_clear_32 atomic_clear_int 232 #define atomic_clear_acq_32 atomic_clear_acq_int 233 #define atomic_clear_rel_32 atomic_clear_rel_int 234 235 #ifdef __powerpc64__ 236 #define atomic_clear_64 atomic_clear_long 237 #define atomic_clear_acq_64 atomic_clear_acq_long 238 #define atomic_clear_rel_64 atomic_clear_rel_long 239 240 #define atomic_clear_ptr atomic_clear_long 241 #define atomic_clear_acq_ptr atomic_clear_acq_long 242 #define atomic_clear_rel_ptr atomic_clear_rel_long 243 #else 244 #define atomic_clear_ptr atomic_clear_int 245 #define atomic_clear_acq_ptr atomic_clear_acq_int 246 #define atomic_clear_rel_ptr atomic_clear_rel_int 247 #endif 248 #undef _ATOMIC_CLEAR 249 #undef __atomic_clear_long 250 #undef __atomic_clear_int 251 252 /* 253 * atomic_cmpset(p, o, n) 254 */ 255 /* TODO -- see below */ 256 257 /* 258 * atomic_load_acq(p) 259 */ 260 /* TODO -- see below */ 261 262 /* 263 * atomic_readandclear(p) 264 */ 265 /* TODO -- see below */ 266 267 /* 268 * atomic_set(p, v) 269 * { *p |= v; } 270 */ 271 272 #define __atomic_set_int(p, v, t) \ 273 __asm __volatile( \ 274 "1: lwarx %0, 0, %2\n" \ 275 " or %0, %3, %0\n" \ 276 " stwcx. %0, 0, %2\n" \ 277 " bne- 1b\n" \ 278 : "=&r" (t), "=m" (*p) \ 279 : "r" (p), "r" (v), "m" (*p) \ 280 : "cr0", "memory") \ 281 /* __atomic_set_int */ 282 283 #ifdef __powerpc64__ 284 #define __atomic_set_long(p, v, t) \ 285 __asm __volatile( \ 286 "1: ldarx %0, 0, %2\n" \ 287 " or %0, %3, %0\n" \ 288 " stdcx. %0, 0, %2\n" \ 289 " bne- 1b\n" \ 290 : "=&r" (t), "=m" (*p) \ 291 : "r" (p), "r" (v), "m" (*p) \ 292 : "cr0", "memory") \ 293 /* __atomic_set_long */ 294 #else 295 #define __atomic_set_long(p, v, t) \ 296 __asm __volatile( \ 297 "1: lwarx %0, 0, %2\n" \ 298 " or %0, %3, %0\n" \ 299 " stwcx. %0, 0, %2\n" \ 300 " bne- 1b\n" \ 301 : "=&r" (t), "=m" (*p) \ 302 : "r" (p), "r" (v), "m" (*p) \ 303 : "cr0", "memory") \ 304 /* __atomic_set_long */ 305 #endif 306 307 #define _ATOMIC_SET(type) \ 308 static __inline void \ 309 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 310 u_##type t; \ 311 __atomic_set_##type(p, v, t); \ 312 } \ 313 \ 314 static __inline void \ 315 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 316 u_##type t; \ 317 __atomic_set_##type(p, v, t); \ 318 __ATOMIC_ACQ(); \ 319 } \ 320 \ 321 static __inline void \ 322 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 323 u_##type t; \ 324 __ATOMIC_REL(); \ 325 __atomic_set_##type(p, v, t); \ 326 } \ 327 /* _ATOMIC_SET */ 328 329 _ATOMIC_SET(int) 330 _ATOMIC_SET(long) 331 332 #define atomic_set_32 atomic_set_int 333 #define atomic_set_acq_32 atomic_set_acq_int 334 #define atomic_set_rel_32 atomic_set_rel_int 335 336 #ifdef __powerpc64__ 337 #define atomic_set_64 atomic_set_long 338 #define atomic_set_acq_64 atomic_set_acq_long 339 #define atomic_set_rel_64 atomic_set_rel_long 340 341 #define atomic_set_ptr atomic_set_long 342 #define atomic_set_acq_ptr atomic_set_acq_long 343 #define atomic_set_rel_ptr atomic_set_rel_long 344 #else 345 #define atomic_set_ptr atomic_set_int 346 #define atomic_set_acq_ptr atomic_set_acq_int 347 #define atomic_set_rel_ptr atomic_set_rel_int 348 #endif 349 #undef _ATOMIC_SET 350 #undef __atomic_set_long 351 #undef __atomic_set_int 352 353 /* 354 * atomic_subtract(p, v) 355 * { *p -= v; } 356 */ 357 358 #define __atomic_subtract_int(p, v, t) \ 359 __asm __volatile( \ 360 "1: lwarx %0, 0, %2\n" \ 361 " subf %0, %3, %0\n" \ 362 " stwcx. %0, 0, %2\n" \ 363 " bne- 1b\n" \ 364 : "=&r" (t), "=m" (*p) \ 365 : "r" (p), "r" (v), "m" (*p) \ 366 : "cr0", "memory") \ 367 /* __atomic_subtract_int */ 368 369 #ifdef __powerpc64__ 370 #define __atomic_subtract_long(p, v, t) \ 371 __asm __volatile( \ 372 "1: ldarx %0, 0, %2\n" \ 373 " subf %0, %3, %0\n" \ 374 " stdcx. %0, 0, %2\n" \ 375 " bne- 1b\n" \ 376 : "=&r" (t), "=m" (*p) \ 377 : "r" (p), "r" (v), "m" (*p) \ 378 : "cr0", "memory") \ 379 /* __atomic_subtract_long */ 380 #else 381 #define __atomic_subtract_long(p, v, t) \ 382 __asm __volatile( \ 383 "1: lwarx %0, 0, %2\n" \ 384 " subf %0, %3, %0\n" \ 385 " stwcx. %0, 0, %2\n" \ 386 " bne- 1b\n" \ 387 : "=&r" (t), "=m" (*p) \ 388 : "r" (p), "r" (v), "m" (*p) \ 389 : "cr0", "memory") \ 390 /* __atomic_subtract_long */ 391 #endif 392 393 #define _ATOMIC_SUBTRACT(type) \ 394 static __inline void \ 395 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 396 u_##type t; \ 397 __atomic_subtract_##type(p, v, t); \ 398 } \ 399 \ 400 static __inline void \ 401 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 402 u_##type t; \ 403 __atomic_subtract_##type(p, v, t); \ 404 __ATOMIC_ACQ(); \ 405 } \ 406 \ 407 static __inline void \ 408 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 409 u_##type t; \ 410 __ATOMIC_REL(); \ 411 __atomic_subtract_##type(p, v, t); \ 412 } \ 413 /* _ATOMIC_SUBTRACT */ 414 415 _ATOMIC_SUBTRACT(int) 416 _ATOMIC_SUBTRACT(long) 417 418 #define atomic_subtract_32 atomic_subtract_int 419 #define atomic_subtract_acq_32 atomic_subtract_acq_int 420 #define atomic_subtract_rel_32 atomic_subtract_rel_int 421 422 #ifdef __powerpc64__ 423 #define atomic_subtract_64 atomic_subtract_long 424 #define atomic_subtract_acq_64 atomic_subract_acq_long 425 #define atomic_subtract_rel_64 atomic_subtract_rel_long 426 427 #define atomic_subtract_ptr atomic_subtract_long 428 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 429 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 430 #else 431 #define atomic_subtract_ptr atomic_subtract_int 432 #define atomic_subtract_acq_ptr atomic_subtract_acq_int 433 #define atomic_subtract_rel_ptr atomic_subtract_rel_int 434 #endif 435 #undef _ATOMIC_SUBTRACT 436 #undef __atomic_subtract_long 437 #undef __atomic_subtract_int 438 439 /* 440 * atomic_store_rel(p, v) 441 */ 442 /* TODO -- see below */ 443 444 /* 445 * Old/original implementations that still need revisiting. 446 */ 447 448 static __inline u_int 449 atomic_readandclear_int(volatile u_int *addr) 450 { 451 u_int result,temp; 452 453 __asm __volatile ( 454 "\tsync\n" /* drain writes */ 455 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 456 "li %1, 0\n\t" /* load new value */ 457 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 458 "bne- 1b\n\t" /* spin if failed */ 459 : "=&r"(result), "=&r"(temp), "=m" (*addr) 460 : "r" (addr), "m" (*addr) 461 : "cr0", "memory"); 462 463 return (result); 464 } 465 466 #ifdef __powerpc64__ 467 static __inline u_long 468 atomic_readandclear_long(volatile u_long *addr) 469 { 470 u_long result,temp; 471 472 __asm __volatile ( 473 "\tsync\n" /* drain writes */ 474 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 475 "li %1, 0\n\t" /* load new value */ 476 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 477 "bne- 1b\n\t" /* spin if failed */ 478 : "=&r"(result), "=&r"(temp), "=m" (*addr) 479 : "r" (addr), "m" (*addr) 480 : "cr0", "memory"); 481 482 return (result); 483 } 484 #endif 485 486 #define atomic_readandclear_32 atomic_readandclear_int 487 488 #ifdef __powerpc64__ 489 #define atomic_readandclear_64 atomic_readandclear_long 490 491 #define atomic_readandclear_ptr atomic_readandclear_long 492 #else 493 static __inline u_long 494 atomic_readandclear_long(volatile u_long *addr) 495 { 496 497 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 498 } 499 500 #define atomic_readandclear_ptr atomic_readandclear_int 501 #endif 502 503 /* 504 * We assume that a = b will do atomic loads and stores. 505 */ 506 #define ATOMIC_STORE_LOAD(TYPE) \ 507 static __inline u_##TYPE \ 508 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 509 { \ 510 u_##TYPE v; \ 511 \ 512 v = *p; \ 513 powerpc_lwsync(); \ 514 return (v); \ 515 } \ 516 \ 517 static __inline void \ 518 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 519 { \ 520 \ 521 powerpc_lwsync(); \ 522 *p = v; \ 523 } 524 525 ATOMIC_STORE_LOAD(int) 526 527 #define atomic_load_acq_32 atomic_load_acq_int 528 #define atomic_store_rel_32 atomic_store_rel_int 529 530 #ifdef __powerpc64__ 531 ATOMIC_STORE_LOAD(long) 532 533 #define atomic_load_acq_64 atomic_load_acq_long 534 #define atomic_store_rel_64 atomic_store_rel_long 535 536 #define atomic_load_acq_ptr atomic_load_acq_long 537 #define atomic_store_rel_ptr atomic_store_rel_long 538 #else 539 static __inline u_long 540 atomic_load_acq_long(volatile u_long *addr) 541 { 542 543 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 544 } 545 546 static __inline void 547 atomic_store_rel_long(volatile u_long *addr, u_long val) 548 { 549 550 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 551 } 552 553 #define atomic_load_acq_ptr atomic_load_acq_int 554 #define atomic_store_rel_ptr atomic_store_rel_int 555 #endif 556 #undef ATOMIC_STORE_LOAD 557 558 /* 559 * Atomically compare the value stored at *p with cmpval and if the 560 * two values are equal, update the value of *p with newval. Returns 561 * zero if the compare failed, nonzero otherwise. 562 */ 563 static __inline int 564 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 565 { 566 int ret; 567 568 __asm __volatile ( 569 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 570 "cmplw %3, %0\n\t" /* compare */ 571 "bne 2f\n\t" /* exit if not equal */ 572 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 573 "bne- 1b\n\t" /* spin if failed */ 574 "li %0, 1\n\t" /* success - retval = 1 */ 575 "b 3f\n\t" /* we've succeeded */ 576 "2:\n\t" 577 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 578 "li %0, 0\n\t" /* failure - retval = 0 */ 579 "3:\n\t" 580 : "=&r" (ret), "=m" (*p) 581 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 582 : "cr0", "memory"); 583 584 return (ret); 585 } 586 static __inline int 587 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 588 { 589 int ret; 590 591 __asm __volatile ( 592 #ifdef __powerpc64__ 593 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 594 "cmpld %3, %0\n\t" /* compare */ 595 "bne 2f\n\t" /* exit if not equal */ 596 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 597 #else 598 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 599 "cmplw %3, %0\n\t" /* compare */ 600 "bne 2f\n\t" /* exit if not equal */ 601 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 602 #endif 603 "bne- 1b\n\t" /* spin if failed */ 604 "li %0, 1\n\t" /* success - retval = 1 */ 605 "b 3f\n\t" /* we've succeeded */ 606 "2:\n\t" 607 #ifdef __powerpc64__ 608 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 609 #else 610 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 611 #endif 612 "li %0, 0\n\t" /* failure - retval = 0 */ 613 "3:\n\t" 614 : "=&r" (ret), "=m" (*p) 615 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 616 : "cr0", "memory"); 617 618 return (ret); 619 } 620 621 static __inline int 622 atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) 623 { 624 int retval; 625 626 retval = atomic_cmpset_int(p, cmpval, newval); 627 __ATOMIC_ACQ(); 628 return (retval); 629 } 630 631 static __inline int 632 atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) 633 { 634 __ATOMIC_REL(); 635 return (atomic_cmpset_int(p, cmpval, newval)); 636 } 637 638 static __inline int 639 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) 640 { 641 u_long retval; 642 643 retval = atomic_cmpset_long(p, cmpval, newval); 644 __ATOMIC_ACQ(); 645 return (retval); 646 } 647 648 static __inline int 649 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) 650 { 651 __ATOMIC_REL(); 652 return (atomic_cmpset_long(p, cmpval, newval)); 653 } 654 655 #define atomic_cmpset_32 atomic_cmpset_int 656 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 657 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 658 659 #ifdef __powerpc64__ 660 #define atomic_cmpset_64 atomic_cmpset_long 661 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 662 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 663 664 #define atomic_cmpset_ptr atomic_cmpset_long 665 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 666 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 667 #else 668 #define atomic_cmpset_ptr atomic_cmpset_int 669 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 670 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 671 #endif 672 673 /* 674 * Atomically compare the value stored at *p with *cmpval and if the 675 * two values are equal, update the value of *p with newval. Returns 676 * zero if the compare failed and sets *cmpval to the read value from *p, 677 * nonzero otherwise. 678 */ 679 static __inline int 680 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) 681 { 682 int ret; 683 684 __asm __volatile ( 685 "lwarx %0, 0, %3\n\t" /* load old value */ 686 "cmplw %4, %0\n\t" /* compare */ 687 "bne 1f\n\t" /* exit if not equal */ 688 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 689 "bne- 1f\n\t" /* exit if failed */ 690 "li %0, 1\n\t" /* success - retval = 1 */ 691 "b 2f\n\t" /* we've succeeded */ 692 "1:\n\t" 693 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 694 "stwx %0, 0, %7\n\t" 695 "li %0, 0\n\t" /* failure - retval = 0 */ 696 "2:\n\t" 697 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 698 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 699 : "cr0", "memory"); 700 701 return (ret); 702 } 703 static __inline int 704 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) 705 { 706 int ret; 707 708 __asm __volatile ( 709 #ifdef __powerpc64__ 710 "ldarx %0, 0, %3\n\t" /* load old value */ 711 "cmpld %4, %0\n\t" /* compare */ 712 "bne 1f\n\t" /* exit if not equal */ 713 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 714 #else 715 "lwarx %0, 0, %3\n\t" /* load old value */ 716 "cmplw %4, %0\n\t" /* compare */ 717 "bne 1f\n\t" /* exit if not equal */ 718 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 719 #endif 720 "bne- 1f\n\t" /* exit if failed */ 721 "li %0, 1\n\t" /* success - retval = 1 */ 722 "b 2f\n\t" /* we've succeeded */ 723 "1:\n\t" 724 #ifdef __powerpc64__ 725 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 726 "stdx %0, 0, %7\n\t" 727 #else 728 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 729 "stwx %0, 0, %7\n\t" 730 #endif 731 "li %0, 0\n\t" /* failure - retval = 0 */ 732 "2:\n\t" 733 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 734 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 735 : "cr0", "memory"); 736 737 return (ret); 738 } 739 740 static __inline int 741 atomic_fcmpset_acq_int(volatile u_int *p, u_int *cmpval, u_int newval) 742 { 743 int retval; 744 745 retval = atomic_fcmpset_int(p, cmpval, newval); 746 __ATOMIC_ACQ(); 747 return (retval); 748 } 749 750 static __inline int 751 atomic_fcmpset_rel_int(volatile u_int *p, u_int *cmpval, u_int newval) 752 { 753 __ATOMIC_REL(); 754 return (atomic_fcmpset_int(p, cmpval, newval)); 755 } 756 757 static __inline int 758 atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval) 759 { 760 u_long retval; 761 762 retval = atomic_fcmpset_long(p, cmpval, newval); 763 __ATOMIC_ACQ(); 764 return (retval); 765 } 766 767 static __inline int 768 atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval) 769 { 770 __ATOMIC_REL(); 771 return (atomic_fcmpset_long(p, cmpval, newval)); 772 } 773 774 #define atomic_fcmpset_32 atomic_fcmpset_int 775 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 776 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 777 778 #ifdef __powerpc64__ 779 #define atomic_fcmpset_64 atomic_fcmpset_long 780 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long 781 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long 782 783 #define atomic_fcmpset_ptr atomic_fcmpset_long 784 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 785 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 786 #else 787 #define atomic_fcmpset_ptr atomic_fcmpset_int 788 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int 789 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int 790 #endif 791 792 static __inline u_int 793 atomic_fetchadd_int(volatile u_int *p, u_int v) 794 { 795 u_int value; 796 797 do { 798 value = *p; 799 } while (!atomic_cmpset_int(p, value, value + v)); 800 return (value); 801 } 802 803 static __inline u_long 804 atomic_fetchadd_long(volatile u_long *p, u_long v) 805 { 806 u_long value; 807 808 do { 809 value = *p; 810 } while (!atomic_cmpset_long(p, value, value + v)); 811 return (value); 812 } 813 814 static __inline u_int 815 atomic_swap_32(volatile u_int *p, u_int v) 816 { 817 u_int prev; 818 819 __asm __volatile( 820 "1: lwarx %0,0,%2\n" 821 " stwcx. %3,0,%2\n" 822 " bne- 1b\n" 823 : "=&r" (prev), "+m" (*(volatile u_int *)p) 824 : "r" (p), "r" (v) 825 : "cr0", "memory"); 826 827 return (prev); 828 } 829 830 #ifdef __powerpc64__ 831 static __inline u_long 832 atomic_swap_64(volatile u_long *p, u_long v) 833 { 834 u_long prev; 835 836 __asm __volatile( 837 "1: ldarx %0,0,%2\n" 838 " stdcx. %3,0,%2\n" 839 " bne- 1b\n" 840 : "=&r" (prev), "+m" (*(volatile u_long *)p) 841 : "r" (p), "r" (v) 842 : "cr0", "memory"); 843 844 return (prev); 845 } 846 #endif 847 848 #define atomic_fetchadd_32 atomic_fetchadd_int 849 #define atomic_swap_int atomic_swap_32 850 851 #ifdef __powerpc64__ 852 #define atomic_fetchadd_64 atomic_fetchadd_long 853 #define atomic_swap_long atomic_swap_64 854 #define atomic_swap_ptr atomic_swap_64 855 #else 856 #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) 857 #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) 858 #endif 859 860 #undef __ATOMIC_REL 861 #undef __ATOMIC_ACQ 862 863 static __inline void 864 atomic_thread_fence_acq(void) 865 { 866 867 powerpc_lwsync(); 868 } 869 870 static __inline void 871 atomic_thread_fence_rel(void) 872 { 873 874 powerpc_lwsync(); 875 } 876 877 static __inline void 878 atomic_thread_fence_acq_rel(void) 879 { 880 881 powerpc_lwsync(); 882 } 883 884 static __inline void 885 atomic_thread_fence_seq_cst(void) 886 { 887 888 __asm __volatile("sync" : : : "memory"); 889 } 890 891 #endif /* ! _MACHINE_ATOMIC_H_ */ 892