1 /*- 2 * Copyright (c) 2008 Marcel Moolenaar 3 * Copyright (c) 2001 Benno Rice 4 * Copyright (c) 2001 David E. O'Brien 5 * Copyright (c) 1998 Doug Rabson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #ifndef _MACHINE_ATOMIC_H_ 33 #define _MACHINE_ATOMIC_H_ 34 35 #ifndef _SYS_CDEFS_H_ 36 #error this file needs sys/cdefs.h as a prerequisite 37 #endif 38 39 /* 40 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 41 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 42 * of this file. See also Appendix B.2 of Book II of the architecture manual. 43 * 44 * Note that not all Book-E processors accept the light-weight sync variant. 45 * In particular, early models of E500 cores are known to wedge. Bank on all 46 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 47 * to use the heavier-weight sync. 48 */ 49 50 #ifdef __powerpc64__ 51 #define mb() __asm __volatile("sync" : : : "memory") 52 #define rmb() __asm __volatile("lwsync" : : : "memory") 53 #define wmb() __asm __volatile("lwsync" : : : "memory") 54 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 55 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 56 #else 57 #define mb() __asm __volatile("sync" : : : "memory") 58 #define rmb() __asm __volatile("sync" : : : "memory") 59 #define wmb() __asm __volatile("sync" : : : "memory") 60 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 61 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 62 #endif 63 64 static __inline void 65 powerpc_lwsync(void) 66 { 67 68 #ifdef __powerpc64__ 69 __asm __volatile("lwsync" : : : "memory"); 70 #else 71 __asm __volatile("sync" : : : "memory"); 72 #endif 73 } 74 75 /* 76 * atomic_add(p, v) 77 * { *p += v; } 78 */ 79 80 #define __atomic_add_int(p, v, t) \ 81 __asm __volatile( \ 82 "1: lwarx %0, 0, %2\n" \ 83 " add %0, %3, %0\n" \ 84 " stwcx. %0, 0, %2\n" \ 85 " bne- 1b\n" \ 86 : "=&r" (t), "=m" (*p) \ 87 : "r" (p), "r" (v), "m" (*p) \ 88 : "cr0", "memory") \ 89 /* __atomic_add_int */ 90 91 #ifdef __powerpc64__ 92 #define __atomic_add_long(p, v, t) \ 93 __asm __volatile( \ 94 "1: ldarx %0, 0, %2\n" \ 95 " add %0, %3, %0\n" \ 96 " stdcx. %0, 0, %2\n" \ 97 " bne- 1b\n" \ 98 : "=&r" (t), "=m" (*p) \ 99 : "r" (p), "r" (v), "m" (*p) \ 100 : "cr0", "memory") \ 101 /* __atomic_add_long */ 102 #else 103 #define __atomic_add_long(p, v, t) \ 104 __asm __volatile( \ 105 "1: lwarx %0, 0, %2\n" \ 106 " add %0, %3, %0\n" \ 107 " stwcx. %0, 0, %2\n" \ 108 " bne- 1b\n" \ 109 : "=&r" (t), "=m" (*p) \ 110 : "r" (p), "r" (v), "m" (*p) \ 111 : "cr0", "memory") \ 112 /* __atomic_add_long */ 113 #endif 114 115 #define _ATOMIC_ADD(type) \ 116 static __inline void \ 117 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 118 u_##type t; \ 119 __atomic_add_##type(p, v, t); \ 120 } \ 121 \ 122 static __inline void \ 123 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 124 u_##type t; \ 125 __atomic_add_##type(p, v, t); \ 126 __ATOMIC_ACQ(); \ 127 } \ 128 \ 129 static __inline void \ 130 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 131 u_##type t; \ 132 __ATOMIC_REL(); \ 133 __atomic_add_##type(p, v, t); \ 134 } \ 135 /* _ATOMIC_ADD */ 136 137 _ATOMIC_ADD(int) 138 _ATOMIC_ADD(long) 139 140 #define atomic_add_32 atomic_add_int 141 #define atomic_add_acq_32 atomic_add_acq_int 142 #define atomic_add_rel_32 atomic_add_rel_int 143 144 #ifdef __powerpc64__ 145 #define atomic_add_64 atomic_add_long 146 #define atomic_add_acq_64 atomic_add_acq_long 147 #define atomic_add_rel_64 atomic_add_rel_long 148 149 #define atomic_add_ptr atomic_add_long 150 #define atomic_add_acq_ptr atomic_add_acq_long 151 #define atomic_add_rel_ptr atomic_add_rel_long 152 #else 153 #define atomic_add_ptr atomic_add_int 154 #define atomic_add_acq_ptr atomic_add_acq_int 155 #define atomic_add_rel_ptr atomic_add_rel_int 156 #endif 157 #undef _ATOMIC_ADD 158 #undef __atomic_add_long 159 #undef __atomic_add_int 160 161 /* 162 * atomic_clear(p, v) 163 * { *p &= ~v; } 164 */ 165 166 #define __atomic_clear_int(p, v, t) \ 167 __asm __volatile( \ 168 "1: lwarx %0, 0, %2\n" \ 169 " andc %0, %0, %3\n" \ 170 " stwcx. %0, 0, %2\n" \ 171 " bne- 1b\n" \ 172 : "=&r" (t), "=m" (*p) \ 173 : "r" (p), "r" (v), "m" (*p) \ 174 : "cr0", "memory") \ 175 /* __atomic_clear_int */ 176 177 #ifdef __powerpc64__ 178 #define __atomic_clear_long(p, v, t) \ 179 __asm __volatile( \ 180 "1: ldarx %0, 0, %2\n" \ 181 " andc %0, %0, %3\n" \ 182 " stdcx. %0, 0, %2\n" \ 183 " bne- 1b\n" \ 184 : "=&r" (t), "=m" (*p) \ 185 : "r" (p), "r" (v), "m" (*p) \ 186 : "cr0", "memory") \ 187 /* __atomic_clear_long */ 188 #else 189 #define __atomic_clear_long(p, v, t) \ 190 __asm __volatile( \ 191 "1: lwarx %0, 0, %2\n" \ 192 " andc %0, %0, %3\n" \ 193 " stwcx. %0, 0, %2\n" \ 194 " bne- 1b\n" \ 195 : "=&r" (t), "=m" (*p) \ 196 : "r" (p), "r" (v), "m" (*p) \ 197 : "cr0", "memory") \ 198 /* __atomic_clear_long */ 199 #endif 200 201 #define _ATOMIC_CLEAR(type) \ 202 static __inline void \ 203 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 204 u_##type t; \ 205 __atomic_clear_##type(p, v, t); \ 206 } \ 207 \ 208 static __inline void \ 209 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 210 u_##type t; \ 211 __atomic_clear_##type(p, v, t); \ 212 __ATOMIC_ACQ(); \ 213 } \ 214 \ 215 static __inline void \ 216 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 217 u_##type t; \ 218 __ATOMIC_REL(); \ 219 __atomic_clear_##type(p, v, t); \ 220 } \ 221 /* _ATOMIC_CLEAR */ 222 223 224 _ATOMIC_CLEAR(int) 225 _ATOMIC_CLEAR(long) 226 227 #define atomic_clear_32 atomic_clear_int 228 #define atomic_clear_acq_32 atomic_clear_acq_int 229 #define atomic_clear_rel_32 atomic_clear_rel_int 230 231 #ifdef __powerpc64__ 232 #define atomic_clear_64 atomic_clear_long 233 #define atomic_clear_acq_64 atomic_clear_acq_long 234 #define atomic_clear_rel_64 atomic_clear_rel_long 235 236 #define atomic_clear_ptr atomic_clear_long 237 #define atomic_clear_acq_ptr atomic_clear_acq_long 238 #define atomic_clear_rel_ptr atomic_clear_rel_long 239 #else 240 #define atomic_clear_ptr atomic_clear_int 241 #define atomic_clear_acq_ptr atomic_clear_acq_int 242 #define atomic_clear_rel_ptr atomic_clear_rel_int 243 #endif 244 #undef _ATOMIC_CLEAR 245 #undef __atomic_clear_long 246 #undef __atomic_clear_int 247 248 /* 249 * atomic_cmpset(p, o, n) 250 */ 251 /* TODO -- see below */ 252 253 /* 254 * atomic_load_acq(p) 255 */ 256 /* TODO -- see below */ 257 258 /* 259 * atomic_readandclear(p) 260 */ 261 /* TODO -- see below */ 262 263 /* 264 * atomic_set(p, v) 265 * { *p |= v; } 266 */ 267 268 #define __atomic_set_int(p, v, t) \ 269 __asm __volatile( \ 270 "1: lwarx %0, 0, %2\n" \ 271 " or %0, %3, %0\n" \ 272 " stwcx. %0, 0, %2\n" \ 273 " bne- 1b\n" \ 274 : "=&r" (t), "=m" (*p) \ 275 : "r" (p), "r" (v), "m" (*p) \ 276 : "cr0", "memory") \ 277 /* __atomic_set_int */ 278 279 #ifdef __powerpc64__ 280 #define __atomic_set_long(p, v, t) \ 281 __asm __volatile( \ 282 "1: ldarx %0, 0, %2\n" \ 283 " or %0, %3, %0\n" \ 284 " stdcx. %0, 0, %2\n" \ 285 " bne- 1b\n" \ 286 : "=&r" (t), "=m" (*p) \ 287 : "r" (p), "r" (v), "m" (*p) \ 288 : "cr0", "memory") \ 289 /* __atomic_set_long */ 290 #else 291 #define __atomic_set_long(p, v, t) \ 292 __asm __volatile( \ 293 "1: lwarx %0, 0, %2\n" \ 294 " or %0, %3, %0\n" \ 295 " stwcx. %0, 0, %2\n" \ 296 " bne- 1b\n" \ 297 : "=&r" (t), "=m" (*p) \ 298 : "r" (p), "r" (v), "m" (*p) \ 299 : "cr0", "memory") \ 300 /* __atomic_set_long */ 301 #endif 302 303 #define _ATOMIC_SET(type) \ 304 static __inline void \ 305 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 306 u_##type t; \ 307 __atomic_set_##type(p, v, t); \ 308 } \ 309 \ 310 static __inline void \ 311 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 312 u_##type t; \ 313 __atomic_set_##type(p, v, t); \ 314 __ATOMIC_ACQ(); \ 315 } \ 316 \ 317 static __inline void \ 318 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 319 u_##type t; \ 320 __ATOMIC_REL(); \ 321 __atomic_set_##type(p, v, t); \ 322 } \ 323 /* _ATOMIC_SET */ 324 325 _ATOMIC_SET(int) 326 _ATOMIC_SET(long) 327 328 #define atomic_set_32 atomic_set_int 329 #define atomic_set_acq_32 atomic_set_acq_int 330 #define atomic_set_rel_32 atomic_set_rel_int 331 332 #ifdef __powerpc64__ 333 #define atomic_set_64 atomic_set_long 334 #define atomic_set_acq_64 atomic_set_acq_long 335 #define atomic_set_rel_64 atomic_set_rel_long 336 337 #define atomic_set_ptr atomic_set_long 338 #define atomic_set_acq_ptr atomic_set_acq_long 339 #define atomic_set_rel_ptr atomic_set_rel_long 340 #else 341 #define atomic_set_ptr atomic_set_int 342 #define atomic_set_acq_ptr atomic_set_acq_int 343 #define atomic_set_rel_ptr atomic_set_rel_int 344 #endif 345 #undef _ATOMIC_SET 346 #undef __atomic_set_long 347 #undef __atomic_set_int 348 349 /* 350 * atomic_subtract(p, v) 351 * { *p -= v; } 352 */ 353 354 #define __atomic_subtract_int(p, v, t) \ 355 __asm __volatile( \ 356 "1: lwarx %0, 0, %2\n" \ 357 " subf %0, %3, %0\n" \ 358 " stwcx. %0, 0, %2\n" \ 359 " bne- 1b\n" \ 360 : "=&r" (t), "=m" (*p) \ 361 : "r" (p), "r" (v), "m" (*p) \ 362 : "cr0", "memory") \ 363 /* __atomic_subtract_int */ 364 365 #ifdef __powerpc64__ 366 #define __atomic_subtract_long(p, v, t) \ 367 __asm __volatile( \ 368 "1: ldarx %0, 0, %2\n" \ 369 " subf %0, %3, %0\n" \ 370 " stdcx. %0, 0, %2\n" \ 371 " bne- 1b\n" \ 372 : "=&r" (t), "=m" (*p) \ 373 : "r" (p), "r" (v), "m" (*p) \ 374 : "cr0", "memory") \ 375 /* __atomic_subtract_long */ 376 #else 377 #define __atomic_subtract_long(p, v, t) \ 378 __asm __volatile( \ 379 "1: lwarx %0, 0, %2\n" \ 380 " subf %0, %3, %0\n" \ 381 " stwcx. %0, 0, %2\n" \ 382 " bne- 1b\n" \ 383 : "=&r" (t), "=m" (*p) \ 384 : "r" (p), "r" (v), "m" (*p) \ 385 : "cr0", "memory") \ 386 /* __atomic_subtract_long */ 387 #endif 388 389 #define _ATOMIC_SUBTRACT(type) \ 390 static __inline void \ 391 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 392 u_##type t; \ 393 __atomic_subtract_##type(p, v, t); \ 394 } \ 395 \ 396 static __inline void \ 397 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 398 u_##type t; \ 399 __atomic_subtract_##type(p, v, t); \ 400 __ATOMIC_ACQ(); \ 401 } \ 402 \ 403 static __inline void \ 404 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 405 u_##type t; \ 406 __ATOMIC_REL(); \ 407 __atomic_subtract_##type(p, v, t); \ 408 } \ 409 /* _ATOMIC_SUBTRACT */ 410 411 _ATOMIC_SUBTRACT(int) 412 _ATOMIC_SUBTRACT(long) 413 414 #define atomic_subtract_32 atomic_subtract_int 415 #define atomic_subtract_acq_32 atomic_subtract_acq_int 416 #define atomic_subtract_rel_32 atomic_subtract_rel_int 417 418 #ifdef __powerpc64__ 419 #define atomic_subtract_64 atomic_subtract_long 420 #define atomic_subtract_acq_64 atomic_subract_acq_long 421 #define atomic_subtract_rel_64 atomic_subtract_rel_long 422 423 #define atomic_subtract_ptr atomic_subtract_long 424 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 425 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 426 #else 427 #define atomic_subtract_ptr atomic_subtract_int 428 #define atomic_subtract_acq_ptr atomic_subtract_acq_int 429 #define atomic_subtract_rel_ptr atomic_subtract_rel_int 430 #endif 431 #undef _ATOMIC_SUBTRACT 432 #undef __atomic_subtract_long 433 #undef __atomic_subtract_int 434 435 /* 436 * atomic_store_rel(p, v) 437 */ 438 /* TODO -- see below */ 439 440 /* 441 * Old/original implementations that still need revisiting. 442 */ 443 444 static __inline u_int 445 atomic_readandclear_int(volatile u_int *addr) 446 { 447 u_int result,temp; 448 449 __asm __volatile ( 450 "\tsync\n" /* drain writes */ 451 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 452 "li %1, 0\n\t" /* load new value */ 453 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 454 "bne- 1b\n\t" /* spin if failed */ 455 : "=&r"(result), "=&r"(temp), "=m" (*addr) 456 : "r" (addr), "m" (*addr) 457 : "cr0", "memory"); 458 459 return (result); 460 } 461 462 #ifdef __powerpc64__ 463 static __inline u_long 464 atomic_readandclear_long(volatile u_long *addr) 465 { 466 u_long result,temp; 467 468 __asm __volatile ( 469 "\tsync\n" /* drain writes */ 470 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 471 "li %1, 0\n\t" /* load new value */ 472 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 473 "bne- 1b\n\t" /* spin if failed */ 474 : "=&r"(result), "=&r"(temp), "=m" (*addr) 475 : "r" (addr), "m" (*addr) 476 : "cr0", "memory"); 477 478 return (result); 479 } 480 #endif 481 482 #define atomic_readandclear_32 atomic_readandclear_int 483 484 #ifdef __powerpc64__ 485 #define atomic_readandclear_64 atomic_readandclear_long 486 487 #define atomic_readandclear_ptr atomic_readandclear_long 488 #else 489 static __inline u_long 490 atomic_readandclear_long(volatile u_long *addr) 491 { 492 493 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 494 } 495 496 #define atomic_readandclear_ptr atomic_readandclear_int 497 #endif 498 499 /* 500 * We assume that a = b will do atomic loads and stores. 501 */ 502 #define ATOMIC_STORE_LOAD(TYPE) \ 503 static __inline u_##TYPE \ 504 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 505 { \ 506 u_##TYPE v; \ 507 \ 508 v = *p; \ 509 mb(); \ 510 return (v); \ 511 } \ 512 \ 513 static __inline void \ 514 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 515 { \ 516 \ 517 powerpc_lwsync(); \ 518 *p = v; \ 519 } 520 521 ATOMIC_STORE_LOAD(int) 522 523 #define atomic_load_acq_32 atomic_load_acq_int 524 #define atomic_store_rel_32 atomic_store_rel_int 525 526 #ifdef __powerpc64__ 527 ATOMIC_STORE_LOAD(long) 528 529 #define atomic_load_acq_64 atomic_load_acq_long 530 #define atomic_store_rel_64 atomic_store_rel_long 531 532 #define atomic_load_acq_ptr atomic_load_acq_long 533 #define atomic_store_rel_ptr atomic_store_rel_long 534 #else 535 static __inline u_long 536 atomic_load_acq_long(volatile u_long *addr) 537 { 538 539 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 540 } 541 542 static __inline void 543 atomic_store_rel_long(volatile u_long *addr, u_long val) 544 { 545 546 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 547 } 548 549 #define atomic_load_acq_ptr atomic_load_acq_int 550 #define atomic_store_rel_ptr atomic_store_rel_int 551 #endif 552 #undef ATOMIC_STORE_LOAD 553 554 /* 555 * Atomically compare the value stored at *p with cmpval and if the 556 * two values are equal, update the value of *p with newval. Returns 557 * zero if the compare failed, nonzero otherwise. 558 */ 559 static __inline int 560 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 561 { 562 int ret; 563 564 __asm __volatile ( 565 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 566 "cmplw %3, %0\n\t" /* compare */ 567 "bne 2f\n\t" /* exit if not equal */ 568 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 569 "bne- 1b\n\t" /* spin if failed */ 570 "li %0, 1\n\t" /* success - retval = 1 */ 571 "b 3f\n\t" /* we've succeeded */ 572 "2:\n\t" 573 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 574 "li %0, 0\n\t" /* failure - retval = 0 */ 575 "3:\n\t" 576 : "=&r" (ret), "=m" (*p) 577 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 578 : "cr0", "memory"); 579 580 return (ret); 581 } 582 static __inline int 583 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 584 { 585 int ret; 586 587 __asm __volatile ( 588 #ifdef __powerpc64__ 589 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 590 "cmpld %3, %0\n\t" /* compare */ 591 "bne 2f\n\t" /* exit if not equal */ 592 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 593 #else 594 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 595 "cmplw %3, %0\n\t" /* compare */ 596 "bne 2f\n\t" /* exit if not equal */ 597 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 598 #endif 599 "bne- 1b\n\t" /* spin if failed */ 600 "li %0, 1\n\t" /* success - retval = 1 */ 601 "b 3f\n\t" /* we've succeeded */ 602 "2:\n\t" 603 #ifdef __powerpc64__ 604 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 605 #else 606 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 607 #endif 608 "li %0, 0\n\t" /* failure - retval = 0 */ 609 "3:\n\t" 610 : "=&r" (ret), "=m" (*p) 611 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 612 : "cr0", "memory"); 613 614 return (ret); 615 } 616 617 static __inline int 618 atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) 619 { 620 int retval; 621 622 retval = atomic_cmpset_int(p, cmpval, newval); 623 __ATOMIC_ACQ(); 624 return (retval); 625 } 626 627 static __inline int 628 atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) 629 { 630 __ATOMIC_REL(); 631 return (atomic_cmpset_int(p, cmpval, newval)); 632 } 633 634 static __inline int 635 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) 636 { 637 u_long retval; 638 639 retval = atomic_cmpset_long(p, cmpval, newval); 640 __ATOMIC_ACQ(); 641 return (retval); 642 } 643 644 static __inline int 645 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) 646 { 647 __ATOMIC_REL(); 648 return (atomic_cmpset_long(p, cmpval, newval)); 649 } 650 651 #define atomic_cmpset_32 atomic_cmpset_int 652 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 653 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 654 655 #ifdef __powerpc64__ 656 #define atomic_cmpset_64 atomic_cmpset_long 657 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 658 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 659 660 #define atomic_cmpset_ptr atomic_cmpset_long 661 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 662 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 663 #else 664 #define atomic_cmpset_ptr atomic_cmpset_int 665 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 666 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 667 #endif 668 669 /* 670 * Atomically compare the value stored at *p with *cmpval and if the 671 * two values are equal, update the value of *p with newval. Returns 672 * zero if the compare failed and sets *cmpval to the read value from *p, 673 * nonzero otherwise. 674 */ 675 static __inline int 676 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) 677 { 678 int ret; 679 680 __asm __volatile ( 681 "lwarx %0, 0, %3\n\t" /* load old value */ 682 "cmplw %4, %0\n\t" /* compare */ 683 "bne 1f\n\t" /* exit if not equal */ 684 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 685 "bne- 1f\n\t" /* exit if failed */ 686 "li %0, 1\n\t" /* success - retval = 1 */ 687 "b 2f\n\t" /* we've succeeded */ 688 "1:\n\t" 689 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 690 "stwx %0, 0, %7\n\t" 691 "li %0, 0\n\t" /* failure - retval = 0 */ 692 "2:\n\t" 693 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 694 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 695 : "cr0", "memory"); 696 697 return (ret); 698 } 699 static __inline int 700 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) 701 { 702 int ret; 703 704 __asm __volatile ( 705 #ifdef __powerpc64__ 706 "ldarx %0, 0, %3\n\t" /* load old value */ 707 "cmpld %4, %0\n\t" /* compare */ 708 "bne 1f\n\t" /* exit if not equal */ 709 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 710 #else 711 "lwarx %0, 0, %3\n\t" /* load old value */ 712 "cmplw %4, %0\n\t" /* compare */ 713 "bne 1f\n\t" /* exit if not equal */ 714 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 715 #endif 716 "bne- 1f\n\t" /* exit if failed */ 717 "li %0, 1\n\t" /* success - retval = 1 */ 718 "b 2f\n\t" /* we've succeeded */ 719 "1:\n\t" 720 #ifdef __powerpc64__ 721 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 722 "stdx %0, 0, %7\n\t" 723 #else 724 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 725 "stwx %0, 0, %7\n\t" 726 #endif 727 "li %0, 0\n\t" /* failure - retval = 0 */ 728 "2:\n\t" 729 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 730 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 731 : "cr0", "memory"); 732 733 return (ret); 734 } 735 736 static __inline int 737 atomic_fcmpset_acq_int(volatile u_int *p, u_int *cmpval, u_int newval) 738 { 739 int retval; 740 741 retval = atomic_fcmpset_int(p, cmpval, newval); 742 __ATOMIC_ACQ(); 743 return (retval); 744 } 745 746 static __inline int 747 atomic_fcmpset_rel_int(volatile u_int *p, u_int *cmpval, u_int newval) 748 { 749 __ATOMIC_REL(); 750 return (atomic_fcmpset_int(p, cmpval, newval)); 751 } 752 753 static __inline int 754 atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval) 755 { 756 u_long retval; 757 758 retval = atomic_fcmpset_long(p, cmpval, newval); 759 __ATOMIC_ACQ(); 760 return (retval); 761 } 762 763 static __inline int 764 atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval) 765 { 766 __ATOMIC_REL(); 767 return (atomic_fcmpset_long(p, cmpval, newval)); 768 } 769 770 #define atomic_fcmpset_32 atomic_fcmpset_int 771 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 772 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 773 774 #ifdef __powerpc64__ 775 #define atomic_fcmpset_64 atomic_fcmpset_long 776 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long 777 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long 778 779 #define atomic_fcmpset_ptr atomic_fcmpset_long 780 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 781 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 782 #else 783 #define atomic_fcmpset_ptr atomic_fcmpset_int 784 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int 785 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int 786 #endif 787 788 static __inline u_int 789 atomic_fetchadd_int(volatile u_int *p, u_int v) 790 { 791 u_int value; 792 793 do { 794 value = *p; 795 } while (!atomic_cmpset_int(p, value, value + v)); 796 return (value); 797 } 798 799 static __inline u_long 800 atomic_fetchadd_long(volatile u_long *p, u_long v) 801 { 802 u_long value; 803 804 do { 805 value = *p; 806 } while (!atomic_cmpset_long(p, value, value + v)); 807 return (value); 808 } 809 810 static __inline u_int 811 atomic_swap_32(volatile u_int *p, u_int v) 812 { 813 u_int prev; 814 815 __asm __volatile( 816 "1: lwarx %0,0,%2\n" 817 " stwcx. %3,0,%2\n" 818 " bne- 1b\n" 819 : "=&r" (prev), "+m" (*(volatile u_int *)p) 820 : "r" (p), "r" (v) 821 : "cr0", "memory"); 822 823 return (prev); 824 } 825 826 #ifdef __powerpc64__ 827 static __inline u_long 828 atomic_swap_64(volatile u_long *p, u_long v) 829 { 830 u_long prev; 831 832 __asm __volatile( 833 "1: ldarx %0,0,%2\n" 834 " stdcx. %3,0,%2\n" 835 " bne- 1b\n" 836 : "=&r" (prev), "+m" (*(volatile u_long *)p) 837 : "r" (p), "r" (v) 838 : "cr0", "memory"); 839 840 return (prev); 841 } 842 #endif 843 844 #define atomic_fetchadd_32 atomic_fetchadd_int 845 #define atomic_swap_int atomic_swap_32 846 847 #ifdef __powerpc64__ 848 #define atomic_fetchadd_64 atomic_fetchadd_long 849 #define atomic_swap_long atomic_swap_64 850 #define atomic_swap_ptr atomic_swap_64 851 #endif 852 853 #undef __ATOMIC_REL 854 #undef __ATOMIC_ACQ 855 856 static __inline void 857 atomic_thread_fence_acq(void) 858 { 859 860 powerpc_lwsync(); 861 } 862 863 static __inline void 864 atomic_thread_fence_rel(void) 865 { 866 867 powerpc_lwsync(); 868 } 869 870 static __inline void 871 atomic_thread_fence_acq_rel(void) 872 { 873 874 powerpc_lwsync(); 875 } 876 877 static __inline void 878 atomic_thread_fence_seq_cst(void) 879 { 880 881 __asm __volatile("sync" : : : "memory"); 882 } 883 884 #endif /* ! _MACHINE_ATOMIC_H_ */ 885