1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2008 Marcel Moolenaar 5 * Copyright (c) 2001 Benno Rice 6 * Copyright (c) 2001 David E. O'Brien 7 * Copyright (c) 1998 Doug Rabson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 #ifndef _MACHINE_ATOMIC_H_ 35 #define _MACHINE_ATOMIC_H_ 36 37 #ifndef _SYS_CDEFS_H_ 38 #error this file needs sys/cdefs.h as a prerequisite 39 #endif 40 41 #include <sys/atomic_common.h> 42 43 /* 44 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 45 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 46 * of this file. See also Appendix B.2 of Book II of the architecture manual. 47 * 48 * Note that not all Book-E processors accept the light-weight sync variant. 49 * In particular, early models of E500 cores are known to wedge. Bank on all 50 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 51 * to use the heavier-weight sync. 52 */ 53 54 #ifdef __powerpc64__ 55 #define mb() __asm __volatile("sync" : : : "memory") 56 #define rmb() __asm __volatile("lwsync" : : : "memory") 57 #define wmb() __asm __volatile("lwsync" : : : "memory") 58 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 59 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 60 #else 61 #define mb() __asm __volatile("sync" : : : "memory") 62 #define rmb() __asm __volatile("sync" : : : "memory") 63 #define wmb() __asm __volatile("sync" : : : "memory") 64 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 65 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 66 #endif 67 68 static __inline void 69 powerpc_lwsync(void) 70 { 71 72 #ifdef __powerpc64__ 73 __asm __volatile("lwsync" : : : "memory"); 74 #else 75 __asm __volatile("sync" : : : "memory"); 76 #endif 77 } 78 79 /* 80 * atomic_add(p, v) 81 * { *p += v; } 82 */ 83 84 #define __atomic_add_int(p, v, t) \ 85 __asm __volatile( \ 86 "1: lwarx %0, 0, %2\n" \ 87 " add %0, %3, %0\n" \ 88 " stwcx. %0, 0, %2\n" \ 89 " bne- 1b\n" \ 90 : "=&r" (t), "=m" (*p) \ 91 : "r" (p), "r" (v), "m" (*p) \ 92 : "cr0", "memory") \ 93 /* __atomic_add_int */ 94 95 #ifdef __powerpc64__ 96 #define __atomic_add_long(p, v, t) \ 97 __asm __volatile( \ 98 "1: ldarx %0, 0, %2\n" \ 99 " add %0, %3, %0\n" \ 100 " stdcx. %0, 0, %2\n" \ 101 " bne- 1b\n" \ 102 : "=&r" (t), "=m" (*p) \ 103 : "r" (p), "r" (v), "m" (*p) \ 104 : "cr0", "memory") \ 105 /* __atomic_add_long */ 106 #else 107 #define __atomic_add_long(p, v, t) \ 108 __asm __volatile( \ 109 "1: lwarx %0, 0, %2\n" \ 110 " add %0, %3, %0\n" \ 111 " stwcx. %0, 0, %2\n" \ 112 " bne- 1b\n" \ 113 : "=&r" (t), "=m" (*p) \ 114 : "r" (p), "r" (v), "m" (*p) \ 115 : "cr0", "memory") \ 116 /* __atomic_add_long */ 117 #endif 118 119 #define _ATOMIC_ADD(type) \ 120 static __inline void \ 121 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 122 u_##type t; \ 123 __atomic_add_##type(p, v, t); \ 124 } \ 125 \ 126 static __inline void \ 127 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 128 u_##type t; \ 129 __atomic_add_##type(p, v, t); \ 130 __ATOMIC_ACQ(); \ 131 } \ 132 \ 133 static __inline void \ 134 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 135 u_##type t; \ 136 __ATOMIC_REL(); \ 137 __atomic_add_##type(p, v, t); \ 138 } \ 139 /* _ATOMIC_ADD */ 140 141 _ATOMIC_ADD(int) 142 _ATOMIC_ADD(long) 143 144 #define atomic_add_32 atomic_add_int 145 #define atomic_add_acq_32 atomic_add_acq_int 146 #define atomic_add_rel_32 atomic_add_rel_int 147 148 #ifdef __powerpc64__ 149 #define atomic_add_64 atomic_add_long 150 #define atomic_add_acq_64 atomic_add_acq_long 151 #define atomic_add_rel_64 atomic_add_rel_long 152 153 #define atomic_add_ptr atomic_add_long 154 #define atomic_add_acq_ptr atomic_add_acq_long 155 #define atomic_add_rel_ptr atomic_add_rel_long 156 #else 157 #define atomic_add_ptr atomic_add_int 158 #define atomic_add_acq_ptr atomic_add_acq_int 159 #define atomic_add_rel_ptr atomic_add_rel_int 160 #endif 161 #undef _ATOMIC_ADD 162 #undef __atomic_add_long 163 #undef __atomic_add_int 164 165 /* 166 * atomic_clear(p, v) 167 * { *p &= ~v; } 168 */ 169 170 #define __atomic_clear_int(p, v, t) \ 171 __asm __volatile( \ 172 "1: lwarx %0, 0, %2\n" \ 173 " andc %0, %0, %3\n" \ 174 " stwcx. %0, 0, %2\n" \ 175 " bne- 1b\n" \ 176 : "=&r" (t), "=m" (*p) \ 177 : "r" (p), "r" (v), "m" (*p) \ 178 : "cr0", "memory") \ 179 /* __atomic_clear_int */ 180 181 #ifdef __powerpc64__ 182 #define __atomic_clear_long(p, v, t) \ 183 __asm __volatile( \ 184 "1: ldarx %0, 0, %2\n" \ 185 " andc %0, %0, %3\n" \ 186 " stdcx. %0, 0, %2\n" \ 187 " bne- 1b\n" \ 188 : "=&r" (t), "=m" (*p) \ 189 : "r" (p), "r" (v), "m" (*p) \ 190 : "cr0", "memory") \ 191 /* __atomic_clear_long */ 192 #else 193 #define __atomic_clear_long(p, v, t) \ 194 __asm __volatile( \ 195 "1: lwarx %0, 0, %2\n" \ 196 " andc %0, %0, %3\n" \ 197 " stwcx. %0, 0, %2\n" \ 198 " bne- 1b\n" \ 199 : "=&r" (t), "=m" (*p) \ 200 : "r" (p), "r" (v), "m" (*p) \ 201 : "cr0", "memory") \ 202 /* __atomic_clear_long */ 203 #endif 204 205 #define _ATOMIC_CLEAR(type) \ 206 static __inline void \ 207 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 208 u_##type t; \ 209 __atomic_clear_##type(p, v, t); \ 210 } \ 211 \ 212 static __inline void \ 213 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 214 u_##type t; \ 215 __atomic_clear_##type(p, v, t); \ 216 __ATOMIC_ACQ(); \ 217 } \ 218 \ 219 static __inline void \ 220 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 221 u_##type t; \ 222 __ATOMIC_REL(); \ 223 __atomic_clear_##type(p, v, t); \ 224 } \ 225 /* _ATOMIC_CLEAR */ 226 227 228 _ATOMIC_CLEAR(int) 229 _ATOMIC_CLEAR(long) 230 231 #define atomic_clear_32 atomic_clear_int 232 #define atomic_clear_acq_32 atomic_clear_acq_int 233 #define atomic_clear_rel_32 atomic_clear_rel_int 234 235 #ifdef __powerpc64__ 236 #define atomic_clear_64 atomic_clear_long 237 #define atomic_clear_acq_64 atomic_clear_acq_long 238 #define atomic_clear_rel_64 atomic_clear_rel_long 239 240 #define atomic_clear_ptr atomic_clear_long 241 #define atomic_clear_acq_ptr atomic_clear_acq_long 242 #define atomic_clear_rel_ptr atomic_clear_rel_long 243 #else 244 #define atomic_clear_ptr atomic_clear_int 245 #define atomic_clear_acq_ptr atomic_clear_acq_int 246 #define atomic_clear_rel_ptr atomic_clear_rel_int 247 #endif 248 #undef _ATOMIC_CLEAR 249 #undef __atomic_clear_long 250 #undef __atomic_clear_int 251 252 /* 253 * atomic_cmpset(p, o, n) 254 */ 255 /* TODO -- see below */ 256 257 /* 258 * atomic_load_acq(p) 259 */ 260 /* TODO -- see below */ 261 262 /* 263 * atomic_readandclear(p) 264 */ 265 /* TODO -- see below */ 266 267 /* 268 * atomic_set(p, v) 269 * { *p |= v; } 270 */ 271 272 #define __atomic_set_int(p, v, t) \ 273 __asm __volatile( \ 274 "1: lwarx %0, 0, %2\n" \ 275 " or %0, %3, %0\n" \ 276 " stwcx. %0, 0, %2\n" \ 277 " bne- 1b\n" \ 278 : "=&r" (t), "=m" (*p) \ 279 : "r" (p), "r" (v), "m" (*p) \ 280 : "cr0", "memory") \ 281 /* __atomic_set_int */ 282 283 #ifdef __powerpc64__ 284 #define __atomic_set_long(p, v, t) \ 285 __asm __volatile( \ 286 "1: ldarx %0, 0, %2\n" \ 287 " or %0, %3, %0\n" \ 288 " stdcx. %0, 0, %2\n" \ 289 " bne- 1b\n" \ 290 : "=&r" (t), "=m" (*p) \ 291 : "r" (p), "r" (v), "m" (*p) \ 292 : "cr0", "memory") \ 293 /* __atomic_set_long */ 294 #else 295 #define __atomic_set_long(p, v, t) \ 296 __asm __volatile( \ 297 "1: lwarx %0, 0, %2\n" \ 298 " or %0, %3, %0\n" \ 299 " stwcx. %0, 0, %2\n" \ 300 " bne- 1b\n" \ 301 : "=&r" (t), "=m" (*p) \ 302 : "r" (p), "r" (v), "m" (*p) \ 303 : "cr0", "memory") \ 304 /* __atomic_set_long */ 305 #endif 306 307 #define _ATOMIC_SET(type) \ 308 static __inline void \ 309 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 310 u_##type t; \ 311 __atomic_set_##type(p, v, t); \ 312 } \ 313 \ 314 static __inline void \ 315 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 316 u_##type t; \ 317 __atomic_set_##type(p, v, t); \ 318 __ATOMIC_ACQ(); \ 319 } \ 320 \ 321 static __inline void \ 322 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 323 u_##type t; \ 324 __ATOMIC_REL(); \ 325 __atomic_set_##type(p, v, t); \ 326 } \ 327 /* _ATOMIC_SET */ 328 329 _ATOMIC_SET(int) 330 _ATOMIC_SET(long) 331 332 #define atomic_set_32 atomic_set_int 333 #define atomic_set_acq_32 atomic_set_acq_int 334 #define atomic_set_rel_32 atomic_set_rel_int 335 336 #ifdef __powerpc64__ 337 #define atomic_set_64 atomic_set_long 338 #define atomic_set_acq_64 atomic_set_acq_long 339 #define atomic_set_rel_64 atomic_set_rel_long 340 341 #define atomic_set_ptr atomic_set_long 342 #define atomic_set_acq_ptr atomic_set_acq_long 343 #define atomic_set_rel_ptr atomic_set_rel_long 344 #else 345 #define atomic_set_ptr atomic_set_int 346 #define atomic_set_acq_ptr atomic_set_acq_int 347 #define atomic_set_rel_ptr atomic_set_rel_int 348 #endif 349 #undef _ATOMIC_SET 350 #undef __atomic_set_long 351 #undef __atomic_set_int 352 353 /* 354 * atomic_subtract(p, v) 355 * { *p -= v; } 356 */ 357 358 #define __atomic_subtract_int(p, v, t) \ 359 __asm __volatile( \ 360 "1: lwarx %0, 0, %2\n" \ 361 " subf %0, %3, %0\n" \ 362 " stwcx. %0, 0, %2\n" \ 363 " bne- 1b\n" \ 364 : "=&r" (t), "=m" (*p) \ 365 : "r" (p), "r" (v), "m" (*p) \ 366 : "cr0", "memory") \ 367 /* __atomic_subtract_int */ 368 369 #ifdef __powerpc64__ 370 #define __atomic_subtract_long(p, v, t) \ 371 __asm __volatile( \ 372 "1: ldarx %0, 0, %2\n" \ 373 " subf %0, %3, %0\n" \ 374 " stdcx. %0, 0, %2\n" \ 375 " bne- 1b\n" \ 376 : "=&r" (t), "=m" (*p) \ 377 : "r" (p), "r" (v), "m" (*p) \ 378 : "cr0", "memory") \ 379 /* __atomic_subtract_long */ 380 #else 381 #define __atomic_subtract_long(p, v, t) \ 382 __asm __volatile( \ 383 "1: lwarx %0, 0, %2\n" \ 384 " subf %0, %3, %0\n" \ 385 " stwcx. %0, 0, %2\n" \ 386 " bne- 1b\n" \ 387 : "=&r" (t), "=m" (*p) \ 388 : "r" (p), "r" (v), "m" (*p) \ 389 : "cr0", "memory") \ 390 /* __atomic_subtract_long */ 391 #endif 392 393 #define _ATOMIC_SUBTRACT(type) \ 394 static __inline void \ 395 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 396 u_##type t; \ 397 __atomic_subtract_##type(p, v, t); \ 398 } \ 399 \ 400 static __inline void \ 401 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 402 u_##type t; \ 403 __atomic_subtract_##type(p, v, t); \ 404 __ATOMIC_ACQ(); \ 405 } \ 406 \ 407 static __inline void \ 408 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 409 u_##type t; \ 410 __ATOMIC_REL(); \ 411 __atomic_subtract_##type(p, v, t); \ 412 } \ 413 /* _ATOMIC_SUBTRACT */ 414 415 _ATOMIC_SUBTRACT(int) 416 _ATOMIC_SUBTRACT(long) 417 418 #define atomic_subtract_32 atomic_subtract_int 419 #define atomic_subtract_acq_32 atomic_subtract_acq_int 420 #define atomic_subtract_rel_32 atomic_subtract_rel_int 421 422 #ifdef __powerpc64__ 423 #define atomic_subtract_64 atomic_subtract_long 424 #define atomic_subtract_acq_64 atomic_subract_acq_long 425 #define atomic_subtract_rel_64 atomic_subtract_rel_long 426 427 #define atomic_subtract_ptr atomic_subtract_long 428 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 429 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 430 #else 431 #define atomic_subtract_ptr atomic_subtract_int 432 #define atomic_subtract_acq_ptr atomic_subtract_acq_int 433 #define atomic_subtract_rel_ptr atomic_subtract_rel_int 434 #endif 435 #undef _ATOMIC_SUBTRACT 436 #undef __atomic_subtract_long 437 #undef __atomic_subtract_int 438 439 /* 440 * atomic_store_rel(p, v) 441 */ 442 /* TODO -- see below */ 443 444 /* 445 * Old/original implementations that still need revisiting. 446 */ 447 448 static __inline u_int 449 atomic_readandclear_int(volatile u_int *addr) 450 { 451 u_int result,temp; 452 453 __asm __volatile ( 454 "\tsync\n" /* drain writes */ 455 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 456 "li %1, 0\n\t" /* load new value */ 457 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 458 "bne- 1b\n\t" /* spin if failed */ 459 : "=&r"(result), "=&r"(temp), "=m" (*addr) 460 : "r" (addr), "m" (*addr) 461 : "cr0", "memory"); 462 463 return (result); 464 } 465 466 #ifdef __powerpc64__ 467 static __inline u_long 468 atomic_readandclear_long(volatile u_long *addr) 469 { 470 u_long result,temp; 471 472 __asm __volatile ( 473 "\tsync\n" /* drain writes */ 474 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 475 "li %1, 0\n\t" /* load new value */ 476 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 477 "bne- 1b\n\t" /* spin if failed */ 478 : "=&r"(result), "=&r"(temp), "=m" (*addr) 479 : "r" (addr), "m" (*addr) 480 : "cr0", "memory"); 481 482 return (result); 483 } 484 #endif 485 486 #define atomic_readandclear_32 atomic_readandclear_int 487 488 #ifdef __powerpc64__ 489 #define atomic_readandclear_64 atomic_readandclear_long 490 491 #define atomic_readandclear_ptr atomic_readandclear_long 492 #else 493 static __inline u_long 494 atomic_readandclear_long(volatile u_long *addr) 495 { 496 497 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 498 } 499 500 #define atomic_readandclear_ptr atomic_readandclear_int 501 #endif 502 503 /* 504 * We assume that a = b will do atomic loads and stores. 505 */ 506 #define ATOMIC_STORE_LOAD(TYPE) \ 507 static __inline u_##TYPE \ 508 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 509 { \ 510 u_##TYPE v; \ 511 \ 512 v = *p; \ 513 powerpc_lwsync(); \ 514 return (v); \ 515 } \ 516 \ 517 static __inline void \ 518 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 519 { \ 520 \ 521 powerpc_lwsync(); \ 522 *p = v; \ 523 } 524 525 ATOMIC_STORE_LOAD(int) 526 527 #define atomic_load_acq_32 atomic_load_acq_int 528 #define atomic_store_rel_32 atomic_store_rel_int 529 530 #ifdef __powerpc64__ 531 ATOMIC_STORE_LOAD(long) 532 533 #define atomic_load_acq_64 atomic_load_acq_long 534 #define atomic_store_rel_64 atomic_store_rel_long 535 536 #define atomic_load_acq_ptr atomic_load_acq_long 537 #define atomic_store_rel_ptr atomic_store_rel_long 538 #else 539 static __inline u_long 540 atomic_load_acq_long(volatile u_long *addr) 541 { 542 543 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 544 } 545 546 static __inline void 547 atomic_store_rel_long(volatile u_long *addr, u_long val) 548 { 549 550 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 551 } 552 553 #define atomic_load_acq_ptr atomic_load_acq_int 554 #define atomic_store_rel_ptr atomic_store_rel_int 555 #endif 556 #undef ATOMIC_STORE_LOAD 557 558 /* 559 * Atomically compare the value stored at *p with cmpval and if the 560 * two values are equal, update the value of *p with newval. Returns 561 * zero if the compare failed, nonzero otherwise. 562 */ 563 #ifdef ISA_206_ATOMICS 564 static __inline int 565 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) 566 { 567 int ret; 568 569 __asm __volatile ( 570 "1:\tlbarx %0, 0, %2\n\t" /* load old value */ 571 "cmplw %3, %0\n\t" /* compare */ 572 "bne- 2f\n\t" /* exit if not equal */ 573 "stbcx. %4, 0, %2\n\t" /* attempt to store */ 574 "bne- 1b\n\t" /* spin if failed */ 575 "li %0, 1\n\t" /* success - retval = 1 */ 576 "b 3f\n\t" /* we've succeeded */ 577 "2:\n\t" 578 "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 579 "li %0, 0\n\t" /* failure - retval = 0 */ 580 "3:\n\t" 581 : "=&r" (ret), "=m" (*p) 582 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 583 : "cr0", "memory"); 584 585 return (ret); 586 } 587 588 static __inline int 589 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) 590 { 591 int ret; 592 593 __asm __volatile ( 594 "1:\tlharx %0, 0, %2\n\t" /* load old value */ 595 "cmplw %3, %0\n\t" /* compare */ 596 "bne- 2f\n\t" /* exit if not equal */ 597 "sthcx. %4, 0, %2\n\t" /* attempt to store */ 598 "bne- 1b\n\t" /* spin if failed */ 599 "li %0, 1\n\t" /* success - retval = 1 */ 600 "b 3f\n\t" /* we've succeeded */ 601 "2:\n\t" 602 "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 603 "li %0, 0\n\t" /* failure - retval = 0 */ 604 "3:\n\t" 605 : "=&r" (ret), "=m" (*p) 606 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 607 : "cr0", "memory"); 608 609 return (ret); 610 } 611 #else 612 static __inline int 613 atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval, 614 uint32_t mask) 615 { 616 int ret; 617 uint32_t tmp; 618 619 __asm __volatile ( 620 "1:\tlwarx %2, 0, %2\n\t" /* load old value */ 621 "and %0, %2, %7\n\t" 622 "cmplw %4, %0\n\t" /* compare */ 623 "bne- 2f\n\t" /* exit if not equal */ 624 "andc %2, %2, %7\n\t" 625 "or %2, %2, %5\n\t" 626 "stwcx. %2, 0, %3\n\t" /* attempt to store */ 627 "bne- 1b\n\t" /* spin if failed */ 628 "li %0, 1\n\t" /* success - retval = 1 */ 629 "b 3f\n\t" /* we've succeeded */ 630 "2:\n\t" 631 "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */ 632 "li %0, 0\n\t" /* failure - retval = 0 */ 633 "3:\n\t" 634 : "=&r" (ret), "=m" (*p), "+&r" (tmp) 635 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p), 636 "r" (mask) 637 : "cr0", "memory"); 638 639 return (ret); 640 } 641 642 #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m) 643 #endif 644 645 static __inline int 646 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 647 { 648 int ret; 649 650 __asm __volatile ( 651 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 652 "cmplw %3, %0\n\t" /* compare */ 653 "bne- 2f\n\t" /* exit if not equal */ 654 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 655 "bne- 1b\n\t" /* spin if failed */ 656 "li %0, 1\n\t" /* success - retval = 1 */ 657 "b 3f\n\t" /* we've succeeded */ 658 "2:\n\t" 659 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 660 "li %0, 0\n\t" /* failure - retval = 0 */ 661 "3:\n\t" 662 : "=&r" (ret), "=m" (*p) 663 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 664 : "cr0", "memory"); 665 666 return (ret); 667 } 668 static __inline int 669 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 670 { 671 int ret; 672 673 __asm __volatile ( 674 #ifdef __powerpc64__ 675 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 676 "cmpld %3, %0\n\t" /* compare */ 677 "bne- 2f\n\t" /* exit if not equal */ 678 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 679 #else 680 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 681 "cmplw %3, %0\n\t" /* compare */ 682 "bne- 2f\n\t" /* exit if not equal */ 683 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 684 #endif 685 "bne- 1b\n\t" /* spin if failed */ 686 "li %0, 1\n\t" /* success - retval = 1 */ 687 "b 3f\n\t" /* we've succeeded */ 688 "2:\n\t" 689 #ifdef __powerpc64__ 690 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 691 #else 692 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 693 #endif 694 "li %0, 0\n\t" /* failure - retval = 0 */ 695 "3:\n\t" 696 : "=&r" (ret), "=m" (*p) 697 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 698 : "cr0", "memory"); 699 700 return (ret); 701 } 702 703 #define ATOMIC_CMPSET_ACQ_REL(type) \ 704 static __inline int \ 705 atomic_cmpset_acq_##type(volatile u_##type *p, \ 706 u_##type cmpval, u_##type newval)\ 707 {\ 708 u_##type retval; \ 709 retval = atomic_cmpset_##type(p, cmpval, newval);\ 710 __ATOMIC_ACQ();\ 711 return (retval);\ 712 }\ 713 static __inline int \ 714 atomic_cmpset_rel_##type(volatile u_##type *p, \ 715 u_##type cmpval, u_##type newval)\ 716 {\ 717 __ATOMIC_REL();\ 718 return (atomic_cmpset_##type(p, cmpval, newval));\ 719 }\ 720 struct hack 721 722 ATOMIC_CMPSET_ACQ_REL(int); 723 ATOMIC_CMPSET_ACQ_REL(long); 724 725 726 #define atomic_cmpset_8 atomic_cmpset_char 727 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char 728 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char 729 730 #define atomic_cmpset_16 atomic_cmpset_short 731 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short 732 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short 733 734 #define atomic_cmpset_32 atomic_cmpset_int 735 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 736 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 737 738 #ifdef __powerpc64__ 739 #define atomic_cmpset_64 atomic_cmpset_long 740 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 741 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 742 743 #define atomic_cmpset_ptr atomic_cmpset_long 744 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 745 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 746 #else 747 #define atomic_cmpset_ptr atomic_cmpset_int 748 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 749 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 750 #endif 751 752 /* 753 * Atomically compare the value stored at *p with *cmpval and if the 754 * two values are equal, update the value of *p with newval. Returns 755 * zero if the compare failed and sets *cmpval to the read value from *p, 756 * nonzero otherwise. 757 */ 758 #ifdef ISA_206_ATOMICS 759 static __inline int 760 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) 761 { 762 int ret; 763 764 __asm __volatile ( 765 "lbarx %0, 0, %3\n\t" /* load old value */ 766 "cmplw %4, %0\n\t" /* compare */ 767 "bne- 1f\n\t" /* exit if not equal */ 768 "stbcx. %5, 0, %3\n\t" /* attempt to store */ 769 "bne- 1f\n\t" /* exit if failed */ 770 "li %0, 1\n\t" /* success - retval = 1 */ 771 "b 2f\n\t" /* we've succeeded */ 772 "1:\n\t" 773 "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 774 "stwx %0, 0, %7\n\t" 775 "li %0, 0\n\t" /* failure - retval = 0 */ 776 "2:\n\t" 777 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 778 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 779 : "cr0", "memory"); 780 781 return (ret); 782 } 783 784 static __inline int 785 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) 786 { 787 int ret; 788 789 __asm __volatile ( 790 "lharx %0, 0, %3\n\t" /* load old value */ 791 "cmplw %4, %0\n\t" /* compare */ 792 "bne- 1f\n\t" /* exit if not equal */ 793 "sthcx. %5, 0, %3\n\t" /* attempt to store */ 794 "bne- 1f\n\t" /* exit if failed */ 795 "li %0, 1\n\t" /* success - retval = 1 */ 796 "b 2f\n\t" /* we've succeeded */ 797 "1:\n\t" 798 "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 799 "stwx %0, 0, %7\n\t" 800 "li %0, 0\n\t" /* failure - retval = 0 */ 801 "2:\n\t" 802 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 803 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 804 : "cr0", "memory"); 805 806 return (ret); 807 } 808 #endif /* ISA_206_ATOMICS */ 809 810 static __inline int 811 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) 812 { 813 int ret; 814 815 __asm __volatile ( 816 "lwarx %0, 0, %3\n\t" /* load old value */ 817 "cmplw %4, %0\n\t" /* compare */ 818 "bne- 1f\n\t" /* exit if not equal */ 819 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 820 "bne- 1f\n\t" /* exit if failed */ 821 "li %0, 1\n\t" /* success - retval = 1 */ 822 "b 2f\n\t" /* we've succeeded */ 823 "1:\n\t" 824 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 825 "stwx %0, 0, %7\n\t" 826 "li %0, 0\n\t" /* failure - retval = 0 */ 827 "2:\n\t" 828 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 829 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 830 : "cr0", "memory"); 831 832 return (ret); 833 } 834 static __inline int 835 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) 836 { 837 int ret; 838 839 __asm __volatile ( 840 #ifdef __powerpc64__ 841 "ldarx %0, 0, %3\n\t" /* load old value */ 842 "cmpld %4, %0\n\t" /* compare */ 843 "bne- 1f\n\t" /* exit if not equal */ 844 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 845 #else 846 "lwarx %0, 0, %3\n\t" /* load old value */ 847 "cmplw %4, %0\n\t" /* compare */ 848 "bne- 1f\n\t" /* exit if not equal */ 849 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 850 #endif 851 "bne- 1f\n\t" /* exit if failed */ 852 "li %0, 1\n\t" /* success - retval = 1 */ 853 "b 2f\n\t" /* we've succeeded */ 854 "1:\n\t" 855 #ifdef __powerpc64__ 856 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 857 "stdx %0, 0, %7\n\t" 858 #else 859 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 860 "stwx %0, 0, %7\n\t" 861 #endif 862 "li %0, 0\n\t" /* failure - retval = 0 */ 863 "2:\n\t" 864 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 865 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 866 : "cr0", "memory"); 867 868 return (ret); 869 } 870 871 #define ATOMIC_FCMPSET_ACQ_REL(type) \ 872 static __inline int \ 873 atomic_fcmpset_acq_##type(volatile u_##type *p, \ 874 u_##type *cmpval, u_##type newval)\ 875 {\ 876 u_##type retval; \ 877 retval = atomic_fcmpset_##type(p, cmpval, newval);\ 878 __ATOMIC_ACQ();\ 879 return (retval);\ 880 }\ 881 static __inline int \ 882 atomic_fcmpset_rel_##type(volatile u_##type *p, \ 883 u_##type *cmpval, u_##type newval)\ 884 {\ 885 __ATOMIC_REL();\ 886 return (atomic_fcmpset_##type(p, cmpval, newval));\ 887 }\ 888 struct hack 889 890 ATOMIC_FCMPSET_ACQ_REL(int); 891 ATOMIC_FCMPSET_ACQ_REL(long); 892 893 #define atomic_fcmpset_8 atomic_fcmpset_char 894 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char 895 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char 896 897 #define atomic_fcmpset_16 atomic_fcmpset_short 898 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short 899 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short 900 901 #define atomic_fcmpset_32 atomic_fcmpset_int 902 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 903 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 904 905 #ifdef __powerpc64__ 906 #define atomic_fcmpset_64 atomic_fcmpset_long 907 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long 908 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long 909 910 #define atomic_fcmpset_ptr atomic_fcmpset_long 911 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 912 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 913 #else 914 #define atomic_fcmpset_ptr atomic_fcmpset_int 915 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int 916 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int 917 #endif 918 919 static __inline u_int 920 atomic_fetchadd_int(volatile u_int *p, u_int v) 921 { 922 u_int value; 923 924 do { 925 value = *p; 926 } while (!atomic_cmpset_int(p, value, value + v)); 927 return (value); 928 } 929 930 static __inline u_long 931 atomic_fetchadd_long(volatile u_long *p, u_long v) 932 { 933 u_long value; 934 935 do { 936 value = *p; 937 } while (!atomic_cmpset_long(p, value, value + v)); 938 return (value); 939 } 940 941 static __inline u_int 942 atomic_swap_32(volatile u_int *p, u_int v) 943 { 944 u_int prev; 945 946 __asm __volatile( 947 "1: lwarx %0,0,%2\n" 948 " stwcx. %3,0,%2\n" 949 " bne- 1b\n" 950 : "=&r" (prev), "+m" (*(volatile u_int *)p) 951 : "r" (p), "r" (v) 952 : "cr0", "memory"); 953 954 return (prev); 955 } 956 957 #ifdef __powerpc64__ 958 static __inline u_long 959 atomic_swap_64(volatile u_long *p, u_long v) 960 { 961 u_long prev; 962 963 __asm __volatile( 964 "1: ldarx %0,0,%2\n" 965 " stdcx. %3,0,%2\n" 966 " bne- 1b\n" 967 : "=&r" (prev), "+m" (*(volatile u_long *)p) 968 : "r" (p), "r" (v) 969 : "cr0", "memory"); 970 971 return (prev); 972 } 973 #endif 974 975 #define atomic_fetchadd_32 atomic_fetchadd_int 976 #define atomic_swap_int atomic_swap_32 977 978 #ifdef __powerpc64__ 979 #define atomic_fetchadd_64 atomic_fetchadd_long 980 #define atomic_swap_long atomic_swap_64 981 #define atomic_swap_ptr atomic_swap_64 982 #else 983 #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) 984 #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) 985 #endif 986 987 static __inline void 988 atomic_thread_fence_acq(void) 989 { 990 991 powerpc_lwsync(); 992 } 993 994 static __inline void 995 atomic_thread_fence_rel(void) 996 { 997 998 powerpc_lwsync(); 999 } 1000 1001 static __inline void 1002 atomic_thread_fence_acq_rel(void) 1003 { 1004 1005 powerpc_lwsync(); 1006 } 1007 1008 static __inline void 1009 atomic_thread_fence_seq_cst(void) 1010 { 1011 1012 __asm __volatile("sync" : : : "memory"); 1013 } 1014 1015 #ifndef ISA_206_ATOMICS 1016 #include <sys/_atomic_subword.h> 1017 #endif 1018 1019 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ 1020 ATOMIC_CMPSET_ACQ_REL(char); 1021 ATOMIC_CMPSET_ACQ_REL(short); 1022 1023 ATOMIC_FCMPSET_ACQ_REL(char); 1024 ATOMIC_FCMPSET_ACQ_REL(short); 1025 1026 #undef __ATOMIC_REL 1027 #undef __ATOMIC_ACQ 1028 1029 #endif /* ! _MACHINE_ATOMIC_H_ */ 1030