1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2008 Marcel Moolenaar 5 * Copyright (c) 2001 Benno Rice 6 * Copyright (c) 2001 David E. O'Brien 7 * Copyright (c) 1998 Doug Rabson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 #ifndef _MACHINE_ATOMIC_H_ 35 #define _MACHINE_ATOMIC_H_ 36 37 #ifndef _SYS_CDEFS_H_ 38 #error this file needs sys/cdefs.h as a prerequisite 39 #endif 40 41 #include <sys/atomic_common.h> 42 43 #ifndef __powerpc64__ 44 #include <sys/_atomic64e.h> 45 #endif 46 47 /* 48 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 49 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 50 * of this file. See also Appendix B.2 of Book II of the architecture manual. 51 * 52 * Note that not all Book-E processors accept the light-weight sync variant. 53 * In particular, early models of E500 cores are known to wedge. Bank on all 54 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 55 * to use the heavier-weight sync. 56 */ 57 58 #ifdef __powerpc64__ 59 #define mb() __asm __volatile("sync" : : : "memory") 60 #define rmb() __asm __volatile("lwsync" : : : "memory") 61 #define wmb() __asm __volatile("lwsync" : : : "memory") 62 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 63 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 64 #else 65 #define mb() __asm __volatile("sync" : : : "memory") 66 #define rmb() __asm __volatile("sync" : : : "memory") 67 #define wmb() __asm __volatile("sync" : : : "memory") 68 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 69 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 70 #endif 71 72 static __inline void 73 powerpc_lwsync(void) 74 { 75 76 #ifdef __powerpc64__ 77 __asm __volatile("lwsync" : : : "memory"); 78 #else 79 __asm __volatile("sync" : : : "memory"); 80 #endif 81 } 82 83 /* 84 * atomic_add(p, v) 85 * { *p += v; } 86 */ 87 88 #define __atomic_add_int(p, v, t) \ 89 __asm __volatile( \ 90 "1: lwarx %0, 0, %2\n" \ 91 " add %0, %3, %0\n" \ 92 " stwcx. %0, 0, %2\n" \ 93 " bne- 1b\n" \ 94 : "=&r" (t), "=m" (*p) \ 95 : "r" (p), "r" (v), "m" (*p) \ 96 : "cr0", "memory") \ 97 /* __atomic_add_int */ 98 99 #ifdef __powerpc64__ 100 #define __atomic_add_long(p, v, t) \ 101 __asm __volatile( \ 102 "1: ldarx %0, 0, %2\n" \ 103 " add %0, %3, %0\n" \ 104 " stdcx. %0, 0, %2\n" \ 105 " bne- 1b\n" \ 106 : "=&r" (t), "=m" (*p) \ 107 : "r" (p), "r" (v), "m" (*p) \ 108 : "cr0", "memory") \ 109 /* __atomic_add_long */ 110 #else 111 #define __atomic_add_long(p, v, t) \ 112 __asm __volatile( \ 113 "1: lwarx %0, 0, %2\n" \ 114 " add %0, %3, %0\n" \ 115 " stwcx. %0, 0, %2\n" \ 116 " bne- 1b\n" \ 117 : "=&r" (t), "=m" (*p) \ 118 : "r" (p), "r" (v), "m" (*p) \ 119 : "cr0", "memory") \ 120 /* __atomic_add_long */ 121 #endif 122 123 #define _ATOMIC_ADD(type) \ 124 static __inline void \ 125 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 126 u_##type t; \ 127 __atomic_add_##type(p, v, t); \ 128 } \ 129 \ 130 static __inline void \ 131 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 132 u_##type t; \ 133 __atomic_add_##type(p, v, t); \ 134 __ATOMIC_ACQ(); \ 135 } \ 136 \ 137 static __inline void \ 138 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 139 u_##type t; \ 140 __ATOMIC_REL(); \ 141 __atomic_add_##type(p, v, t); \ 142 } \ 143 /* _ATOMIC_ADD */ 144 145 _ATOMIC_ADD(int) 146 _ATOMIC_ADD(long) 147 148 #define atomic_add_32 atomic_add_int 149 #define atomic_add_acq_32 atomic_add_acq_int 150 #define atomic_add_rel_32 atomic_add_rel_int 151 152 #ifdef __powerpc64__ 153 #define atomic_add_64 atomic_add_long 154 #define atomic_add_acq_64 atomic_add_acq_long 155 #define atomic_add_rel_64 atomic_add_rel_long 156 157 #define atomic_add_ptr atomic_add_long 158 #define atomic_add_acq_ptr atomic_add_acq_long 159 #define atomic_add_rel_ptr atomic_add_rel_long 160 #else 161 #define atomic_add_ptr atomic_add_int 162 #define atomic_add_acq_ptr atomic_add_acq_int 163 #define atomic_add_rel_ptr atomic_add_rel_int 164 #endif 165 #undef _ATOMIC_ADD 166 #undef __atomic_add_long 167 #undef __atomic_add_int 168 169 /* 170 * atomic_clear(p, v) 171 * { *p &= ~v; } 172 */ 173 174 #define __atomic_clear_int(p, v, t) \ 175 __asm __volatile( \ 176 "1: lwarx %0, 0, %2\n" \ 177 " andc %0, %0, %3\n" \ 178 " stwcx. %0, 0, %2\n" \ 179 " bne- 1b\n" \ 180 : "=&r" (t), "=m" (*p) \ 181 : "r" (p), "r" (v), "m" (*p) \ 182 : "cr0", "memory") \ 183 /* __atomic_clear_int */ 184 185 #ifdef __powerpc64__ 186 #define __atomic_clear_long(p, v, t) \ 187 __asm __volatile( \ 188 "1: ldarx %0, 0, %2\n" \ 189 " andc %0, %0, %3\n" \ 190 " stdcx. %0, 0, %2\n" \ 191 " bne- 1b\n" \ 192 : "=&r" (t), "=m" (*p) \ 193 : "r" (p), "r" (v), "m" (*p) \ 194 : "cr0", "memory") \ 195 /* __atomic_clear_long */ 196 #else 197 #define __atomic_clear_long(p, v, t) \ 198 __asm __volatile( \ 199 "1: lwarx %0, 0, %2\n" \ 200 " andc %0, %0, %3\n" \ 201 " stwcx. %0, 0, %2\n" \ 202 " bne- 1b\n" \ 203 : "=&r" (t), "=m" (*p) \ 204 : "r" (p), "r" (v), "m" (*p) \ 205 : "cr0", "memory") \ 206 /* __atomic_clear_long */ 207 #endif 208 209 #define _ATOMIC_CLEAR(type) \ 210 static __inline void \ 211 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 212 u_##type t; \ 213 __atomic_clear_##type(p, v, t); \ 214 } \ 215 \ 216 static __inline void \ 217 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 218 u_##type t; \ 219 __atomic_clear_##type(p, v, t); \ 220 __ATOMIC_ACQ(); \ 221 } \ 222 \ 223 static __inline void \ 224 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 225 u_##type t; \ 226 __ATOMIC_REL(); \ 227 __atomic_clear_##type(p, v, t); \ 228 } \ 229 /* _ATOMIC_CLEAR */ 230 231 232 _ATOMIC_CLEAR(int) 233 _ATOMIC_CLEAR(long) 234 235 #define atomic_clear_32 atomic_clear_int 236 #define atomic_clear_acq_32 atomic_clear_acq_int 237 #define atomic_clear_rel_32 atomic_clear_rel_int 238 239 #ifdef __powerpc64__ 240 #define atomic_clear_64 atomic_clear_long 241 #define atomic_clear_acq_64 atomic_clear_acq_long 242 #define atomic_clear_rel_64 atomic_clear_rel_long 243 244 #define atomic_clear_ptr atomic_clear_long 245 #define atomic_clear_acq_ptr atomic_clear_acq_long 246 #define atomic_clear_rel_ptr atomic_clear_rel_long 247 #else 248 #define atomic_clear_ptr atomic_clear_int 249 #define atomic_clear_acq_ptr atomic_clear_acq_int 250 #define atomic_clear_rel_ptr atomic_clear_rel_int 251 #endif 252 #undef _ATOMIC_CLEAR 253 #undef __atomic_clear_long 254 #undef __atomic_clear_int 255 256 /* 257 * atomic_cmpset(p, o, n) 258 */ 259 /* TODO -- see below */ 260 261 /* 262 * atomic_load_acq(p) 263 */ 264 /* TODO -- see below */ 265 266 /* 267 * atomic_readandclear(p) 268 */ 269 /* TODO -- see below */ 270 271 /* 272 * atomic_set(p, v) 273 * { *p |= v; } 274 */ 275 276 #define __atomic_set_int(p, v, t) \ 277 __asm __volatile( \ 278 "1: lwarx %0, 0, %2\n" \ 279 " or %0, %3, %0\n" \ 280 " stwcx. %0, 0, %2\n" \ 281 " bne- 1b\n" \ 282 : "=&r" (t), "=m" (*p) \ 283 : "r" (p), "r" (v), "m" (*p) \ 284 : "cr0", "memory") \ 285 /* __atomic_set_int */ 286 287 #ifdef __powerpc64__ 288 #define __atomic_set_long(p, v, t) \ 289 __asm __volatile( \ 290 "1: ldarx %0, 0, %2\n" \ 291 " or %0, %3, %0\n" \ 292 " stdcx. %0, 0, %2\n" \ 293 " bne- 1b\n" \ 294 : "=&r" (t), "=m" (*p) \ 295 : "r" (p), "r" (v), "m" (*p) \ 296 : "cr0", "memory") \ 297 /* __atomic_set_long */ 298 #else 299 #define __atomic_set_long(p, v, t) \ 300 __asm __volatile( \ 301 "1: lwarx %0, 0, %2\n" \ 302 " or %0, %3, %0\n" \ 303 " stwcx. %0, 0, %2\n" \ 304 " bne- 1b\n" \ 305 : "=&r" (t), "=m" (*p) \ 306 : "r" (p), "r" (v), "m" (*p) \ 307 : "cr0", "memory") \ 308 /* __atomic_set_long */ 309 #endif 310 311 #define _ATOMIC_SET(type) \ 312 static __inline void \ 313 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 314 u_##type t; \ 315 __atomic_set_##type(p, v, t); \ 316 } \ 317 \ 318 static __inline void \ 319 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 320 u_##type t; \ 321 __atomic_set_##type(p, v, t); \ 322 __ATOMIC_ACQ(); \ 323 } \ 324 \ 325 static __inline void \ 326 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 327 u_##type t; \ 328 __ATOMIC_REL(); \ 329 __atomic_set_##type(p, v, t); \ 330 } \ 331 /* _ATOMIC_SET */ 332 333 _ATOMIC_SET(int) 334 _ATOMIC_SET(long) 335 336 #define atomic_set_32 atomic_set_int 337 #define atomic_set_acq_32 atomic_set_acq_int 338 #define atomic_set_rel_32 atomic_set_rel_int 339 340 #ifdef __powerpc64__ 341 #define atomic_set_64 atomic_set_long 342 #define atomic_set_acq_64 atomic_set_acq_long 343 #define atomic_set_rel_64 atomic_set_rel_long 344 345 #define atomic_set_ptr atomic_set_long 346 #define atomic_set_acq_ptr atomic_set_acq_long 347 #define atomic_set_rel_ptr atomic_set_rel_long 348 #else 349 #define atomic_set_ptr atomic_set_int 350 #define atomic_set_acq_ptr atomic_set_acq_int 351 #define atomic_set_rel_ptr atomic_set_rel_int 352 #endif 353 #undef _ATOMIC_SET 354 #undef __atomic_set_long 355 #undef __atomic_set_int 356 357 /* 358 * atomic_subtract(p, v) 359 * { *p -= v; } 360 */ 361 362 #define __atomic_subtract_int(p, v, t) \ 363 __asm __volatile( \ 364 "1: lwarx %0, 0, %2\n" \ 365 " subf %0, %3, %0\n" \ 366 " stwcx. %0, 0, %2\n" \ 367 " bne- 1b\n" \ 368 : "=&r" (t), "=m" (*p) \ 369 : "r" (p), "r" (v), "m" (*p) \ 370 : "cr0", "memory") \ 371 /* __atomic_subtract_int */ 372 373 #ifdef __powerpc64__ 374 #define __atomic_subtract_long(p, v, t) \ 375 __asm __volatile( \ 376 "1: ldarx %0, 0, %2\n" \ 377 " subf %0, %3, %0\n" \ 378 " stdcx. %0, 0, %2\n" \ 379 " bne- 1b\n" \ 380 : "=&r" (t), "=m" (*p) \ 381 : "r" (p), "r" (v), "m" (*p) \ 382 : "cr0", "memory") \ 383 /* __atomic_subtract_long */ 384 #else 385 #define __atomic_subtract_long(p, v, t) \ 386 __asm __volatile( \ 387 "1: lwarx %0, 0, %2\n" \ 388 " subf %0, %3, %0\n" \ 389 " stwcx. %0, 0, %2\n" \ 390 " bne- 1b\n" \ 391 : "=&r" (t), "=m" (*p) \ 392 : "r" (p), "r" (v), "m" (*p) \ 393 : "cr0", "memory") \ 394 /* __atomic_subtract_long */ 395 #endif 396 397 #define _ATOMIC_SUBTRACT(type) \ 398 static __inline void \ 399 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 400 u_##type t; \ 401 __atomic_subtract_##type(p, v, t); \ 402 } \ 403 \ 404 static __inline void \ 405 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 406 u_##type t; \ 407 __atomic_subtract_##type(p, v, t); \ 408 __ATOMIC_ACQ(); \ 409 } \ 410 \ 411 static __inline void \ 412 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 413 u_##type t; \ 414 __ATOMIC_REL(); \ 415 __atomic_subtract_##type(p, v, t); \ 416 } \ 417 /* _ATOMIC_SUBTRACT */ 418 419 _ATOMIC_SUBTRACT(int) 420 _ATOMIC_SUBTRACT(long) 421 422 #define atomic_subtract_32 atomic_subtract_int 423 #define atomic_subtract_acq_32 atomic_subtract_acq_int 424 #define atomic_subtract_rel_32 atomic_subtract_rel_int 425 426 #ifdef __powerpc64__ 427 #define atomic_subtract_64 atomic_subtract_long 428 #define atomic_subtract_acq_64 atomic_subract_acq_long 429 #define atomic_subtract_rel_64 atomic_subtract_rel_long 430 431 #define atomic_subtract_ptr atomic_subtract_long 432 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 433 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 434 #else 435 #define atomic_subtract_ptr atomic_subtract_int 436 #define atomic_subtract_acq_ptr atomic_subtract_acq_int 437 #define atomic_subtract_rel_ptr atomic_subtract_rel_int 438 #endif 439 #undef _ATOMIC_SUBTRACT 440 #undef __atomic_subtract_long 441 #undef __atomic_subtract_int 442 443 /* 444 * atomic_store_rel(p, v) 445 */ 446 /* TODO -- see below */ 447 448 /* 449 * Old/original implementations that still need revisiting. 450 */ 451 452 static __inline u_int 453 atomic_readandclear_int(volatile u_int *addr) 454 { 455 u_int result,temp; 456 457 __asm __volatile ( 458 "\tsync\n" /* drain writes */ 459 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 460 "li %1, 0\n\t" /* load new value */ 461 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 462 "bne- 1b\n\t" /* spin if failed */ 463 : "=&r"(result), "=&r"(temp), "=m" (*addr) 464 : "r" (addr), "m" (*addr) 465 : "cr0", "memory"); 466 467 return (result); 468 } 469 470 #ifdef __powerpc64__ 471 static __inline u_long 472 atomic_readandclear_long(volatile u_long *addr) 473 { 474 u_long result,temp; 475 476 __asm __volatile ( 477 "\tsync\n" /* drain writes */ 478 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 479 "li %1, 0\n\t" /* load new value */ 480 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 481 "bne- 1b\n\t" /* spin if failed */ 482 : "=&r"(result), "=&r"(temp), "=m" (*addr) 483 : "r" (addr), "m" (*addr) 484 : "cr0", "memory"); 485 486 return (result); 487 } 488 #endif 489 490 #define atomic_readandclear_32 atomic_readandclear_int 491 492 #ifdef __powerpc64__ 493 #define atomic_readandclear_64 atomic_readandclear_long 494 495 #define atomic_readandclear_ptr atomic_readandclear_long 496 #else 497 static __inline u_long 498 atomic_readandclear_long(volatile u_long *addr) 499 { 500 501 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 502 } 503 504 #define atomic_readandclear_ptr atomic_readandclear_int 505 #endif 506 507 /* 508 * We assume that a = b will do atomic loads and stores. 509 */ 510 #define ATOMIC_STORE_LOAD(TYPE) \ 511 static __inline u_##TYPE \ 512 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 513 { \ 514 u_##TYPE v; \ 515 \ 516 v = *p; \ 517 powerpc_lwsync(); \ 518 return (v); \ 519 } \ 520 \ 521 static __inline void \ 522 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 523 { \ 524 \ 525 powerpc_lwsync(); \ 526 *p = v; \ 527 } 528 529 ATOMIC_STORE_LOAD(int) 530 531 #define atomic_load_acq_32 atomic_load_acq_int 532 #define atomic_store_rel_32 atomic_store_rel_int 533 534 #ifdef __powerpc64__ 535 ATOMIC_STORE_LOAD(long) 536 537 #define atomic_load_acq_64 atomic_load_acq_long 538 #define atomic_store_rel_64 atomic_store_rel_long 539 540 #define atomic_load_acq_ptr atomic_load_acq_long 541 #define atomic_store_rel_ptr atomic_store_rel_long 542 #else 543 static __inline u_long 544 atomic_load_acq_long(volatile u_long *addr) 545 { 546 547 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 548 } 549 550 static __inline void 551 atomic_store_rel_long(volatile u_long *addr, u_long val) 552 { 553 554 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 555 } 556 557 #define atomic_load_acq_ptr atomic_load_acq_int 558 #define atomic_store_rel_ptr atomic_store_rel_int 559 #endif 560 #undef ATOMIC_STORE_LOAD 561 562 /* 563 * Atomically compare the value stored at *p with cmpval and if the 564 * two values are equal, update the value of *p with newval. Returns 565 * zero if the compare failed, nonzero otherwise. 566 */ 567 #ifdef ISA_206_ATOMICS 568 static __inline int 569 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) 570 { 571 int ret; 572 573 __asm __volatile ( 574 "1:\tlbarx %0, 0, %2\n\t" /* load old value */ 575 "cmplw %3, %0\n\t" /* compare */ 576 "bne- 2f\n\t" /* exit if not equal */ 577 "stbcx. %4, 0, %2\n\t" /* attempt to store */ 578 "bne- 1b\n\t" /* spin if failed */ 579 "li %0, 1\n\t" /* success - retval = 1 */ 580 "b 3f\n\t" /* we've succeeded */ 581 "2:\n\t" 582 "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 583 "li %0, 0\n\t" /* failure - retval = 0 */ 584 "3:\n\t" 585 : "=&r" (ret), "=m" (*p) 586 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 587 : "cr0", "memory"); 588 589 return (ret); 590 } 591 592 static __inline int 593 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) 594 { 595 int ret; 596 597 __asm __volatile ( 598 "1:\tlharx %0, 0, %2\n\t" /* load old value */ 599 "cmplw %3, %0\n\t" /* compare */ 600 "bne- 2f\n\t" /* exit if not equal */ 601 "sthcx. %4, 0, %2\n\t" /* attempt to store */ 602 "bne- 1b\n\t" /* spin if failed */ 603 "li %0, 1\n\t" /* success - retval = 1 */ 604 "b 3f\n\t" /* we've succeeded */ 605 "2:\n\t" 606 "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 607 "li %0, 0\n\t" /* failure - retval = 0 */ 608 "3:\n\t" 609 : "=&r" (ret), "=m" (*p) 610 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 611 : "cr0", "memory"); 612 613 return (ret); 614 } 615 #else 616 static __inline int 617 atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval, 618 uint32_t mask) 619 { 620 int ret; 621 uint32_t tmp; 622 623 __asm __volatile ( 624 "1:\tlwarx %2, 0, %2\n\t" /* load old value */ 625 "and %0, %2, %7\n\t" 626 "cmplw %4, %0\n\t" /* compare */ 627 "bne- 2f\n\t" /* exit if not equal */ 628 "andc %2, %2, %7\n\t" 629 "or %2, %2, %5\n\t" 630 "stwcx. %2, 0, %3\n\t" /* attempt to store */ 631 "bne- 1b\n\t" /* spin if failed */ 632 "li %0, 1\n\t" /* success - retval = 1 */ 633 "b 3f\n\t" /* we've succeeded */ 634 "2:\n\t" 635 "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */ 636 "li %0, 0\n\t" /* failure - retval = 0 */ 637 "3:\n\t" 638 : "=&r" (ret), "=m" (*p), "+&r" (tmp) 639 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p), 640 "r" (mask) 641 : "cr0", "memory"); 642 643 return (ret); 644 } 645 646 #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m) 647 #endif 648 649 static __inline int 650 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 651 { 652 int ret; 653 654 __asm __volatile ( 655 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 656 "cmplw %3, %0\n\t" /* compare */ 657 "bne- 2f\n\t" /* exit if not equal */ 658 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 659 "bne- 1b\n\t" /* spin if failed */ 660 "li %0, 1\n\t" /* success - retval = 1 */ 661 "b 3f\n\t" /* we've succeeded */ 662 "2:\n\t" 663 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 664 "li %0, 0\n\t" /* failure - retval = 0 */ 665 "3:\n\t" 666 : "=&r" (ret), "=m" (*p) 667 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 668 : "cr0", "memory"); 669 670 return (ret); 671 } 672 static __inline int 673 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 674 { 675 int ret; 676 677 __asm __volatile ( 678 #ifdef __powerpc64__ 679 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 680 "cmpld %3, %0\n\t" /* compare */ 681 "bne- 2f\n\t" /* exit if not equal */ 682 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 683 #else 684 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 685 "cmplw %3, %0\n\t" /* compare */ 686 "bne- 2f\n\t" /* exit if not equal */ 687 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 688 #endif 689 "bne- 1b\n\t" /* spin if failed */ 690 "li %0, 1\n\t" /* success - retval = 1 */ 691 "b 3f\n\t" /* we've succeeded */ 692 "2:\n\t" 693 #ifdef __powerpc64__ 694 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 695 #else 696 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 697 #endif 698 "li %0, 0\n\t" /* failure - retval = 0 */ 699 "3:\n\t" 700 : "=&r" (ret), "=m" (*p) 701 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 702 : "cr0", "memory"); 703 704 return (ret); 705 } 706 707 #define ATOMIC_CMPSET_ACQ_REL(type) \ 708 static __inline int \ 709 atomic_cmpset_acq_##type(volatile u_##type *p, \ 710 u_##type cmpval, u_##type newval)\ 711 {\ 712 u_##type retval; \ 713 retval = atomic_cmpset_##type(p, cmpval, newval);\ 714 __ATOMIC_ACQ();\ 715 return (retval);\ 716 }\ 717 static __inline int \ 718 atomic_cmpset_rel_##type(volatile u_##type *p, \ 719 u_##type cmpval, u_##type newval)\ 720 {\ 721 __ATOMIC_REL();\ 722 return (atomic_cmpset_##type(p, cmpval, newval));\ 723 }\ 724 struct hack 725 726 ATOMIC_CMPSET_ACQ_REL(int); 727 ATOMIC_CMPSET_ACQ_REL(long); 728 729 730 #define atomic_cmpset_8 atomic_cmpset_char 731 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char 732 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char 733 734 #define atomic_cmpset_16 atomic_cmpset_short 735 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short 736 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short 737 738 #define atomic_cmpset_32 atomic_cmpset_int 739 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 740 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 741 742 #ifdef __powerpc64__ 743 #define atomic_cmpset_64 atomic_cmpset_long 744 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 745 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 746 747 #define atomic_cmpset_ptr atomic_cmpset_long 748 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 749 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 750 #else 751 #define atomic_cmpset_ptr atomic_cmpset_int 752 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 753 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 754 #endif 755 756 /* 757 * Atomically compare the value stored at *p with *cmpval and if the 758 * two values are equal, update the value of *p with newval. Returns 759 * zero if the compare failed and sets *cmpval to the read value from *p, 760 * nonzero otherwise. 761 */ 762 #ifdef ISA_206_ATOMICS 763 static __inline int 764 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) 765 { 766 int ret; 767 768 __asm __volatile ( 769 "lbarx %0, 0, %3\n\t" /* load old value */ 770 "cmplw %4, %0\n\t" /* compare */ 771 "bne- 1f\n\t" /* exit if not equal */ 772 "stbcx. %5, 0, %3\n\t" /* attempt to store */ 773 "bne- 1f\n\t" /* exit if failed */ 774 "li %0, 1\n\t" /* success - retval = 1 */ 775 "b 2f\n\t" /* we've succeeded */ 776 "1:\n\t" 777 "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 778 "stwx %0, 0, %7\n\t" 779 "li %0, 0\n\t" /* failure - retval = 0 */ 780 "2:\n\t" 781 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 782 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 783 : "cr0", "memory"); 784 785 return (ret); 786 } 787 788 static __inline int 789 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) 790 { 791 int ret; 792 793 __asm __volatile ( 794 "lharx %0, 0, %3\n\t" /* load old value */ 795 "cmplw %4, %0\n\t" /* compare */ 796 "bne- 1f\n\t" /* exit if not equal */ 797 "sthcx. %5, 0, %3\n\t" /* attempt to store */ 798 "bne- 1f\n\t" /* exit if failed */ 799 "li %0, 1\n\t" /* success - retval = 1 */ 800 "b 2f\n\t" /* we've succeeded */ 801 "1:\n\t" 802 "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 803 "stwx %0, 0, %7\n\t" 804 "li %0, 0\n\t" /* failure - retval = 0 */ 805 "2:\n\t" 806 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 807 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 808 : "cr0", "memory"); 809 810 return (ret); 811 } 812 #endif /* ISA_206_ATOMICS */ 813 814 static __inline int 815 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) 816 { 817 int ret; 818 819 __asm __volatile ( 820 "lwarx %0, 0, %3\n\t" /* load old value */ 821 "cmplw %4, %0\n\t" /* compare */ 822 "bne- 1f\n\t" /* exit if not equal */ 823 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 824 "bne- 1f\n\t" /* exit if failed */ 825 "li %0, 1\n\t" /* success - retval = 1 */ 826 "b 2f\n\t" /* we've succeeded */ 827 "1:\n\t" 828 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 829 "stwx %0, 0, %7\n\t" 830 "li %0, 0\n\t" /* failure - retval = 0 */ 831 "2:\n\t" 832 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 833 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 834 : "cr0", "memory"); 835 836 return (ret); 837 } 838 static __inline int 839 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) 840 { 841 int ret; 842 843 __asm __volatile ( 844 #ifdef __powerpc64__ 845 "ldarx %0, 0, %3\n\t" /* load old value */ 846 "cmpld %4, %0\n\t" /* compare */ 847 "bne- 1f\n\t" /* exit if not equal */ 848 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 849 #else 850 "lwarx %0, 0, %3\n\t" /* load old value */ 851 "cmplw %4, %0\n\t" /* compare */ 852 "bne- 1f\n\t" /* exit if not equal */ 853 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 854 #endif 855 "bne- 1f\n\t" /* exit if failed */ 856 "li %0, 1\n\t" /* success - retval = 1 */ 857 "b 2f\n\t" /* we've succeeded */ 858 "1:\n\t" 859 #ifdef __powerpc64__ 860 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 861 "stdx %0, 0, %7\n\t" 862 #else 863 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 864 "stwx %0, 0, %7\n\t" 865 #endif 866 "li %0, 0\n\t" /* failure - retval = 0 */ 867 "2:\n\t" 868 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 869 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 870 : "cr0", "memory"); 871 872 return (ret); 873 } 874 875 #define ATOMIC_FCMPSET_ACQ_REL(type) \ 876 static __inline int \ 877 atomic_fcmpset_acq_##type(volatile u_##type *p, \ 878 u_##type *cmpval, u_##type newval)\ 879 {\ 880 u_##type retval; \ 881 retval = atomic_fcmpset_##type(p, cmpval, newval);\ 882 __ATOMIC_ACQ();\ 883 return (retval);\ 884 }\ 885 static __inline int \ 886 atomic_fcmpset_rel_##type(volatile u_##type *p, \ 887 u_##type *cmpval, u_##type newval)\ 888 {\ 889 __ATOMIC_REL();\ 890 return (atomic_fcmpset_##type(p, cmpval, newval));\ 891 }\ 892 struct hack 893 894 ATOMIC_FCMPSET_ACQ_REL(int); 895 ATOMIC_FCMPSET_ACQ_REL(long); 896 897 #define atomic_fcmpset_8 atomic_fcmpset_char 898 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char 899 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char 900 901 #define atomic_fcmpset_16 atomic_fcmpset_short 902 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short 903 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short 904 905 #define atomic_fcmpset_32 atomic_fcmpset_int 906 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 907 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 908 909 #ifdef __powerpc64__ 910 #define atomic_fcmpset_64 atomic_fcmpset_long 911 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long 912 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long 913 914 #define atomic_fcmpset_ptr atomic_fcmpset_long 915 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 916 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 917 #else 918 #define atomic_fcmpset_ptr atomic_fcmpset_int 919 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int 920 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int 921 #endif 922 923 static __inline u_int 924 atomic_fetchadd_int(volatile u_int *p, u_int v) 925 { 926 u_int value; 927 928 do { 929 value = *p; 930 } while (!atomic_cmpset_int(p, value, value + v)); 931 return (value); 932 } 933 934 static __inline u_long 935 atomic_fetchadd_long(volatile u_long *p, u_long v) 936 { 937 u_long value; 938 939 do { 940 value = *p; 941 } while (!atomic_cmpset_long(p, value, value + v)); 942 return (value); 943 } 944 945 static __inline u_int 946 atomic_swap_32(volatile u_int *p, u_int v) 947 { 948 u_int prev; 949 950 __asm __volatile( 951 "1: lwarx %0,0,%2\n" 952 " stwcx. %3,0,%2\n" 953 " bne- 1b\n" 954 : "=&r" (prev), "+m" (*(volatile u_int *)p) 955 : "r" (p), "r" (v) 956 : "cr0", "memory"); 957 958 return (prev); 959 } 960 961 #ifdef __powerpc64__ 962 static __inline u_long 963 atomic_swap_64(volatile u_long *p, u_long v) 964 { 965 u_long prev; 966 967 __asm __volatile( 968 "1: ldarx %0,0,%2\n" 969 " stdcx. %3,0,%2\n" 970 " bne- 1b\n" 971 : "=&r" (prev), "+m" (*(volatile u_long *)p) 972 : "r" (p), "r" (v) 973 : "cr0", "memory"); 974 975 return (prev); 976 } 977 #endif 978 979 #define atomic_fetchadd_32 atomic_fetchadd_int 980 #define atomic_swap_int atomic_swap_32 981 982 #ifdef __powerpc64__ 983 #define atomic_fetchadd_64 atomic_fetchadd_long 984 #define atomic_swap_long atomic_swap_64 985 #define atomic_swap_ptr atomic_swap_64 986 #else 987 #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) 988 #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) 989 #endif 990 991 static __inline void 992 atomic_thread_fence_acq(void) 993 { 994 995 powerpc_lwsync(); 996 } 997 998 static __inline void 999 atomic_thread_fence_rel(void) 1000 { 1001 1002 powerpc_lwsync(); 1003 } 1004 1005 static __inline void 1006 atomic_thread_fence_acq_rel(void) 1007 { 1008 1009 powerpc_lwsync(); 1010 } 1011 1012 static __inline void 1013 atomic_thread_fence_seq_cst(void) 1014 { 1015 1016 __asm __volatile("sync" : : : "memory"); 1017 } 1018 1019 #ifndef ISA_206_ATOMICS 1020 #include <sys/_atomic_subword.h> 1021 #endif 1022 1023 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ 1024 ATOMIC_CMPSET_ACQ_REL(char); 1025 ATOMIC_CMPSET_ACQ_REL(short); 1026 1027 ATOMIC_FCMPSET_ACQ_REL(char); 1028 ATOMIC_FCMPSET_ACQ_REL(short); 1029 1030 #undef __ATOMIC_REL 1031 #undef __ATOMIC_ACQ 1032 1033 #endif /* ! _MACHINE_ATOMIC_H_ */ 1034