1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2008 Marcel Moolenaar 5 * Copyright (c) 2001 Benno Rice 6 * Copyright (c) 2001 David E. O'Brien 7 * Copyright (c) 1998 Doug Rabson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 #ifndef _MACHINE_ATOMIC_H_ 35 #define _MACHINE_ATOMIC_H_ 36 37 #ifndef _SYS_CDEFS_H_ 38 #error this file needs sys/cdefs.h as a prerequisite 39 #endif 40 41 #include <sys/atomic_common.h> 42 43 #ifndef __powerpc64__ 44 #include <sys/_atomic64e.h> 45 #endif 46 47 /* 48 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 49 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 50 * of this file. See also Appendix B.2 of Book II of the architecture manual. 51 * 52 * Note that not all Book-E processors accept the light-weight sync variant. 53 * In particular, early models of E500 cores are known to wedge. Bank on all 54 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 55 * to use the heavier-weight sync. 56 */ 57 58 #ifdef __powerpc64__ 59 #define mb() __asm __volatile("sync" : : : "memory") 60 #define rmb() __asm __volatile("lwsync" : : : "memory") 61 #define wmb() __asm __volatile("lwsync" : : : "memory") 62 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 63 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 64 #else 65 #define mb() __asm __volatile("sync" : : : "memory") 66 #define rmb() __asm __volatile("sync" : : : "memory") 67 #define wmb() __asm __volatile("sync" : : : "memory") 68 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 69 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 70 #endif 71 72 static __inline void 73 powerpc_lwsync(void) 74 { 75 76 #ifdef __powerpc64__ 77 __asm __volatile("lwsync" : : : "memory"); 78 #else 79 __asm __volatile("sync" : : : "memory"); 80 #endif 81 } 82 83 /* 84 * atomic_add(p, v) 85 * { *p += v; } 86 */ 87 88 #define __atomic_add_int(p, v, t) \ 89 __asm __volatile( \ 90 "1: lwarx %0, 0, %2\n" \ 91 " add %0, %3, %0\n" \ 92 " stwcx. %0, 0, %2\n" \ 93 " bne- 1b\n" \ 94 : "=&r" (t), "=m" (*p) \ 95 : "r" (p), "r" (v), "m" (*p) \ 96 : "cr0", "memory") \ 97 /* __atomic_add_int */ 98 99 #ifdef __powerpc64__ 100 #define __atomic_add_long(p, v, t) \ 101 __asm __volatile( \ 102 "1: ldarx %0, 0, %2\n" \ 103 " add %0, %3, %0\n" \ 104 " stdcx. %0, 0, %2\n" \ 105 " bne- 1b\n" \ 106 : "=&r" (t), "=m" (*p) \ 107 : "r" (p), "r" (v), "m" (*p) \ 108 : "cr0", "memory") \ 109 /* __atomic_add_long */ 110 #else 111 #define __atomic_add_long(p, v, t) \ 112 __asm __volatile( \ 113 "1: lwarx %0, 0, %2\n" \ 114 " add %0, %3, %0\n" \ 115 " stwcx. %0, 0, %2\n" \ 116 " bne- 1b\n" \ 117 : "=&r" (t), "=m" (*p) \ 118 : "r" (p), "r" (v), "m" (*p) \ 119 : "cr0", "memory") \ 120 /* __atomic_add_long */ 121 #endif 122 123 #define _ATOMIC_ADD(type) \ 124 static __inline void \ 125 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 126 u_##type t; \ 127 __atomic_add_##type(p, v, t); \ 128 } \ 129 \ 130 static __inline void \ 131 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 132 u_##type t; \ 133 __atomic_add_##type(p, v, t); \ 134 __ATOMIC_ACQ(); \ 135 } \ 136 \ 137 static __inline void \ 138 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 139 u_##type t; \ 140 __ATOMIC_REL(); \ 141 __atomic_add_##type(p, v, t); \ 142 } \ 143 /* _ATOMIC_ADD */ 144 145 _ATOMIC_ADD(int) 146 _ATOMIC_ADD(long) 147 148 #define atomic_add_32 atomic_add_int 149 #define atomic_add_acq_32 atomic_add_acq_int 150 #define atomic_add_rel_32 atomic_add_rel_int 151 152 #ifdef __powerpc64__ 153 #define atomic_add_64 atomic_add_long 154 #define atomic_add_acq_64 atomic_add_acq_long 155 #define atomic_add_rel_64 atomic_add_rel_long 156 157 #define atomic_add_ptr atomic_add_long 158 #define atomic_add_acq_ptr atomic_add_acq_long 159 #define atomic_add_rel_ptr atomic_add_rel_long 160 #else 161 #define atomic_add_ptr atomic_add_int 162 #define atomic_add_acq_ptr atomic_add_acq_int 163 #define atomic_add_rel_ptr atomic_add_rel_int 164 #endif 165 #undef _ATOMIC_ADD 166 #undef __atomic_add_long 167 #undef __atomic_add_int 168 169 /* 170 * atomic_clear(p, v) 171 * { *p &= ~v; } 172 */ 173 174 #define __atomic_clear_int(p, v, t) \ 175 __asm __volatile( \ 176 "1: lwarx %0, 0, %2\n" \ 177 " andc %0, %0, %3\n" \ 178 " stwcx. %0, 0, %2\n" \ 179 " bne- 1b\n" \ 180 : "=&r" (t), "=m" (*p) \ 181 : "r" (p), "r" (v), "m" (*p) \ 182 : "cr0", "memory") \ 183 /* __atomic_clear_int */ 184 185 #ifdef __powerpc64__ 186 #define __atomic_clear_long(p, v, t) \ 187 __asm __volatile( \ 188 "1: ldarx %0, 0, %2\n" \ 189 " andc %0, %0, %3\n" \ 190 " stdcx. %0, 0, %2\n" \ 191 " bne- 1b\n" \ 192 : "=&r" (t), "=m" (*p) \ 193 : "r" (p), "r" (v), "m" (*p) \ 194 : "cr0", "memory") \ 195 /* __atomic_clear_long */ 196 #else 197 #define __atomic_clear_long(p, v, t) \ 198 __asm __volatile( \ 199 "1: lwarx %0, 0, %2\n" \ 200 " andc %0, %0, %3\n" \ 201 " stwcx. %0, 0, %2\n" \ 202 " bne- 1b\n" \ 203 : "=&r" (t), "=m" (*p) \ 204 : "r" (p), "r" (v), "m" (*p) \ 205 : "cr0", "memory") \ 206 /* __atomic_clear_long */ 207 #endif 208 209 #define _ATOMIC_CLEAR(type) \ 210 static __inline void \ 211 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 212 u_##type t; \ 213 __atomic_clear_##type(p, v, t); \ 214 } \ 215 \ 216 static __inline void \ 217 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 218 u_##type t; \ 219 __atomic_clear_##type(p, v, t); \ 220 __ATOMIC_ACQ(); \ 221 } \ 222 \ 223 static __inline void \ 224 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 225 u_##type t; \ 226 __ATOMIC_REL(); \ 227 __atomic_clear_##type(p, v, t); \ 228 } \ 229 /* _ATOMIC_CLEAR */ 230 231 _ATOMIC_CLEAR(int) 232 _ATOMIC_CLEAR(long) 233 234 #define atomic_clear_32 atomic_clear_int 235 #define atomic_clear_acq_32 atomic_clear_acq_int 236 #define atomic_clear_rel_32 atomic_clear_rel_int 237 238 #ifdef __powerpc64__ 239 #define atomic_clear_64 atomic_clear_long 240 #define atomic_clear_acq_64 atomic_clear_acq_long 241 #define atomic_clear_rel_64 atomic_clear_rel_long 242 243 #define atomic_clear_ptr atomic_clear_long 244 #define atomic_clear_acq_ptr atomic_clear_acq_long 245 #define atomic_clear_rel_ptr atomic_clear_rel_long 246 #else 247 #define atomic_clear_ptr atomic_clear_int 248 #define atomic_clear_acq_ptr atomic_clear_acq_int 249 #define atomic_clear_rel_ptr atomic_clear_rel_int 250 #endif 251 #undef _ATOMIC_CLEAR 252 #undef __atomic_clear_long 253 #undef __atomic_clear_int 254 255 /* 256 * atomic_cmpset(p, o, n) 257 */ 258 /* TODO -- see below */ 259 260 /* 261 * atomic_load_acq(p) 262 */ 263 /* TODO -- see below */ 264 265 /* 266 * atomic_readandclear(p) 267 */ 268 /* TODO -- see below */ 269 270 /* 271 * atomic_set(p, v) 272 * { *p |= v; } 273 */ 274 275 #define __atomic_set_int(p, v, t) \ 276 __asm __volatile( \ 277 "1: lwarx %0, 0, %2\n" \ 278 " or %0, %3, %0\n" \ 279 " stwcx. %0, 0, %2\n" \ 280 " bne- 1b\n" \ 281 : "=&r" (t), "=m" (*p) \ 282 : "r" (p), "r" (v), "m" (*p) \ 283 : "cr0", "memory") \ 284 /* __atomic_set_int */ 285 286 #ifdef __powerpc64__ 287 #define __atomic_set_long(p, v, t) \ 288 __asm __volatile( \ 289 "1: ldarx %0, 0, %2\n" \ 290 " or %0, %3, %0\n" \ 291 " stdcx. %0, 0, %2\n" \ 292 " bne- 1b\n" \ 293 : "=&r" (t), "=m" (*p) \ 294 : "r" (p), "r" (v), "m" (*p) \ 295 : "cr0", "memory") \ 296 /* __atomic_set_long */ 297 #else 298 #define __atomic_set_long(p, v, t) \ 299 __asm __volatile( \ 300 "1: lwarx %0, 0, %2\n" \ 301 " or %0, %3, %0\n" \ 302 " stwcx. %0, 0, %2\n" \ 303 " bne- 1b\n" \ 304 : "=&r" (t), "=m" (*p) \ 305 : "r" (p), "r" (v), "m" (*p) \ 306 : "cr0", "memory") \ 307 /* __atomic_set_long */ 308 #endif 309 310 #define _ATOMIC_SET(type) \ 311 static __inline void \ 312 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 313 u_##type t; \ 314 __atomic_set_##type(p, v, t); \ 315 } \ 316 \ 317 static __inline void \ 318 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 319 u_##type t; \ 320 __atomic_set_##type(p, v, t); \ 321 __ATOMIC_ACQ(); \ 322 } \ 323 \ 324 static __inline void \ 325 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 326 u_##type t; \ 327 __ATOMIC_REL(); \ 328 __atomic_set_##type(p, v, t); \ 329 } \ 330 /* _ATOMIC_SET */ 331 332 _ATOMIC_SET(int) 333 _ATOMIC_SET(long) 334 335 #define atomic_set_32 atomic_set_int 336 #define atomic_set_acq_32 atomic_set_acq_int 337 #define atomic_set_rel_32 atomic_set_rel_int 338 339 #ifdef __powerpc64__ 340 #define atomic_set_64 atomic_set_long 341 #define atomic_set_acq_64 atomic_set_acq_long 342 #define atomic_set_rel_64 atomic_set_rel_long 343 344 #define atomic_set_ptr atomic_set_long 345 #define atomic_set_acq_ptr atomic_set_acq_long 346 #define atomic_set_rel_ptr atomic_set_rel_long 347 #else 348 #define atomic_set_ptr atomic_set_int 349 #define atomic_set_acq_ptr atomic_set_acq_int 350 #define atomic_set_rel_ptr atomic_set_rel_int 351 #endif 352 #undef _ATOMIC_SET 353 #undef __atomic_set_long 354 #undef __atomic_set_int 355 356 /* 357 * atomic_subtract(p, v) 358 * { *p -= v; } 359 */ 360 361 #define __atomic_subtract_int(p, v, t) \ 362 __asm __volatile( \ 363 "1: lwarx %0, 0, %2\n" \ 364 " subf %0, %3, %0\n" \ 365 " stwcx. %0, 0, %2\n" \ 366 " bne- 1b\n" \ 367 : "=&r" (t), "=m" (*p) \ 368 : "r" (p), "r" (v), "m" (*p) \ 369 : "cr0", "memory") \ 370 /* __atomic_subtract_int */ 371 372 #ifdef __powerpc64__ 373 #define __atomic_subtract_long(p, v, t) \ 374 __asm __volatile( \ 375 "1: ldarx %0, 0, %2\n" \ 376 " subf %0, %3, %0\n" \ 377 " stdcx. %0, 0, %2\n" \ 378 " bne- 1b\n" \ 379 : "=&r" (t), "=m" (*p) \ 380 : "r" (p), "r" (v), "m" (*p) \ 381 : "cr0", "memory") \ 382 /* __atomic_subtract_long */ 383 #else 384 #define __atomic_subtract_long(p, v, t) \ 385 __asm __volatile( \ 386 "1: lwarx %0, 0, %2\n" \ 387 " subf %0, %3, %0\n" \ 388 " stwcx. %0, 0, %2\n" \ 389 " bne- 1b\n" \ 390 : "=&r" (t), "=m" (*p) \ 391 : "r" (p), "r" (v), "m" (*p) \ 392 : "cr0", "memory") \ 393 /* __atomic_subtract_long */ 394 #endif 395 396 #define _ATOMIC_SUBTRACT(type) \ 397 static __inline void \ 398 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 399 u_##type t; \ 400 __atomic_subtract_##type(p, v, t); \ 401 } \ 402 \ 403 static __inline void \ 404 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 405 u_##type t; \ 406 __atomic_subtract_##type(p, v, t); \ 407 __ATOMIC_ACQ(); \ 408 } \ 409 \ 410 static __inline void \ 411 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 412 u_##type t; \ 413 __ATOMIC_REL(); \ 414 __atomic_subtract_##type(p, v, t); \ 415 } \ 416 /* _ATOMIC_SUBTRACT */ 417 418 _ATOMIC_SUBTRACT(int) 419 _ATOMIC_SUBTRACT(long) 420 421 #define atomic_subtract_32 atomic_subtract_int 422 #define atomic_subtract_acq_32 atomic_subtract_acq_int 423 #define atomic_subtract_rel_32 atomic_subtract_rel_int 424 425 #ifdef __powerpc64__ 426 #define atomic_subtract_64 atomic_subtract_long 427 #define atomic_subtract_acq_64 atomic_subract_acq_long 428 #define atomic_subtract_rel_64 atomic_subtract_rel_long 429 430 #define atomic_subtract_ptr atomic_subtract_long 431 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 432 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 433 #else 434 #define atomic_subtract_ptr atomic_subtract_int 435 #define atomic_subtract_acq_ptr atomic_subtract_acq_int 436 #define atomic_subtract_rel_ptr atomic_subtract_rel_int 437 #endif 438 #undef _ATOMIC_SUBTRACT 439 #undef __atomic_subtract_long 440 #undef __atomic_subtract_int 441 442 /* 443 * atomic_store_rel(p, v) 444 */ 445 /* TODO -- see below */ 446 447 /* 448 * Old/original implementations that still need revisiting. 449 */ 450 451 static __inline u_int 452 atomic_readandclear_int(volatile u_int *addr) 453 { 454 u_int result,temp; 455 456 __asm __volatile ( 457 "\tsync\n" /* drain writes */ 458 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 459 "li %1, 0\n\t" /* load new value */ 460 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 461 "bne- 1b\n\t" /* spin if failed */ 462 : "=&r"(result), "=&r"(temp), "=m" (*addr) 463 : "r" (addr), "m" (*addr) 464 : "cr0", "memory"); 465 466 return (result); 467 } 468 469 #ifdef __powerpc64__ 470 static __inline u_long 471 atomic_readandclear_long(volatile u_long *addr) 472 { 473 u_long result,temp; 474 475 __asm __volatile ( 476 "\tsync\n" /* drain writes */ 477 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 478 "li %1, 0\n\t" /* load new value */ 479 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 480 "bne- 1b\n\t" /* spin if failed */ 481 : "=&r"(result), "=&r"(temp), "=m" (*addr) 482 : "r" (addr), "m" (*addr) 483 : "cr0", "memory"); 484 485 return (result); 486 } 487 #endif 488 489 #define atomic_readandclear_32 atomic_readandclear_int 490 491 #ifdef __powerpc64__ 492 #define atomic_readandclear_64 atomic_readandclear_long 493 494 #define atomic_readandclear_ptr atomic_readandclear_long 495 #else 496 static __inline u_long 497 atomic_readandclear_long(volatile u_long *addr) 498 { 499 500 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 501 } 502 503 #define atomic_readandclear_ptr atomic_readandclear_int 504 #endif 505 506 /* 507 * We assume that a = b will do atomic loads and stores. 508 */ 509 #define ATOMIC_STORE_LOAD(TYPE) \ 510 static __inline u_##TYPE \ 511 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 512 { \ 513 u_##TYPE v; \ 514 \ 515 v = *p; \ 516 powerpc_lwsync(); \ 517 return (v); \ 518 } \ 519 \ 520 static __inline void \ 521 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 522 { \ 523 \ 524 powerpc_lwsync(); \ 525 *p = v; \ 526 } 527 528 ATOMIC_STORE_LOAD(int) 529 530 #define atomic_load_acq_32 atomic_load_acq_int 531 #define atomic_store_rel_32 atomic_store_rel_int 532 533 #ifdef __powerpc64__ 534 ATOMIC_STORE_LOAD(long) 535 536 #define atomic_load_acq_64 atomic_load_acq_long 537 #define atomic_store_rel_64 atomic_store_rel_long 538 539 #define atomic_load_acq_ptr atomic_load_acq_long 540 #define atomic_store_rel_ptr atomic_store_rel_long 541 #else 542 static __inline u_long 543 atomic_load_acq_long(volatile u_long *addr) 544 { 545 546 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 547 } 548 549 static __inline void 550 atomic_store_rel_long(volatile u_long *addr, u_long val) 551 { 552 553 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 554 } 555 556 #define atomic_load_acq_ptr atomic_load_acq_int 557 #define atomic_store_rel_ptr atomic_store_rel_int 558 #endif 559 #undef ATOMIC_STORE_LOAD 560 561 /* 562 * Atomically compare the value stored at *p with cmpval and if the 563 * two values are equal, update the value of *p with newval. Returns 564 * zero if the compare failed, nonzero otherwise. 565 */ 566 #ifdef ISA_206_ATOMICS 567 static __inline int 568 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) 569 { 570 int ret; 571 572 __asm __volatile ( 573 "1:\tlbarx %0, 0, %2\n\t" /* load old value */ 574 "cmplw %3, %0\n\t" /* compare */ 575 "bne- 2f\n\t" /* exit if not equal */ 576 "stbcx. %4, 0, %2\n\t" /* attempt to store */ 577 "bne- 1b\n\t" /* spin if failed */ 578 "li %0, 1\n\t" /* success - retval = 1 */ 579 "b 3f\n\t" /* we've succeeded */ 580 "2:\n\t" 581 "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 582 "li %0, 0\n\t" /* failure - retval = 0 */ 583 "3:\n\t" 584 : "=&r" (ret), "=m" (*p) 585 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 586 : "cr0", "memory"); 587 588 return (ret); 589 } 590 591 static __inline int 592 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) 593 { 594 int ret; 595 596 __asm __volatile ( 597 "1:\tlharx %0, 0, %2\n\t" /* load old value */ 598 "cmplw %3, %0\n\t" /* compare */ 599 "bne- 2f\n\t" /* exit if not equal */ 600 "sthcx. %4, 0, %2\n\t" /* attempt to store */ 601 "bne- 1b\n\t" /* spin if failed */ 602 "li %0, 1\n\t" /* success - retval = 1 */ 603 "b 3f\n\t" /* we've succeeded */ 604 "2:\n\t" 605 "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 606 "li %0, 0\n\t" /* failure - retval = 0 */ 607 "3:\n\t" 608 : "=&r" (ret), "=m" (*p) 609 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 610 : "cr0", "memory"); 611 612 return (ret); 613 } 614 #else 615 static __inline int 616 atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval, 617 uint32_t mask) 618 { 619 int ret; 620 uint32_t tmp; 621 622 __asm __volatile ( 623 "1:\tlwarx %2, 0, %3\n\t" /* load old value */ 624 "and %0, %2, %7\n\t" 625 "cmplw %4, %0\n\t" /* compare */ 626 "bne- 2f\n\t" /* exit if not equal */ 627 "andc %2, %2, %7\n\t" 628 "or %2, %2, %5\n\t" 629 "stwcx. %2, 0, %3\n\t" /* attempt to store */ 630 "bne- 1b\n\t" /* spin if failed */ 631 "li %0, 1\n\t" /* success - retval = 1 */ 632 "b 3f\n\t" /* we've succeeded */ 633 "2:\n\t" 634 "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */ 635 "li %0, 0\n\t" /* failure - retval = 0 */ 636 "3:\n\t" 637 : "=&r" (ret), "=m" (*p), "+&r" (tmp) 638 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p), 639 "r" (mask) 640 : "cr0", "memory"); 641 642 return (ret); 643 } 644 645 #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m) 646 #endif 647 648 static __inline int 649 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 650 { 651 int ret; 652 653 __asm __volatile ( 654 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 655 "cmplw %3, %0\n\t" /* compare */ 656 "bne- 2f\n\t" /* exit if not equal */ 657 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 658 "bne- 1b\n\t" /* spin if failed */ 659 "li %0, 1\n\t" /* success - retval = 1 */ 660 "b 3f\n\t" /* we've succeeded */ 661 "2:\n\t" 662 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 663 "li %0, 0\n\t" /* failure - retval = 0 */ 664 "3:\n\t" 665 : "=&r" (ret), "=m" (*p) 666 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 667 : "cr0", "memory"); 668 669 return (ret); 670 } 671 static __inline int 672 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 673 { 674 int ret; 675 676 __asm __volatile ( 677 #ifdef __powerpc64__ 678 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 679 "cmpld %3, %0\n\t" /* compare */ 680 "bne- 2f\n\t" /* exit if not equal */ 681 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 682 #else 683 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 684 "cmplw %3, %0\n\t" /* compare */ 685 "bne- 2f\n\t" /* exit if not equal */ 686 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 687 #endif 688 "bne- 1b\n\t" /* spin if failed */ 689 "li %0, 1\n\t" /* success - retval = 1 */ 690 "b 3f\n\t" /* we've succeeded */ 691 "2:\n\t" 692 #ifdef __powerpc64__ 693 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 694 #else 695 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 696 #endif 697 "li %0, 0\n\t" /* failure - retval = 0 */ 698 "3:\n\t" 699 : "=&r" (ret), "=m" (*p) 700 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 701 : "cr0", "memory"); 702 703 return (ret); 704 } 705 706 #define ATOMIC_CMPSET_ACQ_REL(type) \ 707 static __inline int \ 708 atomic_cmpset_acq_##type(volatile u_##type *p, \ 709 u_##type cmpval, u_##type newval)\ 710 {\ 711 u_##type retval; \ 712 retval = atomic_cmpset_##type(p, cmpval, newval);\ 713 __ATOMIC_ACQ();\ 714 return (retval);\ 715 }\ 716 static __inline int \ 717 atomic_cmpset_rel_##type(volatile u_##type *p, \ 718 u_##type cmpval, u_##type newval)\ 719 {\ 720 __ATOMIC_REL();\ 721 return (atomic_cmpset_##type(p, cmpval, newval));\ 722 }\ 723 struct hack 724 725 ATOMIC_CMPSET_ACQ_REL(int); 726 ATOMIC_CMPSET_ACQ_REL(long); 727 728 #ifdef ISA_206_ATOMICS 729 #define atomic_cmpset_8 atomic_cmpset_char 730 #endif 731 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char 732 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char 733 734 #ifdef ISA_206_ATOMICS 735 #define atomic_cmpset_16 atomic_cmpset_short 736 #endif 737 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short 738 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short 739 740 #define atomic_cmpset_32 atomic_cmpset_int 741 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 742 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 743 744 #ifdef __powerpc64__ 745 #define atomic_cmpset_64 atomic_cmpset_long 746 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 747 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 748 749 #define atomic_cmpset_ptr atomic_cmpset_long 750 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 751 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 752 #else 753 #define atomic_cmpset_ptr atomic_cmpset_int 754 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 755 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 756 #endif 757 758 /* 759 * Atomically compare the value stored at *p with *cmpval and if the 760 * two values are equal, update the value of *p with newval. Returns 761 * zero if the compare failed and sets *cmpval to the read value from *p, 762 * nonzero otherwise. 763 */ 764 #ifdef ISA_206_ATOMICS 765 static __inline int 766 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) 767 { 768 int ret; 769 770 __asm __volatile ( 771 "lbarx %0, 0, %3\n\t" /* load old value */ 772 "cmplw %4, %0\n\t" /* compare */ 773 "bne- 1f\n\t" /* exit if not equal */ 774 "stbcx. %5, 0, %3\n\t" /* attempt to store */ 775 "bne- 1f\n\t" /* exit if failed */ 776 "li %0, 1\n\t" /* success - retval = 1 */ 777 "b 2f\n\t" /* we've succeeded */ 778 "1:\n\t" 779 "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 780 "stwx %0, 0, %7\n\t" 781 "li %0, 0\n\t" /* failure - retval = 0 */ 782 "2:\n\t" 783 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 784 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 785 : "cr0", "memory"); 786 787 return (ret); 788 } 789 790 static __inline int 791 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) 792 { 793 int ret; 794 795 __asm __volatile ( 796 "lharx %0, 0, %3\n\t" /* load old value */ 797 "cmplw %4, %0\n\t" /* compare */ 798 "bne- 1f\n\t" /* exit if not equal */ 799 "sthcx. %5, 0, %3\n\t" /* attempt to store */ 800 "bne- 1f\n\t" /* exit if failed */ 801 "li %0, 1\n\t" /* success - retval = 1 */ 802 "b 2f\n\t" /* we've succeeded */ 803 "1:\n\t" 804 "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 805 "stwx %0, 0, %7\n\t" 806 "li %0, 0\n\t" /* failure - retval = 0 */ 807 "2:\n\t" 808 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 809 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 810 : "cr0", "memory"); 811 812 return (ret); 813 } 814 #endif /* ISA_206_ATOMICS */ 815 816 static __inline int 817 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) 818 { 819 int ret; 820 821 __asm __volatile ( 822 "lwarx %0, 0, %3\n\t" /* load old value */ 823 "cmplw %4, %0\n\t" /* compare */ 824 "bne- 1f\n\t" /* exit if not equal */ 825 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 826 "bne- 1f\n\t" /* exit if failed */ 827 "li %0, 1\n\t" /* success - retval = 1 */ 828 "b 2f\n\t" /* we've succeeded */ 829 "1:\n\t" 830 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 831 "stwx %0, 0, %7\n\t" 832 "li %0, 0\n\t" /* failure - retval = 0 */ 833 "2:\n\t" 834 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 835 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 836 : "cr0", "memory"); 837 838 return (ret); 839 } 840 static __inline int 841 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) 842 { 843 int ret; 844 845 __asm __volatile ( 846 #ifdef __powerpc64__ 847 "ldarx %0, 0, %3\n\t" /* load old value */ 848 "cmpld %4, %0\n\t" /* compare */ 849 "bne- 1f\n\t" /* exit if not equal */ 850 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 851 #else 852 "lwarx %0, 0, %3\n\t" /* load old value */ 853 "cmplw %4, %0\n\t" /* compare */ 854 "bne- 1f\n\t" /* exit if not equal */ 855 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 856 #endif 857 "bne- 1f\n\t" /* exit if failed */ 858 "li %0, 1\n\t" /* success - retval = 1 */ 859 "b 2f\n\t" /* we've succeeded */ 860 "1:\n\t" 861 #ifdef __powerpc64__ 862 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 863 "stdx %0, 0, %7\n\t" 864 #else 865 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 866 "stwx %0, 0, %7\n\t" 867 #endif 868 "li %0, 0\n\t" /* failure - retval = 0 */ 869 "2:\n\t" 870 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 871 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 872 : "cr0", "memory"); 873 874 return (ret); 875 } 876 877 #define ATOMIC_FCMPSET_ACQ_REL(type) \ 878 static __inline int \ 879 atomic_fcmpset_acq_##type(volatile u_##type *p, \ 880 u_##type *cmpval, u_##type newval)\ 881 {\ 882 u_##type retval; \ 883 retval = atomic_fcmpset_##type(p, cmpval, newval);\ 884 __ATOMIC_ACQ();\ 885 return (retval);\ 886 }\ 887 static __inline int \ 888 atomic_fcmpset_rel_##type(volatile u_##type *p, \ 889 u_##type *cmpval, u_##type newval)\ 890 {\ 891 __ATOMIC_REL();\ 892 return (atomic_fcmpset_##type(p, cmpval, newval));\ 893 }\ 894 struct hack 895 896 ATOMIC_FCMPSET_ACQ_REL(int); 897 ATOMIC_FCMPSET_ACQ_REL(long); 898 899 #ifdef ISA_206_ATOMICS 900 #define atomic_fcmpset_8 atomic_fcmpset_char 901 #endif 902 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char 903 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char 904 905 #ifdef ISA_206_ATOMICS 906 #define atomic_fcmpset_16 atomic_fcmpset_short 907 #endif 908 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short 909 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short 910 911 #define atomic_fcmpset_32 atomic_fcmpset_int 912 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 913 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 914 915 #ifdef __powerpc64__ 916 #define atomic_fcmpset_64 atomic_fcmpset_long 917 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long 918 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long 919 920 #define atomic_fcmpset_ptr atomic_fcmpset_long 921 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 922 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 923 #else 924 #define atomic_fcmpset_ptr atomic_fcmpset_int 925 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int 926 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int 927 #endif 928 929 static __inline u_int 930 atomic_fetchadd_int(volatile u_int *p, u_int v) 931 { 932 u_int value; 933 934 do { 935 value = *p; 936 } while (!atomic_cmpset_int(p, value, value + v)); 937 return (value); 938 } 939 940 static __inline u_long 941 atomic_fetchadd_long(volatile u_long *p, u_long v) 942 { 943 u_long value; 944 945 do { 946 value = *p; 947 } while (!atomic_cmpset_long(p, value, value + v)); 948 return (value); 949 } 950 951 static __inline u_int 952 atomic_swap_32(volatile u_int *p, u_int v) 953 { 954 u_int prev; 955 956 __asm __volatile( 957 "1: lwarx %0,0,%2\n" 958 " stwcx. %3,0,%2\n" 959 " bne- 1b\n" 960 : "=&r" (prev), "+m" (*(volatile u_int *)p) 961 : "r" (p), "r" (v) 962 : "cr0", "memory"); 963 964 return (prev); 965 } 966 967 #ifdef __powerpc64__ 968 static __inline u_long 969 atomic_swap_64(volatile u_long *p, u_long v) 970 { 971 u_long prev; 972 973 __asm __volatile( 974 "1: ldarx %0,0,%2\n" 975 " stdcx. %3,0,%2\n" 976 " bne- 1b\n" 977 : "=&r" (prev), "+m" (*(volatile u_long *)p) 978 : "r" (p), "r" (v) 979 : "cr0", "memory"); 980 981 return (prev); 982 } 983 #endif 984 985 #define atomic_fetchadd_32 atomic_fetchadd_int 986 #define atomic_swap_int atomic_swap_32 987 988 #ifdef __powerpc64__ 989 #define atomic_fetchadd_64 atomic_fetchadd_long 990 #define atomic_swap_long atomic_swap_64 991 #define atomic_swap_ptr atomic_swap_64 992 #else 993 #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) 994 #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) 995 #endif 996 997 static __inline void 998 atomic_thread_fence_acq(void) 999 { 1000 1001 powerpc_lwsync(); 1002 } 1003 1004 static __inline void 1005 atomic_thread_fence_rel(void) 1006 { 1007 1008 powerpc_lwsync(); 1009 } 1010 1011 static __inline void 1012 atomic_thread_fence_acq_rel(void) 1013 { 1014 1015 powerpc_lwsync(); 1016 } 1017 1018 static __inline void 1019 atomic_thread_fence_seq_cst(void) 1020 { 1021 1022 __asm __volatile("sync" : : : "memory"); 1023 } 1024 1025 #ifndef ISA_206_ATOMICS 1026 #include <sys/_atomic_subword.h> 1027 #define atomic_cmpset_char atomic_cmpset_8 1028 #define atomic_cmpset_short atomic_cmpset_16 1029 #define atomic_fcmpset_char atomic_fcmpset_8 1030 #define atomic_fcmpset_short atomic_fcmpset_16 1031 #endif 1032 1033 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ 1034 ATOMIC_CMPSET_ACQ_REL(char); 1035 ATOMIC_CMPSET_ACQ_REL(short); 1036 1037 ATOMIC_FCMPSET_ACQ_REL(char); 1038 ATOMIC_FCMPSET_ACQ_REL(short); 1039 1040 #undef __ATOMIC_REL 1041 #undef __ATOMIC_ACQ 1042 1043 #endif /* ! _MACHINE_ATOMIC_H_ */ 1044