1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2008 Marcel Moolenaar 5 * Copyright (c) 2001 Benno Rice 6 * Copyright (c) 2001 David E. O'Brien 7 * Copyright (c) 1998 Doug Rabson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 #ifndef _MACHINE_ATOMIC_H_ 35 #define _MACHINE_ATOMIC_H_ 36 37 #ifndef _SYS_CDEFS_H_ 38 #error this file needs sys/cdefs.h as a prerequisite 39 #endif 40 41 #include <sys/atomic_common.h> 42 43 /* 44 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 45 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 46 * of this file. See also Appendix B.2 of Book II of the architecture manual. 47 * 48 * Note that not all Book-E processors accept the light-weight sync variant. 49 * In particular, early models of E500 cores are known to wedge. Bank on all 50 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 51 * to use the heavier-weight sync. 52 */ 53 54 #ifdef __powerpc64__ 55 #define mb() __asm __volatile("sync" : : : "memory") 56 #define rmb() __asm __volatile("lwsync" : : : "memory") 57 #define wmb() __asm __volatile("lwsync" : : : "memory") 58 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 59 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 60 #else 61 #define mb() __asm __volatile("sync" : : : "memory") 62 #define rmb() __asm __volatile("sync" : : : "memory") 63 #define wmb() __asm __volatile("sync" : : : "memory") 64 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 65 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 66 #endif 67 68 static __inline void 69 powerpc_lwsync(void) 70 { 71 72 #ifdef __powerpc64__ 73 __asm __volatile("lwsync" : : : "memory"); 74 #else 75 __asm __volatile("sync" : : : "memory"); 76 #endif 77 } 78 79 /* 80 * atomic_add(p, v) 81 * { *p += v; } 82 */ 83 84 #define __atomic_add_int(p, v, t) \ 85 __asm __volatile( \ 86 "1: lwarx %0, 0, %2\n" \ 87 " add %0, %3, %0\n" \ 88 " stwcx. %0, 0, %2\n" \ 89 " bne- 1b\n" \ 90 : "=&r" (t), "=m" (*p) \ 91 : "r" (p), "r" (v), "m" (*p) \ 92 : "cr0", "memory") \ 93 /* __atomic_add_int */ 94 95 #ifdef __powerpc64__ 96 #define __atomic_add_long(p, v, t) \ 97 __asm __volatile( \ 98 "1: ldarx %0, 0, %2\n" \ 99 " add %0, %3, %0\n" \ 100 " stdcx. %0, 0, %2\n" \ 101 " bne- 1b\n" \ 102 : "=&r" (t), "=m" (*p) \ 103 : "r" (p), "r" (v), "m" (*p) \ 104 : "cr0", "memory") \ 105 /* __atomic_add_long */ 106 #else 107 #define __atomic_add_long(p, v, t) \ 108 __asm __volatile( \ 109 "1: lwarx %0, 0, %2\n" \ 110 " add %0, %3, %0\n" \ 111 " stwcx. %0, 0, %2\n" \ 112 " bne- 1b\n" \ 113 : "=&r" (t), "=m" (*p) \ 114 : "r" (p), "r" (v), "m" (*p) \ 115 : "cr0", "memory") \ 116 /* __atomic_add_long */ 117 #endif 118 119 #define _ATOMIC_ADD(type) \ 120 static __inline void \ 121 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 122 u_##type t; \ 123 __atomic_add_##type(p, v, t); \ 124 } \ 125 \ 126 static __inline void \ 127 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 128 u_##type t; \ 129 __atomic_add_##type(p, v, t); \ 130 __ATOMIC_ACQ(); \ 131 } \ 132 \ 133 static __inline void \ 134 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 135 u_##type t; \ 136 __ATOMIC_REL(); \ 137 __atomic_add_##type(p, v, t); \ 138 } \ 139 /* _ATOMIC_ADD */ 140 141 _ATOMIC_ADD(int) 142 _ATOMIC_ADD(long) 143 144 #define atomic_add_32 atomic_add_int 145 #define atomic_add_acq_32 atomic_add_acq_int 146 #define atomic_add_rel_32 atomic_add_rel_int 147 148 #ifdef __powerpc64__ 149 #define atomic_add_64 atomic_add_long 150 #define atomic_add_acq_64 atomic_add_acq_long 151 #define atomic_add_rel_64 atomic_add_rel_long 152 153 #define atomic_add_ptr atomic_add_long 154 #define atomic_add_acq_ptr atomic_add_acq_long 155 #define atomic_add_rel_ptr atomic_add_rel_long 156 #else 157 #define atomic_add_ptr atomic_add_int 158 #define atomic_add_acq_ptr atomic_add_acq_int 159 #define atomic_add_rel_ptr atomic_add_rel_int 160 #endif 161 #undef _ATOMIC_ADD 162 #undef __atomic_add_long 163 #undef __atomic_add_int 164 165 /* 166 * atomic_clear(p, v) 167 * { *p &= ~v; } 168 */ 169 170 #define __atomic_clear_int(p, v, t) \ 171 __asm __volatile( \ 172 "1: lwarx %0, 0, %2\n" \ 173 " andc %0, %0, %3\n" \ 174 " stwcx. %0, 0, %2\n" \ 175 " bne- 1b\n" \ 176 : "=&r" (t), "=m" (*p) \ 177 : "r" (p), "r" (v), "m" (*p) \ 178 : "cr0", "memory") \ 179 /* __atomic_clear_int */ 180 181 #ifdef __powerpc64__ 182 #define __atomic_clear_long(p, v, t) \ 183 __asm __volatile( \ 184 "1: ldarx %0, 0, %2\n" \ 185 " andc %0, %0, %3\n" \ 186 " stdcx. %0, 0, %2\n" \ 187 " bne- 1b\n" \ 188 : "=&r" (t), "=m" (*p) \ 189 : "r" (p), "r" (v), "m" (*p) \ 190 : "cr0", "memory") \ 191 /* __atomic_clear_long */ 192 #else 193 #define __atomic_clear_long(p, v, t) \ 194 __asm __volatile( \ 195 "1: lwarx %0, 0, %2\n" \ 196 " andc %0, %0, %3\n" \ 197 " stwcx. %0, 0, %2\n" \ 198 " bne- 1b\n" \ 199 : "=&r" (t), "=m" (*p) \ 200 : "r" (p), "r" (v), "m" (*p) \ 201 : "cr0", "memory") \ 202 /* __atomic_clear_long */ 203 #endif 204 205 #define _ATOMIC_CLEAR(type) \ 206 static __inline void \ 207 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 208 u_##type t; \ 209 __atomic_clear_##type(p, v, t); \ 210 } \ 211 \ 212 static __inline void \ 213 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 214 u_##type t; \ 215 __atomic_clear_##type(p, v, t); \ 216 __ATOMIC_ACQ(); \ 217 } \ 218 \ 219 static __inline void \ 220 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 221 u_##type t; \ 222 __ATOMIC_REL(); \ 223 __atomic_clear_##type(p, v, t); \ 224 } \ 225 /* _ATOMIC_CLEAR */ 226 227 228 _ATOMIC_CLEAR(int) 229 _ATOMIC_CLEAR(long) 230 231 #define atomic_clear_32 atomic_clear_int 232 #define atomic_clear_acq_32 atomic_clear_acq_int 233 #define atomic_clear_rel_32 atomic_clear_rel_int 234 235 #ifdef __powerpc64__ 236 #define atomic_clear_64 atomic_clear_long 237 #define atomic_clear_acq_64 atomic_clear_acq_long 238 #define atomic_clear_rel_64 atomic_clear_rel_long 239 240 #define atomic_clear_ptr atomic_clear_long 241 #define atomic_clear_acq_ptr atomic_clear_acq_long 242 #define atomic_clear_rel_ptr atomic_clear_rel_long 243 #else 244 #define atomic_clear_ptr atomic_clear_int 245 #define atomic_clear_acq_ptr atomic_clear_acq_int 246 #define atomic_clear_rel_ptr atomic_clear_rel_int 247 #endif 248 #undef _ATOMIC_CLEAR 249 #undef __atomic_clear_long 250 #undef __atomic_clear_int 251 252 /* 253 * atomic_cmpset(p, o, n) 254 */ 255 /* TODO -- see below */ 256 257 /* 258 * atomic_load_acq(p) 259 */ 260 /* TODO -- see below */ 261 262 /* 263 * atomic_readandclear(p) 264 */ 265 /* TODO -- see below */ 266 267 /* 268 * atomic_set(p, v) 269 * { *p |= v; } 270 */ 271 272 #define __atomic_set_int(p, v, t) \ 273 __asm __volatile( \ 274 "1: lwarx %0, 0, %2\n" \ 275 " or %0, %3, %0\n" \ 276 " stwcx. %0, 0, %2\n" \ 277 " bne- 1b\n" \ 278 : "=&r" (t), "=m" (*p) \ 279 : "r" (p), "r" (v), "m" (*p) \ 280 : "cr0", "memory") \ 281 /* __atomic_set_int */ 282 283 #ifdef __powerpc64__ 284 #define __atomic_set_long(p, v, t) \ 285 __asm __volatile( \ 286 "1: ldarx %0, 0, %2\n" \ 287 " or %0, %3, %0\n" \ 288 " stdcx. %0, 0, %2\n" \ 289 " bne- 1b\n" \ 290 : "=&r" (t), "=m" (*p) \ 291 : "r" (p), "r" (v), "m" (*p) \ 292 : "cr0", "memory") \ 293 /* __atomic_set_long */ 294 #else 295 #define __atomic_set_long(p, v, t) \ 296 __asm __volatile( \ 297 "1: lwarx %0, 0, %2\n" \ 298 " or %0, %3, %0\n" \ 299 " stwcx. %0, 0, %2\n" \ 300 " bne- 1b\n" \ 301 : "=&r" (t), "=m" (*p) \ 302 : "r" (p), "r" (v), "m" (*p) \ 303 : "cr0", "memory") \ 304 /* __atomic_set_long */ 305 #endif 306 307 #define _ATOMIC_SET(type) \ 308 static __inline void \ 309 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 310 u_##type t; \ 311 __atomic_set_##type(p, v, t); \ 312 } \ 313 \ 314 static __inline void \ 315 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 316 u_##type t; \ 317 __atomic_set_##type(p, v, t); \ 318 __ATOMIC_ACQ(); \ 319 } \ 320 \ 321 static __inline void \ 322 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 323 u_##type t; \ 324 __ATOMIC_REL(); \ 325 __atomic_set_##type(p, v, t); \ 326 } \ 327 /* _ATOMIC_SET */ 328 329 _ATOMIC_SET(int) 330 _ATOMIC_SET(long) 331 332 #define atomic_set_32 atomic_set_int 333 #define atomic_set_acq_32 atomic_set_acq_int 334 #define atomic_set_rel_32 atomic_set_rel_int 335 336 #ifdef __powerpc64__ 337 #define atomic_set_64 atomic_set_long 338 #define atomic_set_acq_64 atomic_set_acq_long 339 #define atomic_set_rel_64 atomic_set_rel_long 340 341 #define atomic_set_ptr atomic_set_long 342 #define atomic_set_acq_ptr atomic_set_acq_long 343 #define atomic_set_rel_ptr atomic_set_rel_long 344 #else 345 #define atomic_set_ptr atomic_set_int 346 #define atomic_set_acq_ptr atomic_set_acq_int 347 #define atomic_set_rel_ptr atomic_set_rel_int 348 #endif 349 #undef _ATOMIC_SET 350 #undef __atomic_set_long 351 #undef __atomic_set_int 352 353 /* 354 * atomic_subtract(p, v) 355 * { *p -= v; } 356 */ 357 358 #define __atomic_subtract_int(p, v, t) \ 359 __asm __volatile( \ 360 "1: lwarx %0, 0, %2\n" \ 361 " subf %0, %3, %0\n" \ 362 " stwcx. %0, 0, %2\n" \ 363 " bne- 1b\n" \ 364 : "=&r" (t), "=m" (*p) \ 365 : "r" (p), "r" (v), "m" (*p) \ 366 : "cr0", "memory") \ 367 /* __atomic_subtract_int */ 368 369 #ifdef __powerpc64__ 370 #define __atomic_subtract_long(p, v, t) \ 371 __asm __volatile( \ 372 "1: ldarx %0, 0, %2\n" \ 373 " subf %0, %3, %0\n" \ 374 " stdcx. %0, 0, %2\n" \ 375 " bne- 1b\n" \ 376 : "=&r" (t), "=m" (*p) \ 377 : "r" (p), "r" (v), "m" (*p) \ 378 : "cr0", "memory") \ 379 /* __atomic_subtract_long */ 380 #else 381 #define __atomic_subtract_long(p, v, t) \ 382 __asm __volatile( \ 383 "1: lwarx %0, 0, %2\n" \ 384 " subf %0, %3, %0\n" \ 385 " stwcx. %0, 0, %2\n" \ 386 " bne- 1b\n" \ 387 : "=&r" (t), "=m" (*p) \ 388 : "r" (p), "r" (v), "m" (*p) \ 389 : "cr0", "memory") \ 390 /* __atomic_subtract_long */ 391 #endif 392 393 #define _ATOMIC_SUBTRACT(type) \ 394 static __inline void \ 395 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 396 u_##type t; \ 397 __atomic_subtract_##type(p, v, t); \ 398 } \ 399 \ 400 static __inline void \ 401 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 402 u_##type t; \ 403 __atomic_subtract_##type(p, v, t); \ 404 __ATOMIC_ACQ(); \ 405 } \ 406 \ 407 static __inline void \ 408 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 409 u_##type t; \ 410 __ATOMIC_REL(); \ 411 __atomic_subtract_##type(p, v, t); \ 412 } \ 413 /* _ATOMIC_SUBTRACT */ 414 415 _ATOMIC_SUBTRACT(int) 416 _ATOMIC_SUBTRACT(long) 417 418 #define atomic_subtract_32 atomic_subtract_int 419 #define atomic_subtract_acq_32 atomic_subtract_acq_int 420 #define atomic_subtract_rel_32 atomic_subtract_rel_int 421 422 #ifdef __powerpc64__ 423 #define atomic_subtract_64 atomic_subtract_long 424 #define atomic_subtract_acq_64 atomic_subract_acq_long 425 #define atomic_subtract_rel_64 atomic_subtract_rel_long 426 427 #define atomic_subtract_ptr atomic_subtract_long 428 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 429 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 430 #else 431 #define atomic_subtract_ptr atomic_subtract_int 432 #define atomic_subtract_acq_ptr atomic_subtract_acq_int 433 #define atomic_subtract_rel_ptr atomic_subtract_rel_int 434 #endif 435 #undef _ATOMIC_SUBTRACT 436 #undef __atomic_subtract_long 437 #undef __atomic_subtract_int 438 439 /* 440 * atomic_store_rel(p, v) 441 */ 442 /* TODO -- see below */ 443 444 /* 445 * Old/original implementations that still need revisiting. 446 */ 447 448 static __inline u_int 449 atomic_readandclear_int(volatile u_int *addr) 450 { 451 u_int result,temp; 452 453 __asm __volatile ( 454 "\tsync\n" /* drain writes */ 455 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 456 "li %1, 0\n\t" /* load new value */ 457 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 458 "bne- 1b\n\t" /* spin if failed */ 459 : "=&r"(result), "=&r"(temp), "=m" (*addr) 460 : "r" (addr), "m" (*addr) 461 : "cr0", "memory"); 462 463 return (result); 464 } 465 466 #ifdef __powerpc64__ 467 static __inline u_long 468 atomic_readandclear_long(volatile u_long *addr) 469 { 470 u_long result,temp; 471 472 __asm __volatile ( 473 "\tsync\n" /* drain writes */ 474 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 475 "li %1, 0\n\t" /* load new value */ 476 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 477 "bne- 1b\n\t" /* spin if failed */ 478 : "=&r"(result), "=&r"(temp), "=m" (*addr) 479 : "r" (addr), "m" (*addr) 480 : "cr0", "memory"); 481 482 return (result); 483 } 484 #endif 485 486 #define atomic_readandclear_32 atomic_readandclear_int 487 488 #ifdef __powerpc64__ 489 #define atomic_readandclear_64 atomic_readandclear_long 490 491 #define atomic_readandclear_ptr atomic_readandclear_long 492 #else 493 static __inline u_long 494 atomic_readandclear_long(volatile u_long *addr) 495 { 496 497 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 498 } 499 500 #define atomic_readandclear_ptr atomic_readandclear_int 501 #endif 502 503 /* 504 * We assume that a = b will do atomic loads and stores. 505 */ 506 #define ATOMIC_STORE_LOAD(TYPE) \ 507 static __inline u_##TYPE \ 508 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 509 { \ 510 u_##TYPE v; \ 511 \ 512 v = *p; \ 513 powerpc_lwsync(); \ 514 return (v); \ 515 } \ 516 \ 517 static __inline void \ 518 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 519 { \ 520 \ 521 powerpc_lwsync(); \ 522 *p = v; \ 523 } 524 525 ATOMIC_STORE_LOAD(int) 526 527 #define atomic_load_acq_32 atomic_load_acq_int 528 #define atomic_store_rel_32 atomic_store_rel_int 529 530 #ifdef __powerpc64__ 531 ATOMIC_STORE_LOAD(long) 532 533 #define atomic_load_acq_64 atomic_load_acq_long 534 #define atomic_store_rel_64 atomic_store_rel_long 535 536 #define atomic_load_acq_ptr atomic_load_acq_long 537 #define atomic_store_rel_ptr atomic_store_rel_long 538 #else 539 static __inline u_long 540 atomic_load_acq_long(volatile u_long *addr) 541 { 542 543 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 544 } 545 546 static __inline void 547 atomic_store_rel_long(volatile u_long *addr, u_long val) 548 { 549 550 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 551 } 552 553 #define atomic_load_acq_ptr atomic_load_acq_int 554 #define atomic_store_rel_ptr atomic_store_rel_int 555 #endif 556 #undef ATOMIC_STORE_LOAD 557 558 /* 559 * Atomically compare the value stored at *p with cmpval and if the 560 * two values are equal, update the value of *p with newval. Returns 561 * zero if the compare failed, nonzero otherwise. 562 */ 563 #ifdef ISA_206_ATOMICS 564 static __inline int 565 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) 566 { 567 int ret; 568 569 __asm __volatile ( 570 "1:\tlbarx %0, 0, %2\n\t" /* load old value */ 571 "cmplw %3, %0\n\t" /* compare */ 572 "bne- 2f\n\t" /* exit if not equal */ 573 "stbcx. %4, 0, %2\n\t" /* attempt to store */ 574 "bne- 1b\n\t" /* spin if failed */ 575 "li %0, 1\n\t" /* success - retval = 1 */ 576 "b 3f\n\t" /* we've succeeded */ 577 "2:\n\t" 578 "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 579 "li %0, 0\n\t" /* failure - retval = 0 */ 580 "3:\n\t" 581 : "=&r" (ret), "=m" (*p) 582 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 583 : "cr0", "memory"); 584 585 return (ret); 586 } 587 588 static __inline int 589 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) 590 { 591 int ret; 592 593 __asm __volatile ( 594 "1:\tlharx %0, 0, %2\n\t" /* load old value */ 595 "cmplw %3, %0\n\t" /* compare */ 596 "bne- 2f\n\t" /* exit if not equal */ 597 "sthcx. %4, 0, %2\n\t" /* attempt to store */ 598 "bne- 1b\n\t" /* spin if failed */ 599 "li %0, 1\n\t" /* success - retval = 1 */ 600 "b 3f\n\t" /* we've succeeded */ 601 "2:\n\t" 602 "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 603 "li %0, 0\n\t" /* failure - retval = 0 */ 604 "3:\n\t" 605 : "=&r" (ret), "=m" (*p) 606 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 607 : "cr0", "memory"); 608 609 return (ret); 610 } 611 #endif 612 613 static __inline int 614 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 615 { 616 int ret; 617 618 __asm __volatile ( 619 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 620 "cmplw %3, %0\n\t" /* compare */ 621 "bne- 2f\n\t" /* exit if not equal */ 622 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 623 "bne- 1b\n\t" /* spin if failed */ 624 "li %0, 1\n\t" /* success - retval = 1 */ 625 "b 3f\n\t" /* we've succeeded */ 626 "2:\n\t" 627 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 628 "li %0, 0\n\t" /* failure - retval = 0 */ 629 "3:\n\t" 630 : "=&r" (ret), "=m" (*p) 631 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 632 : "cr0", "memory"); 633 634 return (ret); 635 } 636 static __inline int 637 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 638 { 639 int ret; 640 641 __asm __volatile ( 642 #ifdef __powerpc64__ 643 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 644 "cmpld %3, %0\n\t" /* compare */ 645 "bne- 2f\n\t" /* exit if not equal */ 646 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 647 #else 648 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 649 "cmplw %3, %0\n\t" /* compare */ 650 "bne- 2f\n\t" /* exit if not equal */ 651 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 652 #endif 653 "bne- 1b\n\t" /* spin if failed */ 654 "li %0, 1\n\t" /* success - retval = 1 */ 655 "b 3f\n\t" /* we've succeeded */ 656 "2:\n\t" 657 #ifdef __powerpc64__ 658 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 659 #else 660 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 661 #endif 662 "li %0, 0\n\t" /* failure - retval = 0 */ 663 "3:\n\t" 664 : "=&r" (ret), "=m" (*p) 665 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 666 : "cr0", "memory"); 667 668 return (ret); 669 } 670 671 #define ATOMIC_CMPSET_ACQ_REL(type) \ 672 static __inline int \ 673 atomic_cmpset_acq_##type(volatile u_##type *p, \ 674 u_##type cmpval, u_##type newval)\ 675 {\ 676 u_##type retval; \ 677 retval = atomic_cmpset_##type(p, cmpval, newval);\ 678 __ATOMIC_ACQ();\ 679 return (retval);\ 680 }\ 681 static __inline int \ 682 atomic_cmpset_rel_##type(volatile u_##type *p, \ 683 u_##type cmpval, u_##type newval)\ 684 {\ 685 __ATOMIC_ACQ();\ 686 return (atomic_cmpset_##type(p, cmpval, newval));\ 687 }\ 688 struct hack 689 690 ATOMIC_CMPSET_ACQ_REL(int); 691 ATOMIC_CMPSET_ACQ_REL(long); 692 693 694 #define atomic_cmpset_8 atomic_cmpset_char 695 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char 696 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char 697 698 #define atomic_cmpset_16 atomic_cmpset_short 699 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short 700 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short 701 702 #define atomic_cmpset_32 atomic_cmpset_int 703 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 704 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 705 706 #ifdef __powerpc64__ 707 #define atomic_cmpset_64 atomic_cmpset_long 708 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 709 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 710 711 #define atomic_cmpset_ptr atomic_cmpset_long 712 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 713 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 714 #else 715 #define atomic_cmpset_ptr atomic_cmpset_int 716 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 717 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 718 #endif 719 720 /* 721 * Atomically compare the value stored at *p with *cmpval and if the 722 * two values are equal, update the value of *p with newval. Returns 723 * zero if the compare failed and sets *cmpval to the read value from *p, 724 * nonzero otherwise. 725 */ 726 #ifdef ISA_206_ATOMICS 727 static __inline int 728 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) 729 { 730 int ret; 731 732 __asm __volatile ( 733 "lbarx %0, 0, %3\n\t" /* load old value */ 734 "cmplw %4, %0\n\t" /* compare */ 735 "bne- 1f\n\t" /* exit if not equal */ 736 "stbcx. %5, 0, %3\n\t" /* attempt to store */ 737 "bne- 1f\n\t" /* exit if failed */ 738 "li %0, 1\n\t" /* success - retval = 1 */ 739 "b 2f\n\t" /* we've succeeded */ 740 "1:\n\t" 741 "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 742 "stwx %0, 0, %7\n\t" 743 "li %0, 0\n\t" /* failure - retval = 0 */ 744 "2:\n\t" 745 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 746 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 747 : "cr0", "memory"); 748 749 return (ret); 750 } 751 752 static __inline int 753 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) 754 { 755 int ret; 756 757 __asm __volatile ( 758 "lharx %0, 0, %3\n\t" /* load old value */ 759 "cmplw %4, %0\n\t" /* compare */ 760 "bne- 1f\n\t" /* exit if not equal */ 761 "sthcx. %5, 0, %3\n\t" /* attempt to store */ 762 "bne- 1f\n\t" /* exit if failed */ 763 "li %0, 1\n\t" /* success - retval = 1 */ 764 "b 2f\n\t" /* we've succeeded */ 765 "1:\n\t" 766 "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 767 "stwx %0, 0, %7\n\t" 768 "li %0, 0\n\t" /* failure - retval = 0 */ 769 "2:\n\t" 770 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 771 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 772 : "cr0", "memory"); 773 774 return (ret); 775 } 776 #endif /* ISA_206_ATOMICS */ 777 778 static __inline int 779 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) 780 { 781 int ret; 782 783 __asm __volatile ( 784 "lwarx %0, 0, %3\n\t" /* load old value */ 785 "cmplw %4, %0\n\t" /* compare */ 786 "bne- 1f\n\t" /* exit if not equal */ 787 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 788 "bne- 1f\n\t" /* exit if failed */ 789 "li %0, 1\n\t" /* success - retval = 1 */ 790 "b 2f\n\t" /* we've succeeded */ 791 "1:\n\t" 792 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 793 "stwx %0, 0, %7\n\t" 794 "li %0, 0\n\t" /* failure - retval = 0 */ 795 "2:\n\t" 796 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 797 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 798 : "cr0", "memory"); 799 800 return (ret); 801 } 802 static __inline int 803 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) 804 { 805 int ret; 806 807 __asm __volatile ( 808 #ifdef __powerpc64__ 809 "ldarx %0, 0, %3\n\t" /* load old value */ 810 "cmpld %4, %0\n\t" /* compare */ 811 "bne- 1f\n\t" /* exit if not equal */ 812 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 813 #else 814 "lwarx %0, 0, %3\n\t" /* load old value */ 815 "cmplw %4, %0\n\t" /* compare */ 816 "bne- 1f\n\t" /* exit if not equal */ 817 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 818 #endif 819 "bne- 1f\n\t" /* exit if failed */ 820 "li %0, 1\n\t" /* success - retval = 1 */ 821 "b 2f\n\t" /* we've succeeded */ 822 "1:\n\t" 823 #ifdef __powerpc64__ 824 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 825 "stdx %0, 0, %7\n\t" 826 #else 827 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 828 "stwx %0, 0, %7\n\t" 829 #endif 830 "li %0, 0\n\t" /* failure - retval = 0 */ 831 "2:\n\t" 832 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 833 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 834 : "cr0", "memory"); 835 836 return (ret); 837 } 838 839 #define ATOMIC_FCMPSET_ACQ_REL(type) \ 840 static __inline int \ 841 atomic_fcmpset_acq_##type(volatile u_##type *p, \ 842 u_##type *cmpval, u_##type newval)\ 843 {\ 844 u_##type retval; \ 845 retval = atomic_fcmpset_##type(p, cmpval, newval);\ 846 __ATOMIC_ACQ();\ 847 return (retval);\ 848 }\ 849 static __inline int \ 850 atomic_fcmpset_rel_##type(volatile u_##type *p, \ 851 u_##type *cmpval, u_##type newval)\ 852 {\ 853 __ATOMIC_REL();\ 854 return (atomic_fcmpset_##type(p, cmpval, newval));\ 855 }\ 856 struct hack 857 858 ATOMIC_FCMPSET_ACQ_REL(int); 859 ATOMIC_FCMPSET_ACQ_REL(long); 860 861 #define atomic_fcmpset_8 atomic_fcmpset_char 862 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char 863 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char 864 865 #define atomic_fcmpset_16 atomic_fcmpset_short 866 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short 867 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short 868 869 #define atomic_fcmpset_32 atomic_fcmpset_int 870 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 871 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 872 873 #ifdef __powerpc64__ 874 #define atomic_fcmpset_64 atomic_fcmpset_long 875 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long 876 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long 877 878 #define atomic_fcmpset_ptr atomic_fcmpset_long 879 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 880 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 881 #else 882 #define atomic_fcmpset_ptr atomic_fcmpset_int 883 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int 884 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int 885 #endif 886 887 static __inline u_int 888 atomic_fetchadd_int(volatile u_int *p, u_int v) 889 { 890 u_int value; 891 892 do { 893 value = *p; 894 } while (!atomic_cmpset_int(p, value, value + v)); 895 return (value); 896 } 897 898 static __inline u_long 899 atomic_fetchadd_long(volatile u_long *p, u_long v) 900 { 901 u_long value; 902 903 do { 904 value = *p; 905 } while (!atomic_cmpset_long(p, value, value + v)); 906 return (value); 907 } 908 909 static __inline u_int 910 atomic_swap_32(volatile u_int *p, u_int v) 911 { 912 u_int prev; 913 914 __asm __volatile( 915 "1: lwarx %0,0,%2\n" 916 " stwcx. %3,0,%2\n" 917 " bne- 1b\n" 918 : "=&r" (prev), "+m" (*(volatile u_int *)p) 919 : "r" (p), "r" (v) 920 : "cr0", "memory"); 921 922 return (prev); 923 } 924 925 #ifdef __powerpc64__ 926 static __inline u_long 927 atomic_swap_64(volatile u_long *p, u_long v) 928 { 929 u_long prev; 930 931 __asm __volatile( 932 "1: ldarx %0,0,%2\n" 933 " stdcx. %3,0,%2\n" 934 " bne- 1b\n" 935 : "=&r" (prev), "+m" (*(volatile u_long *)p) 936 : "r" (p), "r" (v) 937 : "cr0", "memory"); 938 939 return (prev); 940 } 941 #endif 942 943 #define atomic_fetchadd_32 atomic_fetchadd_int 944 #define atomic_swap_int atomic_swap_32 945 946 #ifdef __powerpc64__ 947 #define atomic_fetchadd_64 atomic_fetchadd_long 948 #define atomic_swap_long atomic_swap_64 949 #define atomic_swap_ptr atomic_swap_64 950 #else 951 #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) 952 #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) 953 #endif 954 955 static __inline void 956 atomic_thread_fence_acq(void) 957 { 958 959 powerpc_lwsync(); 960 } 961 962 static __inline void 963 atomic_thread_fence_rel(void) 964 { 965 966 powerpc_lwsync(); 967 } 968 969 static __inline void 970 atomic_thread_fence_acq_rel(void) 971 { 972 973 powerpc_lwsync(); 974 } 975 976 static __inline void 977 atomic_thread_fence_seq_cst(void) 978 { 979 980 __asm __volatile("sync" : : : "memory"); 981 } 982 983 #ifndef ISA_206_ATOMICS 984 #include <sys/_atomic_subword.h> 985 #endif 986 987 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ 988 ATOMIC_CMPSET_ACQ_REL(char); 989 ATOMIC_CMPSET_ACQ_REL(short); 990 991 ATOMIC_FCMPSET_ACQ_REL(char); 992 ATOMIC_FCMPSET_ACQ_REL(short); 993 994 #undef __ATOMIC_REL 995 #undef __ATOMIC_ACQ 996 997 #endif /* ! _MACHINE_ATOMIC_H_ */ 998