1 /*- 2 * Copyright (c) 2008 Marcel Moolenaar 3 * Copyright (c) 2001 Benno Rice 4 * Copyright (c) 2001 David E. O'Brien 5 * Copyright (c) 1998 Doug Rabson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #ifndef _MACHINE_ATOMIC_H_ 33 #define _MACHINE_ATOMIC_H_ 34 35 #ifndef _SYS_CDEFS_H_ 36 #error this file needs sys/cdefs.h as a prerequisite 37 #endif 38 39 #define __ATOMIC_BARRIER \ 40 __asm __volatile("sync" : : : "memory") 41 42 #define mb() __ATOMIC_BARRIER 43 #define wmb() mb() 44 #define rmb() mb() 45 46 /* 47 * atomic_add(p, v) 48 * { *p += v; } 49 */ 50 51 #define __atomic_add_int(p, v, t) \ 52 __asm __volatile( \ 53 "1: lwarx %0, 0, %2\n" \ 54 " add %0, %3, %0\n" \ 55 " stwcx. %0, 0, %2\n" \ 56 " bne- 1b\n" \ 57 : "=&r" (t), "=m" (*p) \ 58 : "r" (p), "r" (v), "m" (*p) \ 59 : "cc", "memory") \ 60 /* __atomic_add_int */ 61 62 #ifdef __powerpc64__ 63 #define __atomic_add_long(p, v, t) \ 64 __asm __volatile( \ 65 "1: ldarx %0, 0, %2\n" \ 66 " add %0, %3, %0\n" \ 67 " stdcx. %0, 0, %2\n" \ 68 " bne- 1b\n" \ 69 : "=&r" (t), "=m" (*p) \ 70 : "r" (p), "r" (v), "m" (*p) \ 71 : "cc", "memory") \ 72 /* __atomic_add_long */ 73 #else 74 #define __atomic_add_long(p, v, t) \ 75 __asm __volatile( \ 76 "1: lwarx %0, 0, %2\n" \ 77 " add %0, %3, %0\n" \ 78 " stwcx. %0, 0, %2\n" \ 79 " bne- 1b\n" \ 80 : "=&r" (t), "=m" (*p) \ 81 : "r" (p), "r" (v), "m" (*p) \ 82 : "cc", "memory") \ 83 /* __atomic_add_long */ 84 #endif 85 86 #define _ATOMIC_ADD(type) \ 87 static __inline void \ 88 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 89 u_##type t; \ 90 __atomic_add_##type(p, v, t); \ 91 } \ 92 \ 93 static __inline void \ 94 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 95 u_##type t; \ 96 __atomic_add_##type(p, v, t); \ 97 __ATOMIC_BARRIER; \ 98 } \ 99 \ 100 static __inline void \ 101 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 102 u_##type t; \ 103 __ATOMIC_BARRIER; \ 104 __atomic_add_##type(p, v, t); \ 105 } \ 106 /* _ATOMIC_ADD */ 107 108 _ATOMIC_ADD(int) 109 _ATOMIC_ADD(long) 110 111 #define atomic_add_32 atomic_add_int 112 #define atomic_add_acq_32 atomic_add_acq_int 113 #define atomic_add_rel_32 atomic_add_rel_int 114 115 #ifdef __powerpc64__ 116 #define atomic_add_64 atomic_add_long 117 #define atomic_add_acq_64 atomic_add_acq_long 118 #define atomic_add_rel_64 atomic_add_rel_long 119 120 #define atomic_add_ptr atomic_add_long 121 #define atomic_add_acq_ptr atomic_add_acq_long 122 #define atomic_add_rel_ptr atomic_add_rel_long 123 #else 124 #define atomic_add_ptr atomic_add_int 125 #define atomic_add_acq_ptr atomic_add_acq_int 126 #define atomic_add_rel_ptr atomic_add_rel_int 127 #endif 128 #undef _ATOMIC_ADD 129 #undef __atomic_add_long 130 #undef __atomic_add_int 131 132 /* 133 * atomic_clear(p, v) 134 * { *p &= ~v; } 135 */ 136 137 #define __atomic_clear_int(p, v, t) \ 138 __asm __volatile( \ 139 "1: lwarx %0, 0, %2\n" \ 140 " andc %0, %0, %3\n" \ 141 " stwcx. %0, 0, %2\n" \ 142 " bne- 1b\n" \ 143 : "=&r" (t), "=m" (*p) \ 144 : "r" (p), "r" (v), "m" (*p) \ 145 : "cc", "memory") \ 146 /* __atomic_clear_int */ 147 148 #ifdef __powerpc64__ 149 #define __atomic_clear_long(p, v, t) \ 150 __asm __volatile( \ 151 "1: ldarx %0, 0, %2\n" \ 152 " andc %0, %0, %3\n" \ 153 " stdcx. %0, 0, %2\n" \ 154 " bne- 1b\n" \ 155 : "=&r" (t), "=m" (*p) \ 156 : "r" (p), "r" (v), "m" (*p) \ 157 : "cc", "memory") \ 158 /* __atomic_clear_long */ 159 #else 160 #define __atomic_clear_long(p, v, t) \ 161 __asm __volatile( \ 162 "1: lwarx %0, 0, %2\n" \ 163 " andc %0, %0, %3\n" \ 164 " stwcx. %0, 0, %2\n" \ 165 " bne- 1b\n" \ 166 : "=&r" (t), "=m" (*p) \ 167 : "r" (p), "r" (v), "m" (*p) \ 168 : "cc", "memory") \ 169 /* __atomic_clear_long */ 170 #endif 171 172 #define _ATOMIC_CLEAR(type) \ 173 static __inline void \ 174 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 175 u_##type t; \ 176 __atomic_clear_##type(p, v, t); \ 177 } \ 178 \ 179 static __inline void \ 180 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 181 u_##type t; \ 182 __atomic_clear_##type(p, v, t); \ 183 __ATOMIC_BARRIER; \ 184 } \ 185 \ 186 static __inline void \ 187 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 188 u_##type t; \ 189 __ATOMIC_BARRIER; \ 190 __atomic_clear_##type(p, v, t); \ 191 } \ 192 /* _ATOMIC_CLEAR */ 193 194 195 _ATOMIC_CLEAR(int) 196 _ATOMIC_CLEAR(long) 197 198 #define atomic_clear_32 atomic_clear_int 199 #define atomic_clear_acq_32 atomic_clear_acq_int 200 #define atomic_clear_rel_32 atomic_clear_rel_int 201 202 #ifdef __powerpc64__ 203 #define atomic_clear_64 atomic_clear_long 204 #define atomic_clear_acq_64 atomic_clear_acq_long 205 #define atomic_clear_rel_64 atomic_clear_rel_long 206 207 #define atomic_clear_ptr atomic_clear_long 208 #define atomic_clear_acq_ptr atomic_clear_acq_long 209 #define atomic_clear_rel_ptr atomic_clear_rel_long 210 #else 211 #define atomic_clear_ptr atomic_clear_int 212 #define atomic_clear_acq_ptr atomic_clear_acq_int 213 #define atomic_clear_rel_ptr atomic_clear_rel_int 214 #endif 215 #undef _ATOMIC_CLEAR 216 #undef __atomic_clear_long 217 #undef __atomic_clear_int 218 219 /* 220 * atomic_cmpset(p, o, n) 221 */ 222 /* TODO -- see below */ 223 224 /* 225 * atomic_load_acq(p) 226 */ 227 /* TODO -- see below */ 228 229 /* 230 * atomic_readandclear(p) 231 */ 232 /* TODO -- see below */ 233 234 /* 235 * atomic_set(p, v) 236 * { *p |= v; } 237 */ 238 239 #define __atomic_set_int(p, v, t) \ 240 __asm __volatile( \ 241 "1: lwarx %0, 0, %2\n" \ 242 " or %0, %3, %0\n" \ 243 " stwcx. %0, 0, %2\n" \ 244 " bne- 1b\n" \ 245 : "=&r" (t), "=m" (*p) \ 246 : "r" (p), "r" (v), "m" (*p) \ 247 : "cc", "memory") \ 248 /* __atomic_set_int */ 249 250 #ifdef __powerpc64__ 251 #define __atomic_set_long(p, v, t) \ 252 __asm __volatile( \ 253 "1: ldarx %0, 0, %2\n" \ 254 " or %0, %3, %0\n" \ 255 " stdcx. %0, 0, %2\n" \ 256 " bne- 1b\n" \ 257 : "=&r" (t), "=m" (*p) \ 258 : "r" (p), "r" (v), "m" (*p) \ 259 : "cc", "memory") \ 260 /* __atomic_set_long */ 261 #else 262 #define __atomic_set_long(p, v, t) \ 263 __asm __volatile( \ 264 "1: lwarx %0, 0, %2\n" \ 265 " or %0, %3, %0\n" \ 266 " stwcx. %0, 0, %2\n" \ 267 " bne- 1b\n" \ 268 : "=&r" (t), "=m" (*p) \ 269 : "r" (p), "r" (v), "m" (*p) \ 270 : "cc", "memory") \ 271 /* __atomic_set_long */ 272 #endif 273 274 #define _ATOMIC_SET(type) \ 275 static __inline void \ 276 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 277 u_##type t; \ 278 __atomic_set_##type(p, v, t); \ 279 } \ 280 \ 281 static __inline void \ 282 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 283 u_##type t; \ 284 __atomic_set_##type(p, v, t); \ 285 __ATOMIC_BARRIER; \ 286 } \ 287 \ 288 static __inline void \ 289 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 290 u_##type t; \ 291 __ATOMIC_BARRIER; \ 292 __atomic_set_##type(p, v, t); \ 293 } \ 294 /* _ATOMIC_SET */ 295 296 _ATOMIC_SET(int) 297 _ATOMIC_SET(long) 298 299 #define atomic_set_32 atomic_set_int 300 #define atomic_set_acq_32 atomic_set_acq_int 301 #define atomic_set_rel_32 atomic_set_rel_int 302 303 #ifdef __powerpc64__ 304 #define atomic_set_64 atomic_set_long 305 #define atomic_set_acq_64 atomic_set_acq_long 306 #define atomic_set_rel_64 atomic_set_rel_long 307 308 #define atomic_set_ptr atomic_set_long 309 #define atomic_set_acq_ptr atomic_set_acq_long 310 #define atomic_set_rel_ptr atomic_set_rel_long 311 #else 312 #define atomic_set_ptr atomic_set_int 313 #define atomic_set_acq_ptr atomic_set_acq_int 314 #define atomic_set_rel_ptr atomic_set_rel_int 315 #endif 316 #undef _ATOMIC_SET 317 #undef __atomic_set_long 318 #undef __atomic_set_int 319 320 /* 321 * atomic_subtract(p, v) 322 * { *p -= v; } 323 */ 324 325 #define __atomic_subtract_int(p, v, t) \ 326 __asm __volatile( \ 327 "1: lwarx %0, 0, %2\n" \ 328 " subf %0, %3, %0\n" \ 329 " stwcx. %0, 0, %2\n" \ 330 " bne- 1b\n" \ 331 : "=&r" (t), "=m" (*p) \ 332 : "r" (p), "r" (v), "m" (*p) \ 333 : "cc", "memory") \ 334 /* __atomic_subtract_int */ 335 336 #ifdef __powerpc64__ 337 #define __atomic_subtract_long(p, v, t) \ 338 __asm __volatile( \ 339 "1: ldarx %0, 0, %2\n" \ 340 " subf %0, %3, %0\n" \ 341 " stdcx. %0, 0, %2\n" \ 342 " bne- 1b\n" \ 343 : "=&r" (t), "=m" (*p) \ 344 : "r" (p), "r" (v), "m" (*p) \ 345 : "cc", "memory") \ 346 /* __atomic_subtract_long */ 347 #else 348 #define __atomic_subtract_long(p, v, t) \ 349 __asm __volatile( \ 350 "1: lwarx %0, 0, %2\n" \ 351 " subf %0, %3, %0\n" \ 352 " stwcx. %0, 0, %2\n" \ 353 " bne- 1b\n" \ 354 : "=&r" (t), "=m" (*p) \ 355 : "r" (p), "r" (v), "m" (*p) \ 356 : "cc", "memory") \ 357 /* __atomic_subtract_long */ 358 #endif 359 360 #define _ATOMIC_SUBTRACT(type) \ 361 static __inline void \ 362 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 363 u_##type t; \ 364 __atomic_subtract_##type(p, v, t); \ 365 } \ 366 \ 367 static __inline void \ 368 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 369 u_##type t; \ 370 __atomic_subtract_##type(p, v, t); \ 371 __ATOMIC_BARRIER; \ 372 } \ 373 \ 374 static __inline void \ 375 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 376 u_##type t; \ 377 __ATOMIC_BARRIER; \ 378 __atomic_subtract_##type(p, v, t); \ 379 } \ 380 /* _ATOMIC_SUBTRACT */ 381 382 _ATOMIC_SUBTRACT(int) 383 _ATOMIC_SUBTRACT(long) 384 385 #define atomic_subtract_32 atomic_subtract_int 386 #define atomic_subtract_acq_32 atomic_subtract_acq_int 387 #define atomic_subtract_rel_32 atomic_subtract_rel_int 388 389 #ifdef __powerpc64__ 390 #define atomic_subtract_64 atomic_subtract_long 391 #define atomic_subtract_acq_64 atomic_subract_acq_long 392 #define atomic_subtract_rel_64 atomic_subtract_rel_long 393 394 #define atomic_subtract_ptr atomic_subtract_long 395 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 396 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 397 #else 398 #define atomic_subtract_ptr atomic_subtract_int 399 #define atomic_subtract_acq_ptr atomic_subtract_acq_int 400 #define atomic_subtract_rel_ptr atomic_subtract_rel_int 401 #endif 402 #undef _ATOMIC_SUBTRACT 403 #undef __atomic_subtract_long 404 #undef __atomic_subtract_int 405 406 /* 407 * atomic_store_rel(p, v) 408 */ 409 /* TODO -- see below */ 410 411 /* 412 * Old/original implementations that still need revisiting. 413 */ 414 415 static __inline u_int 416 atomic_readandclear_int(volatile u_int *addr) 417 { 418 u_int result,temp; 419 420 #ifdef __GNUCLIKE_ASM 421 __asm __volatile ( 422 "\tsync\n" /* drain writes */ 423 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 424 "li %1, 0\n\t" /* load new value */ 425 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 426 "bne- 1b\n\t" /* spin if failed */ 427 : "=&r"(result), "=&r"(temp), "=m" (*addr) 428 : "r" (addr), "m" (*addr) 429 : "cc", "memory"); 430 #endif 431 432 return (result); 433 } 434 435 #ifdef __powerpc64__ 436 static __inline u_long 437 atomic_readandclear_long(volatile u_long *addr) 438 { 439 u_long result,temp; 440 441 #ifdef __GNUCLIKE_ASM 442 __asm __volatile ( 443 "\tsync\n" /* drain writes */ 444 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 445 "li %1, 0\n\t" /* load new value */ 446 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 447 "bne- 1b\n\t" /* spin if failed */ 448 : "=&r"(result), "=&r"(temp), "=m" (*addr) 449 : "r" (addr), "m" (*addr) 450 : "cc", "memory"); 451 #endif 452 453 return (result); 454 } 455 #endif 456 457 #define atomic_readandclear_32 atomic_readandclear_int 458 459 #ifdef __powerpc64__ 460 #define atomic_readandclear_64 atomic_readandclear_long 461 462 #define atomic_readandclear_ptr atomic_readandclear_long 463 #else 464 static __inline u_long 465 atomic_readandclear_long(volatile u_long *addr) 466 { 467 468 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 469 } 470 471 #define atomic_readandclear_ptr atomic_readandclear_int 472 #endif 473 474 /* 475 * We assume that a = b will do atomic loads and stores. 476 */ 477 #define ATOMIC_STORE_LOAD(TYPE) \ 478 static __inline u_##TYPE \ 479 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 480 { \ 481 u_##TYPE v; \ 482 \ 483 v = *p; \ 484 __ATOMIC_BARRIER; \ 485 return (v); \ 486 } \ 487 \ 488 static __inline void \ 489 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 490 { \ 491 __ATOMIC_BARRIER; \ 492 *p = v; \ 493 } 494 495 ATOMIC_STORE_LOAD(int) 496 497 #define atomic_load_acq_32 atomic_load_acq_int 498 #define atomic_store_rel_32 atomic_store_rel_int 499 500 #ifdef __powerpc64__ 501 ATOMIC_STORE_LOAD(long) 502 503 #define atomic_load_acq_64 atomic_load_acq_long 504 #define atomic_store_rel_64 atomic_store_rel_long 505 506 #define atomic_load_acq_ptr atomic_load_acq_long 507 #define atomic_store_rel_ptr atomic_store_rel_long 508 #else 509 static __inline u_long 510 atomic_load_acq_long(volatile u_long *addr) 511 { 512 513 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 514 } 515 516 static __inline void 517 atomic_store_rel_long(volatile u_long *addr, u_long val) 518 { 519 520 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 521 } 522 523 #define atomic_load_acq_ptr atomic_load_acq_int 524 #define atomic_store_rel_ptr atomic_store_rel_int 525 #endif 526 #undef ATOMIC_STORE_LOAD 527 528 /* 529 * Atomically compare the value stored at *p with cmpval and if the 530 * two values are equal, update the value of *p with newval. Returns 531 * zero if the compare failed, nonzero otherwise. 532 */ 533 static __inline int 534 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 535 { 536 int ret; 537 538 #ifdef __GNUCLIKE_ASM 539 __asm __volatile ( 540 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 541 "cmplw %3, %0\n\t" /* compare */ 542 "bne 2f\n\t" /* exit if not equal */ 543 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 544 "bne- 1b\n\t" /* spin if failed */ 545 "li %0, 1\n\t" /* success - retval = 1 */ 546 "b 3f\n\t" /* we've succeeded */ 547 "2:\n\t" 548 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 549 "li %0, 0\n\t" /* failure - retval = 0 */ 550 "3:\n\t" 551 : "=&r" (ret), "=m" (*p) 552 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 553 : "cc", "memory"); 554 #endif 555 556 return (ret); 557 } 558 static __inline int 559 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 560 { 561 int ret; 562 563 #ifdef __GNUCLIKE_ASM 564 __asm __volatile ( 565 #ifdef __powerpc64__ 566 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 567 "cmpld %3, %0\n\t" /* compare */ 568 "bne 2f\n\t" /* exit if not equal */ 569 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 570 #else 571 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 572 "cmplw %3, %0\n\t" /* compare */ 573 "bne 2f\n\t" /* exit if not equal */ 574 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 575 #endif 576 "bne- 1b\n\t" /* spin if failed */ 577 "li %0, 1\n\t" /* success - retval = 1 */ 578 "b 3f\n\t" /* we've succeeded */ 579 "2:\n\t" 580 #ifdef __powerpc64__ 581 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 582 #else 583 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 584 #endif 585 "li %0, 0\n\t" /* failure - retval = 0 */ 586 "3:\n\t" 587 : "=&r" (ret), "=m" (*p) 588 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 589 : "cc", "memory"); 590 #endif 591 592 return (ret); 593 } 594 595 static __inline int 596 atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) 597 { 598 int retval; 599 600 retval = atomic_cmpset_int(p, cmpval, newval); 601 __ATOMIC_BARRIER; 602 return (retval); 603 } 604 605 static __inline int 606 atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) 607 { 608 __ATOMIC_BARRIER; 609 return (atomic_cmpset_int(p, cmpval, newval)); 610 } 611 612 static __inline int 613 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) 614 { 615 u_long retval; 616 617 retval = atomic_cmpset_long(p, cmpval, newval); 618 __ATOMIC_BARRIER; 619 return (retval); 620 } 621 622 static __inline int 623 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) 624 { 625 __ATOMIC_BARRIER; 626 return (atomic_cmpset_long(p, cmpval, newval)); 627 } 628 629 #define atomic_cmpset_32 atomic_cmpset_int 630 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 631 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 632 633 #ifdef __powerpc64__ 634 #define atomic_cmpset_64 atomic_cmpset_long 635 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 636 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 637 638 #define atomic_cmpset_ptr atomic_cmpset_long 639 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 640 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 641 #else 642 #define atomic_cmpset_ptr atomic_cmpset_int 643 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 644 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 645 #endif 646 647 static __inline u_int 648 atomic_fetchadd_int(volatile u_int *p, u_int v) 649 { 650 u_int value; 651 652 do { 653 value = *p; 654 } while (!atomic_cmpset_int(p, value, value + v)); 655 return (value); 656 } 657 658 static __inline u_long 659 atomic_fetchadd_long(volatile u_long *p, u_long v) 660 { 661 u_long value; 662 663 do { 664 value = *p; 665 } while (!atomic_cmpset_long(p, value, value + v)); 666 return (value); 667 } 668 669 #define atomic_fetchadd_32 atomic_fetchadd_int 670 671 #ifdef __powerpc64__ 672 #define atomic_fetchadd_64 atomic_fetchadd_long 673 #endif 674 675 #endif /* ! _MACHINE_ATOMIC_H_ */ 676