1 /*- 2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #ifndef _MACHINE_ATOMIC_H_ 30 #define _MACHINE_ATOMIC_H_ 31 32 #define isb() __asm __volatile("isb" : : : "memory") 33 34 /* 35 * Options for DMB and DSB: 36 * oshld Outer Shareable, load 37 * oshst Outer Shareable, store 38 * osh Outer Shareable, all 39 * nshld Non-shareable, load 40 * nshst Non-shareable, store 41 * nsh Non-shareable, all 42 * ishld Inner Shareable, load 43 * ishst Inner Shareable, store 44 * ish Inner Shareable, all 45 * ld Full system, load 46 * st Full system, store 47 * sy Full system, all 48 */ 49 #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") 50 #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") 51 52 #define mb() dmb(sy) /* Full system memory barrier all */ 53 #define wmb() dmb(st) /* Full system memory barrier store */ 54 #define rmb() dmb(ld) /* Full system memory barrier load */ 55 56 #if defined(KCSAN) && !defined(KCSAN_RUNTIME) 57 #include <sys/_cscan_atomic.h> 58 #else 59 60 #include <sys/atomic_common.h> 61 62 #define ATOMIC_OP(op, asm_op, bar, a, l) \ 63 static __inline void \ 64 atomic_##op##_##bar##8(volatile uint8_t *p, uint8_t val) \ 65 { \ 66 uint8_t tmp; \ 67 int res; \ 68 \ 69 __asm __volatile( \ 70 "1: ld"#a"xrb %w0, [%2] \n" \ 71 " "#asm_op" %w0, %w0, %w3 \n" \ 72 " st"#l"xrb %w1, %w0, [%2] \n" \ 73 " cbnz %w1, 1b \n" \ 74 : "=&r"(tmp), "=&r"(res) \ 75 : "r" (p), "r" (val) \ 76 : "memory" \ 77 ); \ 78 } \ 79 \ 80 static __inline void \ 81 atomic_##op##_##bar##16(volatile uint16_t *p, uint16_t val) \ 82 { \ 83 uint16_t tmp; \ 84 int res; \ 85 \ 86 __asm __volatile( \ 87 "1: ld"#a"xrh %w0, [%2] \n" \ 88 " "#asm_op" %w0, %w0, %w3 \n" \ 89 " st"#l"xrh %w1, %w0, [%2] \n" \ 90 " cbnz %w1, 1b \n" \ 91 : "=&r"(tmp), "=&r"(res) \ 92 : "r" (p), "r" (val) \ 93 : "memory" \ 94 ); \ 95 } \ 96 \ 97 static __inline void \ 98 atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val) \ 99 { \ 100 uint32_t tmp; \ 101 int res; \ 102 \ 103 __asm __volatile( \ 104 "1: ld"#a"xr %w0, [%2] \n" \ 105 " "#asm_op" %w0, %w0, %w3 \n" \ 106 " st"#l"xr %w1, %w0, [%2] \n" \ 107 " cbnz %w1, 1b \n" \ 108 : "=&r"(tmp), "=&r"(res) \ 109 : "r" (p), "r" (val) \ 110 : "memory" \ 111 ); \ 112 } \ 113 \ 114 static __inline void \ 115 atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val) \ 116 { \ 117 uint64_t tmp; \ 118 int res; \ 119 \ 120 __asm __volatile( \ 121 "1: ld"#a"xr %0, [%2] \n" \ 122 " "#asm_op" %0, %0, %3 \n" \ 123 " st"#l"xr %w1, %0, [%2] \n" \ 124 " cbnz %w1, 1b \n" \ 125 : "=&r"(tmp), "=&r"(res) \ 126 : "r" (p), "r" (val) \ 127 : "memory" \ 128 ); \ 129 } 130 131 #define ATOMIC(op, asm_op) \ 132 ATOMIC_OP(op, asm_op, , , ) \ 133 ATOMIC_OP(op, asm_op, acq_, a, ) \ 134 ATOMIC_OP(op, asm_op, rel_, , l) \ 135 136 ATOMIC(add, add) 137 ATOMIC(clear, bic) 138 ATOMIC(set, orr) 139 ATOMIC(subtract, sub) 140 141 #define ATOMIC_FCMPSET(bar, a, l) \ 142 static __inline int \ 143 atomic_fcmpset_##bar##8(volatile uint8_t *p, uint8_t *cmpval, \ 144 uint8_t newval) \ 145 { \ 146 uint8_t tmp; \ 147 uint8_t _cmpval = *cmpval; \ 148 int res; \ 149 \ 150 __asm __volatile( \ 151 "1: mov %w1, #1 \n" \ 152 " ld"#a"xrb %w0, [%2] \n" \ 153 " cmp %w0, %w3 \n" \ 154 " b.ne 2f \n" \ 155 " st"#l"xrb %w1, %w4, [%2]\n" \ 156 "2:" \ 157 : "=&r"(tmp), "=&r"(res) \ 158 : "r" (p), "r" (_cmpval), "r" (newval) \ 159 : "cc", "memory" \ 160 ); \ 161 *cmpval = tmp; \ 162 \ 163 return (!res); \ 164 } \ 165 \ 166 static __inline int \ 167 atomic_fcmpset_##bar##16(volatile uint16_t *p, uint16_t *cmpval, \ 168 uint16_t newval) \ 169 { \ 170 uint16_t tmp; \ 171 uint16_t _cmpval = *cmpval; \ 172 int res; \ 173 \ 174 __asm __volatile( \ 175 "1: mov %w1, #1 \n" \ 176 " ld"#a"xrh %w0, [%2] \n" \ 177 " cmp %w0, %w3 \n" \ 178 " b.ne 2f \n" \ 179 " st"#l"xrh %w1, %w4, [%2] \n" \ 180 "2:" \ 181 : "=&r"(tmp), "=&r"(res) \ 182 : "r" (p), "r" (_cmpval), "r" (newval) \ 183 : "cc", "memory" \ 184 ); \ 185 *cmpval = tmp; \ 186 \ 187 return (!res); \ 188 } \ 189 \ 190 static __inline int \ 191 atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval, \ 192 uint32_t newval) \ 193 { \ 194 uint32_t tmp; \ 195 uint32_t _cmpval = *cmpval; \ 196 int res; \ 197 \ 198 __asm __volatile( \ 199 "1: mov %w1, #1 \n" \ 200 " ld"#a"xr %w0, [%2] \n" \ 201 " cmp %w0, %w3 \n" \ 202 " b.ne 2f \n" \ 203 " st"#l"xr %w1, %w4, [%2] \n" \ 204 "2:" \ 205 : "=&r"(tmp), "=&r"(res) \ 206 : "r" (p), "r" (_cmpval), "r" (newval) \ 207 : "cc", "memory" \ 208 ); \ 209 *cmpval = tmp; \ 210 \ 211 return (!res); \ 212 } \ 213 \ 214 static __inline int \ 215 atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval, \ 216 uint64_t newval) \ 217 { \ 218 uint64_t tmp; \ 219 uint64_t _cmpval = *cmpval; \ 220 int res; \ 221 \ 222 __asm __volatile( \ 223 "1: mov %w1, #1 \n" \ 224 " ld"#a"xr %0, [%2] \n" \ 225 " cmp %0, %3 \n" \ 226 " b.ne 2f \n" \ 227 " st"#l"xr %w1, %4, [%2] \n" \ 228 "2:" \ 229 : "=&r"(tmp), "=&r"(res) \ 230 : "r" (p), "r" (_cmpval), "r" (newval) \ 231 : "cc", "memory" \ 232 ); \ 233 *cmpval = tmp; \ 234 \ 235 return (!res); \ 236 } 237 238 ATOMIC_FCMPSET( , , ) 239 ATOMIC_FCMPSET(acq_, a, ) 240 ATOMIC_FCMPSET(rel_, ,l) 241 242 #undef ATOMIC_FCMPSET 243 244 #define ATOMIC_CMPSET(bar, a, l) \ 245 static __inline int \ 246 atomic_cmpset_##bar##8(volatile uint8_t *p, uint8_t cmpval, \ 247 uint8_t newval) \ 248 { \ 249 uint8_t tmp; \ 250 int res; \ 251 \ 252 __asm __volatile( \ 253 "1: mov %w1, #1 \n" \ 254 " ld"#a"xrb %w0, [%2] \n" \ 255 " cmp %w0, %w3 \n" \ 256 " b.ne 2f \n" \ 257 " st"#l"xrb %w1, %w4, [%2] \n" \ 258 " cbnz %w1, 1b \n" \ 259 "2:" \ 260 : "=&r"(tmp), "=&r"(res) \ 261 : "r" (p), "r" (cmpval), "r" (newval) \ 262 : "cc", "memory" \ 263 ); \ 264 \ 265 return (!res); \ 266 } \ 267 \ 268 static __inline int \ 269 atomic_cmpset_##bar##16(volatile uint16_t *p, uint16_t cmpval, \ 270 uint16_t newval) \ 271 { \ 272 uint16_t tmp; \ 273 int res; \ 274 \ 275 __asm __volatile( \ 276 "1: mov %w1, #1 \n" \ 277 " ld"#a"xrh %w0, [%2] \n" \ 278 " cmp %w0, %w3 \n" \ 279 " b.ne 2f \n" \ 280 " st"#l"xrh %w1, %w4, [%2] \n" \ 281 " cbnz %w1, 1b \n" \ 282 "2:" \ 283 : "=&r"(tmp), "=&r"(res) \ 284 : "r" (p), "r" (cmpval), "r" (newval) \ 285 : "cc", "memory" \ 286 ); \ 287 \ 288 return (!res); \ 289 } \ 290 \ 291 static __inline int \ 292 atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \ 293 uint32_t newval) \ 294 { \ 295 uint32_t tmp; \ 296 int res; \ 297 \ 298 __asm __volatile( \ 299 "1: mov %w1, #1 \n" \ 300 " ld"#a"xr %w0, [%2] \n" \ 301 " cmp %w0, %w3 \n" \ 302 " b.ne 2f \n" \ 303 " st"#l"xr %w1, %w4, [%2] \n" \ 304 " cbnz %w1, 1b \n" \ 305 "2:" \ 306 : "=&r"(tmp), "=&r"(res) \ 307 : "r" (p), "r" (cmpval), "r" (newval) \ 308 : "cc", "memory" \ 309 ); \ 310 \ 311 return (!res); \ 312 } \ 313 \ 314 static __inline int \ 315 atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval, \ 316 uint64_t newval) \ 317 { \ 318 uint64_t tmp; \ 319 int res; \ 320 \ 321 __asm __volatile( \ 322 "1: mov %w1, #1 \n" \ 323 " ld"#a"xr %0, [%2] \n" \ 324 " cmp %0, %3 \n" \ 325 " b.ne 2f \n" \ 326 " st"#l"xr %w1, %4, [%2] \n" \ 327 " cbnz %w1, 1b \n" \ 328 "2:" \ 329 : "=&r"(tmp), "=&r"(res) \ 330 : "r" (p), "r" (cmpval), "r" (newval) \ 331 : "cc", "memory" \ 332 ); \ 333 \ 334 return (!res); \ 335 } 336 337 ATOMIC_CMPSET( , , ) 338 ATOMIC_CMPSET(acq_, a, ) 339 ATOMIC_CMPSET(rel_, ,l) 340 341 static __inline uint32_t 342 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 343 { 344 uint32_t tmp, ret; 345 int res; 346 347 __asm __volatile( 348 "1: ldxr %w2, [%3] \n" 349 " add %w0, %w2, %w4 \n" 350 " stxr %w1, %w0, [%3] \n" 351 " cbnz %w1, 1b \n" 352 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 353 : "r" (p), "r" (val) 354 : "memory" 355 ); 356 357 return (ret); 358 } 359 360 static __inline uint64_t 361 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 362 { 363 uint64_t tmp, ret; 364 int res; 365 366 __asm __volatile( 367 "1: ldxr %2, [%3] \n" 368 " add %0, %2, %4 \n" 369 " stxr %w1, %0, [%3] \n" 370 " cbnz %w1, 1b \n" 371 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 372 : "r" (p), "r" (val) 373 : "memory" 374 ); 375 376 return (ret); 377 } 378 379 static __inline uint32_t 380 atomic_readandclear_32(volatile uint32_t *p) 381 { 382 uint32_t ret; 383 int res; 384 385 __asm __volatile( 386 "1: ldxr %w1, [%2] \n" 387 " stxr %w0, wzr, [%2] \n" 388 " cbnz %w0, 1b \n" 389 : "=&r"(res), "=&r"(ret) 390 : "r" (p) 391 : "memory" 392 ); 393 394 return (ret); 395 } 396 397 static __inline uint64_t 398 atomic_readandclear_64(volatile uint64_t *p) 399 { 400 uint64_t ret; 401 int res; 402 403 __asm __volatile( 404 "1: ldxr %1, [%2] \n" 405 " stxr %w0, xzr, [%2] \n" 406 " cbnz %w0, 1b \n" 407 : "=&r"(res), "=&r"(ret) 408 : "r" (p) 409 : "memory" 410 ); 411 412 return (ret); 413 } 414 415 static __inline uint32_t 416 atomic_swap_32(volatile uint32_t *p, uint32_t val) 417 { 418 uint32_t ret; 419 int res; 420 421 __asm __volatile( 422 "1: ldxr %w0, [%2] \n" 423 " stxr %w1, %w3, [%2] \n" 424 " cbnz %w1, 1b \n" 425 : "=&r"(ret), "=&r"(res) 426 : "r" (p), "r" (val) 427 : "memory" 428 ); 429 430 return (ret); 431 } 432 433 static __inline uint64_t 434 atomic_swap_64(volatile uint64_t *p, uint64_t val) 435 { 436 uint64_t ret; 437 int res; 438 439 __asm __volatile( 440 "1: ldxr %0, [%2] \n" 441 " stxr %w1, %3, [%2] \n" 442 " cbnz %w1, 1b \n" 443 : "=&r"(ret), "=&r"(res) 444 : "r" (p), "r" (val) 445 : "memory" 446 ); 447 448 return (ret); 449 } 450 451 static __inline uint8_t 452 atomic_load_acq_8(volatile uint8_t *p) 453 { 454 uint8_t ret; 455 456 __asm __volatile( 457 "ldarb %w0, [%1] \n" 458 : "=&r" (ret) 459 : "r" (p) 460 : "memory"); 461 462 return (ret); 463 } 464 465 static __inline uint16_t 466 atomic_load_acq_16(volatile uint16_t *p) 467 { 468 uint16_t ret; 469 470 __asm __volatile( 471 "ldarh %w0, [%1] \n" 472 : "=&r" (ret) 473 : "r" (p) 474 : "memory"); 475 476 return (ret); 477 } 478 479 static __inline uint32_t 480 atomic_load_acq_32(volatile uint32_t *p) 481 { 482 uint32_t ret; 483 484 __asm __volatile( 485 "ldar %w0, [%1] \n" 486 : "=&r" (ret) 487 : "r" (p) 488 : "memory"); 489 490 return (ret); 491 } 492 493 static __inline uint64_t 494 atomic_load_acq_64(volatile uint64_t *p) 495 { 496 uint64_t ret; 497 498 __asm __volatile( 499 "ldar %0, [%1] \n" 500 : "=&r" (ret) 501 : "r" (p) 502 : "memory"); 503 504 return (ret); 505 } 506 507 static __inline void 508 atomic_store_rel_8(volatile uint8_t *p, uint8_t val) 509 { 510 511 __asm __volatile( 512 "stlrb %w0, [%1] \n" 513 : 514 : "r" (val), "r" (p) 515 : "memory"); 516 } 517 518 static __inline void 519 atomic_store_rel_16(volatile uint16_t *p, uint16_t val) 520 { 521 522 __asm __volatile( 523 "stlrh %w0, [%1] \n" 524 : 525 : "r" (val), "r" (p) 526 : "memory"); 527 } 528 529 static __inline void 530 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 531 { 532 533 __asm __volatile( 534 "stlr %w0, [%1] \n" 535 : 536 : "r" (val), "r" (p) 537 : "memory"); 538 } 539 540 static __inline void 541 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 542 { 543 544 __asm __volatile( 545 "stlr %0, [%1] \n" 546 : 547 : "r" (val), "r" (p) 548 : "memory"); 549 } 550 551 static __inline int 552 atomic_testandclear_32(volatile uint32_t *p, u_int val) 553 { 554 uint32_t mask, old, tmp; 555 int res; 556 557 mask = 1u << (val & 0x1f); 558 __asm __volatile( 559 "1: ldxr %w2, [%3] \n" 560 " bic %w0, %w2, %w4 \n" 561 " stxr %w1, %w0, [%3] \n" 562 " cbnz %w1, 1b \n" 563 : "=&r"(tmp), "=&r"(res), "=&r"(old) 564 : "r" (p), "r" (mask) 565 : "memory" 566 ); 567 568 return ((old & mask) != 0); 569 } 570 571 static __inline int 572 atomic_testandclear_64(volatile uint64_t *p, u_int val) 573 { 574 uint64_t mask, old, tmp; 575 int res; 576 577 mask = 1ul << (val & 0x1f); 578 __asm __volatile( 579 "1: ldxr %2, [%3] \n" 580 " bic %0, %2, %4 \n" 581 " stxr %w1, %0, [%3] \n" 582 " cbnz %w1, 1b \n" 583 : "=&r"(tmp), "=&r"(res), "=&r"(old) 584 : "r" (p), "r" (mask) 585 : "memory" 586 ); 587 588 return ((old & mask) != 0); 589 } 590 591 static __inline int 592 atomic_testandset_32(volatile uint32_t *p, u_int val) 593 { 594 uint32_t mask, old, tmp; 595 int res; 596 597 mask = 1u << (val & 0x1f); 598 __asm __volatile( 599 "1: ldxr %w2, [%3] \n" 600 " orr %w0, %w2, %w4 \n" 601 " stxr %w1, %w0, [%3] \n" 602 " cbnz %w1, 1b \n" 603 : "=&r"(tmp), "=&r"(res), "=&r"(old) 604 : "r" (p), "r" (mask) 605 : "memory" 606 ); 607 608 return ((old & mask) != 0); 609 } 610 611 static __inline int 612 atomic_testandset_64(volatile uint64_t *p, u_int val) 613 { 614 uint64_t mask, old, tmp; 615 int res; 616 617 mask = 1ul << (val & 0x1f); 618 __asm __volatile( 619 "1: ldxr %2, [%3] \n" 620 " orr %0, %2, %4 \n" 621 " stxr %w1, %0, [%3] \n" 622 " cbnz %w1, 1b \n" 623 : "=&r"(tmp), "=&r"(res), "=&r"(old) 624 : "r" (p), "r" (mask) 625 : "memory" 626 ); 627 628 return ((old & mask) != 0); 629 } 630 631 632 #define atomic_add_int atomic_add_32 633 #define atomic_fcmpset_int atomic_fcmpset_32 634 #define atomic_clear_int atomic_clear_32 635 #define atomic_cmpset_int atomic_cmpset_32 636 #define atomic_fetchadd_int atomic_fetchadd_32 637 #define atomic_readandclear_int atomic_readandclear_32 638 #define atomic_set_int atomic_set_32 639 #define atomic_swap_int atomic_swap_32 640 #define atomic_subtract_int atomic_subtract_32 641 #define atomic_testandclear_int atomic_testandclear_32 642 #define atomic_testandset_int atomic_testandset_32 643 644 #define atomic_add_acq_int atomic_add_acq_32 645 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 646 #define atomic_clear_acq_int atomic_clear_acq_32 647 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 648 #define atomic_load_acq_int atomic_load_acq_32 649 #define atomic_set_acq_int atomic_set_acq_32 650 #define atomic_subtract_acq_int atomic_subtract_acq_32 651 652 #define atomic_add_rel_int atomic_add_rel_32 653 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 654 #define atomic_clear_rel_int atomic_clear_rel_32 655 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 656 #define atomic_set_rel_int atomic_set_rel_32 657 #define atomic_subtract_rel_int atomic_subtract_rel_32 658 #define atomic_store_rel_int atomic_store_rel_32 659 660 #define atomic_add_long atomic_add_64 661 #define atomic_fcmpset_long atomic_fcmpset_64 662 #define atomic_clear_long atomic_clear_64 663 #define atomic_cmpset_long atomic_cmpset_64 664 #define atomic_fetchadd_long atomic_fetchadd_64 665 #define atomic_readandclear_long atomic_readandclear_64 666 #define atomic_set_long atomic_set_64 667 #define atomic_swap_long atomic_swap_64 668 #define atomic_subtract_long atomic_subtract_64 669 #define atomic_testandclear_long atomic_testandclear_64 670 #define atomic_testandset_long atomic_testandset_64 671 672 #define atomic_add_ptr atomic_add_64 673 #define atomic_fcmpset_ptr atomic_fcmpset_64 674 #define atomic_clear_ptr atomic_clear_64 675 #define atomic_cmpset_ptr atomic_cmpset_64 676 #define atomic_fetchadd_ptr atomic_fetchadd_64 677 #define atomic_readandclear_ptr atomic_readandclear_64 678 #define atomic_set_ptr atomic_set_64 679 #define atomic_swap_ptr atomic_swap_64 680 #define atomic_subtract_ptr atomic_subtract_64 681 682 #define atomic_add_acq_long atomic_add_acq_64 683 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 684 #define atomic_clear_acq_long atomic_clear_acq_64 685 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 686 #define atomic_load_acq_long atomic_load_acq_64 687 #define atomic_set_acq_long atomic_set_acq_64 688 #define atomic_subtract_acq_long atomic_subtract_acq_64 689 690 #define atomic_add_acq_ptr atomic_add_acq_64 691 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 692 #define atomic_clear_acq_ptr atomic_clear_acq_64 693 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 694 #define atomic_load_acq_ptr atomic_load_acq_64 695 #define atomic_set_acq_ptr atomic_set_acq_64 696 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 697 698 #define atomic_add_rel_long atomic_add_rel_64 699 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 700 #define atomic_clear_rel_long atomic_clear_rel_64 701 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 702 #define atomic_set_rel_long atomic_set_rel_64 703 #define atomic_subtract_rel_long atomic_subtract_rel_64 704 #define atomic_store_rel_long atomic_store_rel_64 705 706 #define atomic_add_rel_ptr atomic_add_rel_64 707 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 708 #define atomic_clear_rel_ptr atomic_clear_rel_64 709 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 710 #define atomic_set_rel_ptr atomic_set_rel_64 711 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 712 #define atomic_store_rel_ptr atomic_store_rel_64 713 714 static __inline void 715 atomic_thread_fence_acq(void) 716 { 717 718 dmb(ld); 719 } 720 721 static __inline void 722 atomic_thread_fence_rel(void) 723 { 724 725 dmb(sy); 726 } 727 728 static __inline void 729 atomic_thread_fence_acq_rel(void) 730 { 731 732 dmb(sy); 733 } 734 735 static __inline void 736 atomic_thread_fence_seq_cst(void) 737 { 738 739 dmb(sy); 740 } 741 742 #endif /* KCSAN && !KCSAN_RUNTIME */ 743 744 #endif /* _MACHINE_ATOMIC_H_ */ 745 746