1 /*- 2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #ifndef _MACHINE_ATOMIC_H_ 30 #define _MACHINE_ATOMIC_H_ 31 32 #define isb() __asm __volatile("isb" : : : "memory") 33 34 /* 35 * Options for DMB and DSB: 36 * oshld Outer Shareable, load 37 * oshst Outer Shareable, store 38 * osh Outer Shareable, all 39 * nshld Non-shareable, load 40 * nshst Non-shareable, store 41 * nsh Non-shareable, all 42 * ishld Inner Shareable, load 43 * ishst Inner Shareable, store 44 * ish Inner Shareable, all 45 * ld Full system, load 46 * st Full system, store 47 * sy Full system, all 48 */ 49 #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") 50 #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") 51 52 #define mb() dmb(sy) /* Full system memory barrier all */ 53 #define wmb() dmb(st) /* Full system memory barrier store */ 54 #define rmb() dmb(ld) /* Full system memory barrier load */ 55 56 #if defined(KCSAN) && !defined(KCSAN_RUNTIME) 57 #include <sys/_cscan_atomic.h> 58 #else 59 60 #include <sys/atomic_common.h> 61 62 #ifdef _KERNEL 63 extern bool lse_supported; 64 65 #ifdef LSE_ATOMICS 66 #define _ATOMIC_LSE_SUPPORTED 1 67 #else 68 #define _ATOMIC_LSE_SUPPORTED lse_supported 69 #endif 70 #else 71 #define _ATOMIC_LSE_SUPPORTED 0 72 #endif 73 74 #define _ATOMIC_OP_PROTO(t, op, bar, flav) \ 75 static __inline void \ 76 atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val) 77 78 #define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ 79 _ATOMIC_OP_PROTO(t, op, bar, _llsc) \ 80 { \ 81 uint##t##_t tmp; \ 82 int res; \ 83 \ 84 pre; \ 85 __asm __volatile( \ 86 "1: ld"#a"xr"#s" %"#w"0, [%2]\n" \ 87 " "#llsc_asm_op" %"#w"0, %"#w"0, %"#w"3\n" \ 88 " st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \ 89 " cbnz %w1, 1b\n" \ 90 : "=&r"(tmp), "=&r"(res) \ 91 : "r" (p), "r" (val) \ 92 : "memory" \ 93 ); \ 94 } \ 95 \ 96 _ATOMIC_OP_PROTO(t, op, bar, _lse) \ 97 { \ 98 uint##t##_t tmp; \ 99 \ 100 pre; \ 101 __asm __volatile( \ 102 ".arch_extension lse\n" \ 103 "ld"#lse_asm_op#a#l#s" %"#w"2, %"#w"0, [%1]\n" \ 104 ".arch_extension nolse\n" \ 105 : "=r" (tmp) \ 106 : "r" (p), "r" (val) \ 107 : "memory" \ 108 ); \ 109 } \ 110 \ 111 _ATOMIC_OP_PROTO(t, op, bar, ) \ 112 { \ 113 if (_ATOMIC_LSE_SUPPORTED) \ 114 atomic_##op##_##bar##t##_lse(p, val); \ 115 else \ 116 atomic_##op##_##bar##t##_llsc(p, val); \ 117 } 118 119 #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ 120 _ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \ 121 bar, a, l) \ 122 _ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \ 123 bar, a, l) \ 124 _ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \ 125 bar, a, l) \ 126 _ATOMIC_OP_IMPL(64, , , op, llsc_asm_op, lse_asm_op, pre, \ 127 bar, a, l) 128 129 #define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre) \ 130 __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, , , ) \ 131 __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a, ) \ 132 __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_, , l) 133 134 _ATOMIC_OP(add, add, add, ) 135 _ATOMIC_OP(clear, bic, clr, ) 136 _ATOMIC_OP(set, orr, set, ) 137 _ATOMIC_OP(subtract, add, add, val = -val) 138 139 #define _ATOMIC_CMPSET_PROTO(t, bar, flav) \ 140 static __inline int \ 141 atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \ 142 uint##t##_t cmpval, uint##t##_t newval) 143 144 #define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \ 145 static __inline int \ 146 atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p, \ 147 uint##t##_t *cmpval, uint##t##_t newval) 148 149 #define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \ 150 _ATOMIC_CMPSET_PROTO(t, bar, _llsc) \ 151 { \ 152 uint##t##_t tmp; \ 153 int res; \ 154 \ 155 __asm __volatile( \ 156 "1: mov %w1, #1\n" \ 157 " ld"#a"xr"#s" %"#w"0, [%2]\n" \ 158 " cmp %"#w"0, %"#w"3\n" \ 159 " b.ne 2f\n" \ 160 " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ 161 " cbnz %w1, 1b\n" \ 162 "2:" \ 163 : "=&r"(tmp), "=&r"(res) \ 164 : "r" (p), "r" (cmpval), "r" (newval) \ 165 : "cc", "memory" \ 166 ); \ 167 \ 168 return (!res); \ 169 } \ 170 \ 171 _ATOMIC_CMPSET_PROTO(t, bar, _lse) \ 172 { \ 173 uint##t##_t oldval; \ 174 int res; \ 175 \ 176 oldval = cmpval; \ 177 __asm __volatile( \ 178 ".arch_extension lse\n" \ 179 "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ 180 "cmp %"#w"1, %"#w"2\n" \ 181 "cset %w0, eq\n" \ 182 ".arch_extension nolse\n" \ 183 : "=r" (res), "+&r" (cmpval) \ 184 : "r" (oldval), "r" (p), "r" (newval) \ 185 : "cc", "memory" \ 186 ); \ 187 \ 188 return (res); \ 189 } \ 190 \ 191 _ATOMIC_CMPSET_PROTO(t, bar, ) \ 192 { \ 193 if (_ATOMIC_LSE_SUPPORTED) \ 194 return (atomic_cmpset_##bar##t##_lse(p, cmpval, \ 195 newval)); \ 196 else \ 197 return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \ 198 newval)); \ 199 } \ 200 \ 201 _ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \ 202 { \ 203 uint##t##_t _cmpval, tmp; \ 204 int res; \ 205 \ 206 _cmpval = *cmpval; \ 207 __asm __volatile( \ 208 " mov %w1, #1\n" \ 209 " ld"#a"xr"#s" %"#w"0, [%2]\n" \ 210 " cmp %"#w"0, %"#w"3\n" \ 211 " b.ne 1f\n" \ 212 " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ 213 "1:" \ 214 : "=&r"(tmp), "=&r"(res) \ 215 : "r" (p), "r" (_cmpval), "r" (newval) \ 216 : "cc", "memory" \ 217 ); \ 218 *cmpval = tmp; \ 219 \ 220 return (!res); \ 221 } \ 222 \ 223 _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \ 224 { \ 225 uint##t##_t _cmpval, tmp; \ 226 int res; \ 227 \ 228 _cmpval = tmp = *cmpval; \ 229 __asm __volatile( \ 230 ".arch_extension lse\n" \ 231 "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ 232 "cmp %"#w"1, %"#w"2\n" \ 233 "cset %w0, eq\n" \ 234 ".arch_extension nolse\n" \ 235 : "=r" (res), "+&r" (tmp) \ 236 : "r" (_cmpval), "r" (p), "r" (newval) \ 237 : "cc", "memory" \ 238 ); \ 239 *cmpval = tmp; \ 240 \ 241 return (res); \ 242 } \ 243 \ 244 _ATOMIC_FCMPSET_PROTO(t, bar, ) \ 245 { \ 246 if (_ATOMIC_LSE_SUPPORTED) \ 247 return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \ 248 newval)); \ 249 else \ 250 return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \ 251 newval)); \ 252 } 253 254 #define _ATOMIC_CMPSET(bar, a, l) \ 255 _ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \ 256 _ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \ 257 _ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \ 258 _ATOMIC_CMPSET_IMPL(64, , , bar, a, l) 259 260 #define atomic_cmpset_8 atomic_cmpset_8 261 #define atomic_fcmpset_8 atomic_fcmpset_8 262 #define atomic_cmpset_16 atomic_cmpset_16 263 #define atomic_fcmpset_16 atomic_fcmpset_16 264 265 _ATOMIC_CMPSET( , , ) 266 _ATOMIC_CMPSET(acq_, a, ) 267 _ATOMIC_CMPSET(rel_, ,l) 268 269 #define _ATOMIC_FETCHADD_PROTO(t, flav) \ 270 static __inline uint##t##_t \ 271 atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val) 272 273 #define _ATOMIC_FETCHADD_IMPL(t, w) \ 274 _ATOMIC_FETCHADD_PROTO(t, _llsc) \ 275 { \ 276 uint##t##_t ret, tmp; \ 277 int res; \ 278 \ 279 __asm __volatile( \ 280 "1: ldxr %"#w"2, [%3]\n" \ 281 " add %"#w"0, %"#w"2, %"#w"4\n" \ 282 " stxr %w1, %"#w"0, [%3]\n" \ 283 " cbnz %w1, 1b\n" \ 284 : "=&r" (tmp), "=&r" (res), "=&r" (ret) \ 285 : "r" (p), "r" (val) \ 286 : "memory" \ 287 ); \ 288 \ 289 return (ret); \ 290 } \ 291 \ 292 _ATOMIC_FETCHADD_PROTO(t, _lse) \ 293 { \ 294 uint##t##_t ret; \ 295 \ 296 __asm __volatile( \ 297 ".arch_extension lse\n" \ 298 "ldadd %"#w"2, %"#w"0, [%1]\n" \ 299 ".arch_extension nolse\n" \ 300 : "=r" (ret) \ 301 : "r" (p), "r" (val) \ 302 : "memory" \ 303 ); \ 304 \ 305 return (ret); \ 306 } \ 307 \ 308 _ATOMIC_FETCHADD_PROTO(t, ) \ 309 { \ 310 if (_ATOMIC_LSE_SUPPORTED) \ 311 return (atomic_fetchadd_##t##_lse(p, val)); \ 312 else \ 313 return (atomic_fetchadd_##t##_llsc(p, val)); \ 314 } 315 316 _ATOMIC_FETCHADD_IMPL(32, w) 317 _ATOMIC_FETCHADD_IMPL(64, ) 318 319 #define _ATOMIC_SWAP_PROTO(t, flav) \ 320 static __inline uint##t##_t \ 321 atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val) 322 323 #define _ATOMIC_READANDCLEAR_PROTO(t, flav) \ 324 static __inline uint##t##_t \ 325 atomic_readandclear_##t##flav(volatile uint##t##_t *p) 326 327 #define _ATOMIC_SWAP_IMPL(t, w, zreg) \ 328 _ATOMIC_SWAP_PROTO(t, _llsc) \ 329 { \ 330 uint##t##_t ret; \ 331 int res; \ 332 \ 333 __asm __volatile( \ 334 "1: ldxr %"#w"1, [%2]\n" \ 335 " stxr %w0, %"#w"3, [%2]\n" \ 336 " cbnz %w0, 1b\n" \ 337 : "=&r" (res), "=&r" (ret) \ 338 : "r" (p), "r" (val) \ 339 : "memory" \ 340 ); \ 341 \ 342 return (ret); \ 343 } \ 344 \ 345 _ATOMIC_SWAP_PROTO(t, _lse) \ 346 { \ 347 uint##t##_t ret; \ 348 \ 349 __asm __volatile( \ 350 ".arch_extension lse\n" \ 351 "swp %"#w"2, %"#w"0, [%1]\n" \ 352 ".arch_extension nolse\n" \ 353 : "=r" (ret) \ 354 : "r" (p), "r" (val) \ 355 : "memory" \ 356 ); \ 357 \ 358 return (ret); \ 359 } \ 360 \ 361 _ATOMIC_SWAP_PROTO(t, ) \ 362 { \ 363 if (_ATOMIC_LSE_SUPPORTED) \ 364 return (atomic_swap_##t##_lse(p, val)); \ 365 else \ 366 return (atomic_swap_##t##_llsc(p, val)); \ 367 } \ 368 \ 369 _ATOMIC_READANDCLEAR_PROTO(t, _llsc) \ 370 { \ 371 uint##t##_t ret; \ 372 int res; \ 373 \ 374 __asm __volatile( \ 375 "1: ldxr %"#w"1, [%2]\n" \ 376 " stxr %w0, "#zreg", [%2]\n" \ 377 " cbnz %w0, 1b\n" \ 378 : "=&r" (res), "=&r" (ret) \ 379 : "r" (p) \ 380 : "memory" \ 381 ); \ 382 \ 383 return (ret); \ 384 } \ 385 \ 386 _ATOMIC_READANDCLEAR_PROTO(t, _lse) \ 387 { \ 388 return (atomic_swap_##t##_lse(p, 0)); \ 389 } \ 390 \ 391 _ATOMIC_READANDCLEAR_PROTO(t, ) \ 392 { \ 393 if (_ATOMIC_LSE_SUPPORTED) \ 394 return (atomic_readandclear_##t##_lse(p)); \ 395 else \ 396 return (atomic_readandclear_##t##_llsc(p)); \ 397 } 398 399 _ATOMIC_SWAP_IMPL(32, w, wzr) 400 _ATOMIC_SWAP_IMPL(64, , xzr) 401 402 #define _ATOMIC_TEST_OP_PROTO(t, op, flav) \ 403 static __inline int \ 404 atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val) 405 406 #define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op) \ 407 _ATOMIC_TEST_OP_PROTO(t, op, _llsc) \ 408 { \ 409 uint##t##_t mask, old, tmp; \ 410 int res; \ 411 \ 412 mask = 1u << (val & 0x1f); \ 413 __asm __volatile( \ 414 "1: ldxr %"#w"2, [%3]\n" \ 415 " "#llsc_asm_op" %"#w"0, %"#w"2, %"#w"4\n" \ 416 " stxr %w1, %"#w"0, [%3]\n" \ 417 " cbnz %w1, 1b\n" \ 418 : "=&r" (tmp), "=&r" (res), "=&r" (old) \ 419 : "r" (p), "r" (mask) \ 420 : "memory" \ 421 ); \ 422 \ 423 return ((old & mask) != 0); \ 424 } \ 425 \ 426 _ATOMIC_TEST_OP_PROTO(t, op, _lse) \ 427 { \ 428 uint##t##_t mask, old; \ 429 \ 430 mask = 1u << (val & 0x1f); \ 431 __asm __volatile( \ 432 ".arch_extension lse\n" \ 433 "ld"#lse_asm_op" %"#w"2, %"#w"0, [%1]\n" \ 434 ".arch_extension nolse\n" \ 435 : "=r" (old) \ 436 : "r" (p), "r" (mask) \ 437 : "memory" \ 438 ); \ 439 \ 440 return ((old & mask) != 0); \ 441 } \ 442 \ 443 _ATOMIC_TEST_OP_PROTO(t, op, ) \ 444 { \ 445 if (_ATOMIC_LSE_SUPPORTED) \ 446 return (atomic_testand##op##_##t##_lse(p, val)); \ 447 else \ 448 return (atomic_testand##op##_##t##_llsc(p, val)); \ 449 } 450 451 #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \ 452 _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op) \ 453 _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op) 454 455 _ATOMIC_TEST_OP(clear, bic, clr) 456 _ATOMIC_TEST_OP(set, orr, set) 457 458 #define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \ 459 static __inline uint##t##_t \ 460 atomic_load_acq_##t(volatile uint##t##_t *p) \ 461 { \ 462 uint##t##_t ret; \ 463 \ 464 __asm __volatile( \ 465 "ldar"#s" %"#w"0, [%1]\n" \ 466 : "=&r" (ret) \ 467 : "r" (p) \ 468 : "memory"); \ 469 \ 470 return (ret); \ 471 } 472 473 #define atomic_load_acq_8 atomic_load_acq_8 474 #define atomic_load_acq_16 atomic_load_acq_16 475 _ATOMIC_LOAD_ACQ_IMPL(8, w, b) 476 _ATOMIC_LOAD_ACQ_IMPL(16, w, h) 477 _ATOMIC_LOAD_ACQ_IMPL(32, w, ) 478 _ATOMIC_LOAD_ACQ_IMPL(64, , ) 479 480 #define _ATOMIC_STORE_REL_IMPL(t, w, s) \ 481 static __inline void \ 482 atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val) \ 483 { \ 484 __asm __volatile( \ 485 "stlr"#s" %"#w"0, [%1]\n" \ 486 : \ 487 : "r" (val), "r" (p) \ 488 : "memory"); \ 489 } 490 491 _ATOMIC_STORE_REL_IMPL(8, w, b) 492 _ATOMIC_STORE_REL_IMPL(16, w, h) 493 _ATOMIC_STORE_REL_IMPL(32, w, ) 494 _ATOMIC_STORE_REL_IMPL(64, , ) 495 496 #define atomic_add_int atomic_add_32 497 #define atomic_fcmpset_int atomic_fcmpset_32 498 #define atomic_clear_int atomic_clear_32 499 #define atomic_cmpset_int atomic_cmpset_32 500 #define atomic_fetchadd_int atomic_fetchadd_32 501 #define atomic_readandclear_int atomic_readandclear_32 502 #define atomic_set_int atomic_set_32 503 #define atomic_swap_int atomic_swap_32 504 #define atomic_subtract_int atomic_subtract_32 505 #define atomic_testandclear_int atomic_testandclear_32 506 #define atomic_testandset_int atomic_testandset_32 507 508 #define atomic_add_acq_int atomic_add_acq_32 509 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 510 #define atomic_clear_acq_int atomic_clear_acq_32 511 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 512 #define atomic_load_acq_int atomic_load_acq_32 513 #define atomic_set_acq_int atomic_set_acq_32 514 #define atomic_subtract_acq_int atomic_subtract_acq_32 515 516 #define atomic_add_rel_int atomic_add_rel_32 517 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 518 #define atomic_clear_rel_int atomic_clear_rel_32 519 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 520 #define atomic_set_rel_int atomic_set_rel_32 521 #define atomic_subtract_rel_int atomic_subtract_rel_32 522 #define atomic_store_rel_int atomic_store_rel_32 523 524 #define atomic_add_long atomic_add_64 525 #define atomic_fcmpset_long atomic_fcmpset_64 526 #define atomic_clear_long atomic_clear_64 527 #define atomic_cmpset_long atomic_cmpset_64 528 #define atomic_fetchadd_long atomic_fetchadd_64 529 #define atomic_readandclear_long atomic_readandclear_64 530 #define atomic_set_long atomic_set_64 531 #define atomic_swap_long atomic_swap_64 532 #define atomic_subtract_long atomic_subtract_64 533 #define atomic_testandclear_long atomic_testandclear_64 534 #define atomic_testandset_long atomic_testandset_64 535 536 #define atomic_add_ptr atomic_add_64 537 #define atomic_fcmpset_ptr atomic_fcmpset_64 538 #define atomic_clear_ptr atomic_clear_64 539 #define atomic_cmpset_ptr atomic_cmpset_64 540 #define atomic_fetchadd_ptr atomic_fetchadd_64 541 #define atomic_readandclear_ptr atomic_readandclear_64 542 #define atomic_set_ptr atomic_set_64 543 #define atomic_swap_ptr atomic_swap_64 544 #define atomic_subtract_ptr atomic_subtract_64 545 546 #define atomic_add_acq_long atomic_add_acq_64 547 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 548 #define atomic_clear_acq_long atomic_clear_acq_64 549 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 550 #define atomic_load_acq_long atomic_load_acq_64 551 #define atomic_set_acq_long atomic_set_acq_64 552 #define atomic_subtract_acq_long atomic_subtract_acq_64 553 554 #define atomic_add_acq_ptr atomic_add_acq_64 555 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 556 #define atomic_clear_acq_ptr atomic_clear_acq_64 557 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 558 #define atomic_load_acq_ptr atomic_load_acq_64 559 #define atomic_set_acq_ptr atomic_set_acq_64 560 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 561 562 #define atomic_add_rel_long atomic_add_rel_64 563 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 564 #define atomic_clear_rel_long atomic_clear_rel_64 565 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 566 #define atomic_set_rel_long atomic_set_rel_64 567 #define atomic_subtract_rel_long atomic_subtract_rel_64 568 #define atomic_store_rel_long atomic_store_rel_64 569 570 #define atomic_add_rel_ptr atomic_add_rel_64 571 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 572 #define atomic_clear_rel_ptr atomic_clear_rel_64 573 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 574 #define atomic_set_rel_ptr atomic_set_rel_64 575 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 576 #define atomic_store_rel_ptr atomic_store_rel_64 577 578 static __inline void 579 atomic_thread_fence_acq(void) 580 { 581 582 dmb(ld); 583 } 584 585 static __inline void 586 atomic_thread_fence_rel(void) 587 { 588 589 dmb(sy); 590 } 591 592 static __inline void 593 atomic_thread_fence_acq_rel(void) 594 { 595 596 dmb(sy); 597 } 598 599 static __inline void 600 atomic_thread_fence_seq_cst(void) 601 { 602 603 dmb(sy); 604 } 605 606 #include <sys/_atomic_subword.h> 607 608 #endif /* KCSAN && !KCSAN_RUNTIME */ 609 #endif /* _MACHINE_ATOMIC_H_ */ 610