1 /*- 2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #ifndef _MACHINE_ATOMIC_H_ 30 #define _MACHINE_ATOMIC_H_ 31 32 #define isb() __asm __volatile("isb" : : : "memory") 33 34 /* 35 * Options for DMB and DSB: 36 * oshld Outer Shareable, load 37 * oshst Outer Shareable, store 38 * osh Outer Shareable, all 39 * nshld Non-shareable, load 40 * nshst Non-shareable, store 41 * nsh Non-shareable, all 42 * ishld Inner Shareable, load 43 * ishst Inner Shareable, store 44 * ish Inner Shareable, all 45 * ld Full system, load 46 * st Full system, store 47 * sy Full system, all 48 */ 49 #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") 50 #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") 51 52 #define mb() dmb(sy) /* Full system memory barrier all */ 53 #define wmb() dmb(st) /* Full system memory barrier store */ 54 #define rmb() dmb(ld) /* Full system memory barrier load */ 55 56 #if defined(KCSAN) && !defined(KCSAN_RUNTIME) 57 #include <sys/_cscan_atomic.h> 58 #else 59 60 #include <sys/atomic_common.h> 61 62 #ifdef _KERNEL 63 extern bool lse_supported; 64 65 #ifdef LSE_ATOMICS 66 #define _ATOMIC_LSE_SUPPORTED 1 67 #else 68 #define _ATOMIC_LSE_SUPPORTED lse_supported 69 #endif 70 #else 71 #define _ATOMIC_LSE_SUPPORTED 0 72 #endif 73 74 #define _ATOMIC_OP_PROTO(t, op, bar, flav) \ 75 static __inline void \ 76 atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val) 77 78 #define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ 79 _ATOMIC_OP_PROTO(t, op, bar, _llsc) \ 80 { \ 81 uint##t##_t tmp; \ 82 int res; \ 83 \ 84 pre; \ 85 __asm __volatile( \ 86 "1: ld"#a"xr"#s" %"#w"0, [%2]\n" \ 87 " "#llsc_asm_op" %"#w"0, %"#w"0, %"#w"3\n" \ 88 " st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \ 89 " cbnz %w1, 1b\n" \ 90 : "=&r"(tmp), "=&r"(res) \ 91 : "r" (p), "r" (val) \ 92 : "memory" \ 93 ); \ 94 } \ 95 \ 96 _ATOMIC_OP_PROTO(t, op, bar, _lse) \ 97 { \ 98 uint##t##_t tmp; \ 99 \ 100 pre; \ 101 __asm __volatile( \ 102 ".arch_extension lse\n" \ 103 "ld"#lse_asm_op#a#l#s" %"#w"2, %"#w"0, [%1]\n" \ 104 ".arch_extension nolse\n" \ 105 : "=r" (tmp) \ 106 : "r" (p), "r" (val) \ 107 : "memory" \ 108 ); \ 109 } \ 110 \ 111 _ATOMIC_OP_PROTO(t, op, bar, ) \ 112 { \ 113 if (_ATOMIC_LSE_SUPPORTED) \ 114 atomic_##op##_##bar##t##_lse(p, val); \ 115 else \ 116 atomic_##op##_##bar##t##_llsc(p, val); \ 117 } 118 119 #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ 120 _ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \ 121 bar, a, l) \ 122 _ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \ 123 bar, a, l) \ 124 _ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \ 125 bar, a, l) \ 126 _ATOMIC_OP_IMPL(64, , , op, llsc_asm_op, lse_asm_op, pre, \ 127 bar, a, l) 128 129 #define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre) \ 130 __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, , , ) \ 131 __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a, ) \ 132 __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_, , l) 133 134 _ATOMIC_OP(add, add, add, ) 135 _ATOMIC_OP(clear, bic, clr, ) 136 _ATOMIC_OP(set, orr, set, ) 137 _ATOMIC_OP(subtract, add, add, val = -val) 138 139 #define _ATOMIC_CMPSET_PROTO(t, bar, flav) \ 140 static __inline int \ 141 atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \ 142 uint##t##_t cmpval, uint##t##_t newval) 143 144 #define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \ 145 static __inline int \ 146 atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p, \ 147 uint##t##_t *cmpval, uint##t##_t newval) 148 149 #define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \ 150 _ATOMIC_CMPSET_PROTO(t, bar, _llsc) \ 151 { \ 152 uint##t##_t tmp; \ 153 int res; \ 154 \ 155 __asm __volatile( \ 156 "1: mov %w1, #1\n" \ 157 " ld"#a"xr"#s" %"#w"0, [%2]\n" \ 158 " cmp %"#w"0, %"#w"3\n" \ 159 " b.ne 2f\n" \ 160 " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ 161 " cbnz %w1, 1b\n" \ 162 "2:" \ 163 : "=&r"(tmp), "=&r"(res) \ 164 : "r" (p), "r" (cmpval), "r" (newval) \ 165 : "cc", "memory" \ 166 ); \ 167 \ 168 return (!res); \ 169 } \ 170 \ 171 _ATOMIC_CMPSET_PROTO(t, bar, _lse) \ 172 { \ 173 uint##t##_t oldval; \ 174 int res; \ 175 \ 176 oldval = cmpval; \ 177 __asm __volatile( \ 178 ".arch_extension lse\n" \ 179 "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ 180 "cmp %"#w"1, %"#w"2\n" \ 181 "cset %w0, eq\n" \ 182 ".arch_extension nolse\n" \ 183 : "=r" (res), "+&r" (cmpval) \ 184 : "r" (oldval), "r" (p), "r" (newval) \ 185 : "cc", "memory" \ 186 ); \ 187 \ 188 return (res); \ 189 } \ 190 \ 191 _ATOMIC_CMPSET_PROTO(t, bar, ) \ 192 { \ 193 if (_ATOMIC_LSE_SUPPORTED) \ 194 return (atomic_cmpset_##bar##t##_lse(p, cmpval, \ 195 newval)); \ 196 else \ 197 return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \ 198 newval)); \ 199 } \ 200 \ 201 _ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \ 202 { \ 203 uint##t##_t _cmpval, tmp; \ 204 int res; \ 205 \ 206 _cmpval = *cmpval; \ 207 __asm __volatile( \ 208 " mov %w1, #1\n" \ 209 " ld"#a"xr"#s" %"#w"0, [%2]\n" \ 210 " cmp %"#w"0, %"#w"3\n" \ 211 " b.ne 1f\n" \ 212 " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ 213 "1:" \ 214 : "=&r"(tmp), "=&r"(res) \ 215 : "r" (p), "r" (_cmpval), "r" (newval) \ 216 : "cc", "memory" \ 217 ); \ 218 *cmpval = tmp; \ 219 \ 220 return (!res); \ 221 } \ 222 \ 223 _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \ 224 { \ 225 uint##t##_t _cmpval, tmp; \ 226 int res; \ 227 \ 228 _cmpval = tmp = *cmpval; \ 229 __asm __volatile( \ 230 ".arch_extension lse\n" \ 231 "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ 232 "cmp %"#w"1, %"#w"2\n" \ 233 "cset %w0, eq\n" \ 234 ".arch_extension nolse\n" \ 235 : "=r" (res), "+&r" (tmp) \ 236 : "r" (_cmpval), "r" (p), "r" (newval) \ 237 : "cc", "memory" \ 238 ); \ 239 *cmpval = tmp; \ 240 \ 241 return (res); \ 242 } \ 243 \ 244 _ATOMIC_FCMPSET_PROTO(t, bar, ) \ 245 { \ 246 if (_ATOMIC_LSE_SUPPORTED) \ 247 return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \ 248 newval)); \ 249 else \ 250 return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \ 251 newval)); \ 252 } 253 254 #define _ATOMIC_CMPSET(bar, a, l) \ 255 _ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \ 256 _ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \ 257 _ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \ 258 _ATOMIC_CMPSET_IMPL(64, , , bar, a, l) 259 260 _ATOMIC_CMPSET( , , ) 261 _ATOMIC_CMPSET(acq_, a, ) 262 _ATOMIC_CMPSET(rel_, ,l) 263 264 #define _ATOMIC_FETCHADD_PROTO(t, flav) \ 265 static __inline uint##t##_t \ 266 atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val) 267 268 #define _ATOMIC_FETCHADD_IMPL(t, w) \ 269 _ATOMIC_FETCHADD_PROTO(t, _llsc) \ 270 { \ 271 uint##t##_t ret, tmp; \ 272 int res; \ 273 \ 274 __asm __volatile( \ 275 "1: ldxr %"#w"2, [%3]\n" \ 276 " add %"#w"0, %"#w"2, %"#w"4\n" \ 277 " stxr %w1, %"#w"0, [%3]\n" \ 278 " cbnz %w1, 1b\n" \ 279 : "=&r" (tmp), "=&r" (res), "=&r" (ret) \ 280 : "r" (p), "r" (val) \ 281 : "memory" \ 282 ); \ 283 \ 284 return (ret); \ 285 } \ 286 \ 287 _ATOMIC_FETCHADD_PROTO(t, _lse) \ 288 { \ 289 uint##t##_t ret; \ 290 \ 291 __asm __volatile( \ 292 ".arch_extension lse\n" \ 293 "ldadd %"#w"2, %"#w"0, [%1]\n" \ 294 ".arch_extension nolse\n" \ 295 : "=r" (ret) \ 296 : "r" (p), "r" (val) \ 297 : "memory" \ 298 ); \ 299 \ 300 return (ret); \ 301 } \ 302 \ 303 _ATOMIC_FETCHADD_PROTO(t, ) \ 304 { \ 305 if (_ATOMIC_LSE_SUPPORTED) \ 306 return (atomic_fetchadd_##t##_lse(p, val)); \ 307 else \ 308 return (atomic_fetchadd_##t##_llsc(p, val)); \ 309 } 310 311 _ATOMIC_FETCHADD_IMPL(32, w) 312 _ATOMIC_FETCHADD_IMPL(64, ) 313 314 #define _ATOMIC_SWAP_PROTO(t, flav) \ 315 static __inline uint##t##_t \ 316 atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val) 317 318 #define _ATOMIC_READANDCLEAR_PROTO(t, flav) \ 319 static __inline uint##t##_t \ 320 atomic_readandclear_##t##flav(volatile uint##t##_t *p) 321 322 #define _ATOMIC_SWAP_IMPL(t, w, zreg) \ 323 _ATOMIC_SWAP_PROTO(t, _llsc) \ 324 { \ 325 uint##t##_t ret; \ 326 int res; \ 327 \ 328 __asm __volatile( \ 329 "1: ldxr %"#w"1, [%2]\n" \ 330 " stxr %w0, %"#w"3, [%2]\n" \ 331 " cbnz %w0, 1b\n" \ 332 : "=&r" (res), "=&r" (ret) \ 333 : "r" (p), "r" (val) \ 334 : "memory" \ 335 ); \ 336 \ 337 return (ret); \ 338 } \ 339 \ 340 _ATOMIC_SWAP_PROTO(t, _lse) \ 341 { \ 342 uint##t##_t ret; \ 343 \ 344 __asm __volatile( \ 345 ".arch_extension lse\n" \ 346 "swp %"#w"2, %"#w"0, [%1]\n" \ 347 ".arch_extension nolse\n" \ 348 : "=r" (ret) \ 349 : "r" (p), "r" (val) \ 350 : "memory" \ 351 ); \ 352 \ 353 return (ret); \ 354 } \ 355 \ 356 _ATOMIC_SWAP_PROTO(t, ) \ 357 { \ 358 if (_ATOMIC_LSE_SUPPORTED) \ 359 return (atomic_swap_##t##_lse(p, val)); \ 360 else \ 361 return (atomic_swap_##t##_llsc(p, val)); \ 362 } \ 363 \ 364 _ATOMIC_READANDCLEAR_PROTO(t, _llsc) \ 365 { \ 366 uint##t##_t ret; \ 367 int res; \ 368 \ 369 __asm __volatile( \ 370 "1: ldxr %"#w"1, [%2]\n" \ 371 " stxr %w0, "#zreg", [%2]\n" \ 372 " cbnz %w0, 1b\n" \ 373 : "=&r" (res), "=&r" (ret) \ 374 : "r" (p) \ 375 : "memory" \ 376 ); \ 377 \ 378 return (ret); \ 379 } \ 380 \ 381 _ATOMIC_READANDCLEAR_PROTO(t, _lse) \ 382 { \ 383 return (atomic_swap_##t##_lse(p, 0)); \ 384 } \ 385 \ 386 _ATOMIC_READANDCLEAR_PROTO(t, ) \ 387 { \ 388 if (_ATOMIC_LSE_SUPPORTED) \ 389 return (atomic_readandclear_##t##_lse(p)); \ 390 else \ 391 return (atomic_readandclear_##t##_llsc(p)); \ 392 } 393 394 _ATOMIC_SWAP_IMPL(32, w, wzr) 395 _ATOMIC_SWAP_IMPL(64, , xzr) 396 397 #define _ATOMIC_TEST_OP_PROTO(t, op, flav) \ 398 static __inline int \ 399 atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val) 400 401 #define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op) \ 402 _ATOMIC_TEST_OP_PROTO(t, op, _llsc) \ 403 { \ 404 uint##t##_t mask, old, tmp; \ 405 int res; \ 406 \ 407 mask = 1u << (val & 0x1f); \ 408 __asm __volatile( \ 409 "1: ldxr %"#w"2, [%3]\n" \ 410 " "#llsc_asm_op" %"#w"0, %"#w"2, %"#w"4\n" \ 411 " stxr %w1, %"#w"0, [%3]\n" \ 412 " cbnz %w1, 1b\n" \ 413 : "=&r" (tmp), "=&r" (res), "=&r" (old) \ 414 : "r" (p), "r" (mask) \ 415 : "memory" \ 416 ); \ 417 \ 418 return ((old & mask) != 0); \ 419 } \ 420 \ 421 _ATOMIC_TEST_OP_PROTO(t, op, _lse) \ 422 { \ 423 uint##t##_t mask, old; \ 424 \ 425 mask = 1u << (val & 0x1f); \ 426 __asm __volatile( \ 427 ".arch_extension lse\n" \ 428 "ld"#lse_asm_op" %"#w"2, %"#w"0, [%1]\n" \ 429 ".arch_extension nolse\n" \ 430 : "=r" (old) \ 431 : "r" (p), "r" (mask) \ 432 : "memory" \ 433 ); \ 434 \ 435 return ((old & mask) != 0); \ 436 } \ 437 \ 438 _ATOMIC_TEST_OP_PROTO(t, op, ) \ 439 { \ 440 if (_ATOMIC_LSE_SUPPORTED) \ 441 return (atomic_testand##op##_##t##_lse(p, val)); \ 442 else \ 443 return (atomic_testand##op##_##t##_llsc(p, val)); \ 444 } 445 446 #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \ 447 _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op) \ 448 _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op) 449 450 _ATOMIC_TEST_OP(clear, bic, clr) 451 _ATOMIC_TEST_OP(set, orr, set) 452 453 #define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \ 454 static __inline uint##t##_t \ 455 atomic_load_acq_##t(volatile uint##t##_t *p) \ 456 { \ 457 uint##t##_t ret; \ 458 \ 459 __asm __volatile( \ 460 "ldar"#s" %"#w"0, [%1]\n" \ 461 : "=&r" (ret) \ 462 : "r" (p) \ 463 : "memory"); \ 464 \ 465 return (ret); \ 466 } 467 468 _ATOMIC_LOAD_ACQ_IMPL(8, w, b) 469 _ATOMIC_LOAD_ACQ_IMPL(16, w, h) 470 _ATOMIC_LOAD_ACQ_IMPL(32, w, ) 471 _ATOMIC_LOAD_ACQ_IMPL(64, , ) 472 473 #define _ATOMIC_STORE_REL_IMPL(t, w, s) \ 474 static __inline void \ 475 atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val) \ 476 { \ 477 __asm __volatile( \ 478 "stlr"#s" %"#w"0, [%1]\n" \ 479 : \ 480 : "r" (val), "r" (p) \ 481 : "memory"); \ 482 } 483 484 _ATOMIC_STORE_REL_IMPL(8, w, b) 485 _ATOMIC_STORE_REL_IMPL(16, w, h) 486 _ATOMIC_STORE_REL_IMPL(32, w, ) 487 _ATOMIC_STORE_REL_IMPL(64, , ) 488 489 #define atomic_add_int atomic_add_32 490 #define atomic_fcmpset_int atomic_fcmpset_32 491 #define atomic_clear_int atomic_clear_32 492 #define atomic_cmpset_int atomic_cmpset_32 493 #define atomic_fetchadd_int atomic_fetchadd_32 494 #define atomic_readandclear_int atomic_readandclear_32 495 #define atomic_set_int atomic_set_32 496 #define atomic_swap_int atomic_swap_32 497 #define atomic_subtract_int atomic_subtract_32 498 #define atomic_testandclear_int atomic_testandclear_32 499 #define atomic_testandset_int atomic_testandset_32 500 501 #define atomic_add_acq_int atomic_add_acq_32 502 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 503 #define atomic_clear_acq_int atomic_clear_acq_32 504 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 505 #define atomic_load_acq_int atomic_load_acq_32 506 #define atomic_set_acq_int atomic_set_acq_32 507 #define atomic_subtract_acq_int atomic_subtract_acq_32 508 509 #define atomic_add_rel_int atomic_add_rel_32 510 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 511 #define atomic_clear_rel_int atomic_clear_rel_32 512 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 513 #define atomic_set_rel_int atomic_set_rel_32 514 #define atomic_subtract_rel_int atomic_subtract_rel_32 515 #define atomic_store_rel_int atomic_store_rel_32 516 517 #define atomic_add_long atomic_add_64 518 #define atomic_fcmpset_long atomic_fcmpset_64 519 #define atomic_clear_long atomic_clear_64 520 #define atomic_cmpset_long atomic_cmpset_64 521 #define atomic_fetchadd_long atomic_fetchadd_64 522 #define atomic_readandclear_long atomic_readandclear_64 523 #define atomic_set_long atomic_set_64 524 #define atomic_swap_long atomic_swap_64 525 #define atomic_subtract_long atomic_subtract_64 526 #define atomic_testandclear_long atomic_testandclear_64 527 #define atomic_testandset_long atomic_testandset_64 528 529 #define atomic_add_ptr atomic_add_64 530 #define atomic_fcmpset_ptr atomic_fcmpset_64 531 #define atomic_clear_ptr atomic_clear_64 532 #define atomic_cmpset_ptr atomic_cmpset_64 533 #define atomic_fetchadd_ptr atomic_fetchadd_64 534 #define atomic_readandclear_ptr atomic_readandclear_64 535 #define atomic_set_ptr atomic_set_64 536 #define atomic_swap_ptr atomic_swap_64 537 #define atomic_subtract_ptr atomic_subtract_64 538 539 #define atomic_add_acq_long atomic_add_acq_64 540 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 541 #define atomic_clear_acq_long atomic_clear_acq_64 542 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 543 #define atomic_load_acq_long atomic_load_acq_64 544 #define atomic_set_acq_long atomic_set_acq_64 545 #define atomic_subtract_acq_long atomic_subtract_acq_64 546 547 #define atomic_add_acq_ptr atomic_add_acq_64 548 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 549 #define atomic_clear_acq_ptr atomic_clear_acq_64 550 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 551 #define atomic_load_acq_ptr atomic_load_acq_64 552 #define atomic_set_acq_ptr atomic_set_acq_64 553 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 554 555 #define atomic_add_rel_long atomic_add_rel_64 556 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 557 #define atomic_clear_rel_long atomic_clear_rel_64 558 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 559 #define atomic_set_rel_long atomic_set_rel_64 560 #define atomic_subtract_rel_long atomic_subtract_rel_64 561 #define atomic_store_rel_long atomic_store_rel_64 562 563 #define atomic_add_rel_ptr atomic_add_rel_64 564 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 565 #define atomic_clear_rel_ptr atomic_clear_rel_64 566 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 567 #define atomic_set_rel_ptr atomic_set_rel_64 568 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 569 #define atomic_store_rel_ptr atomic_store_rel_64 570 571 static __inline void 572 atomic_thread_fence_acq(void) 573 { 574 575 dmb(ld); 576 } 577 578 static __inline void 579 atomic_thread_fence_rel(void) 580 { 581 582 dmb(sy); 583 } 584 585 static __inline void 586 atomic_thread_fence_acq_rel(void) 587 { 588 589 dmb(sy); 590 } 591 592 static __inline void 593 atomic_thread_fence_seq_cst(void) 594 { 595 596 dmb(sy); 597 } 598 599 #endif /* KCSAN && !KCSAN_RUNTIME */ 600 #endif /* _MACHINE_ATOMIC_H_ */ 601