1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1998 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 #ifndef _MACHINE_ATOMIC_H_ 31 #define _MACHINE_ATOMIC_H_ 32 33 /* 34 * To express interprocessor (as opposed to processor and device) memory 35 * ordering constraints, use the atomic_*() functions with acquire and release 36 * semantics rather than the *mb() functions. An architecture's memory 37 * ordering (or memory consistency) model governs the order in which a 38 * program's accesses to different locations may be performed by an 39 * implementation of that architecture. In general, for memory regions 40 * defined as writeback cacheable, the memory ordering implemented by amd64 41 * processors preserves the program ordering of a load followed by a load, a 42 * load followed by a store, and a store followed by a store. Only a store 43 * followed by a load to a different memory location may be reordered. 44 * Therefore, except for special cases, like non-temporal memory accesses or 45 * memory regions defined as write combining, the memory ordering effects 46 * provided by the sfence instruction in the wmb() function and the lfence 47 * instruction in the rmb() function are redundant. In contrast, the 48 * atomic_*() functions with acquire and release semantics do not perform 49 * redundant instructions for ordinary cases of interprocessor memory 50 * ordering on any architecture. 51 */ 52 #define mb() __asm __volatile("mfence;" : : : "memory") 53 #define wmb() __asm __volatile("sfence;" : : : "memory") 54 #define rmb() __asm __volatile("lfence;" : : : "memory") 55 56 #ifdef _KERNEL 57 /* 58 * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). 59 * 60 * The open-coded number is used instead of the symbolic expression to 61 * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. 62 * An assertion in amd64/vm_machdep.c ensures that the value is correct. 63 */ 64 #define OFFSETOF_MONITORBUF 0x100 65 #endif 66 67 #if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) 68 #include <sys/atomic_san.h> 69 #else 70 #include <sys/atomic_common.h> 71 72 /* 73 * Various simple operations on memory, each of which is atomic in the 74 * presence of interrupts and multiple processors. 75 * 76 * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) 77 * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) 78 * atomic_add_char(P, V) (*(u_char *)(P) += (V)) 79 * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) 80 * 81 * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) 82 * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) 83 * atomic_add_short(P, V) (*(u_short *)(P) += (V)) 84 * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) 85 * 86 * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) 87 * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) 88 * atomic_add_int(P, V) (*(u_int *)(P) += (V)) 89 * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) 90 * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) 91 * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) 92 * 93 * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) 94 * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) 95 * atomic_add_long(P, V) (*(u_long *)(P) += (V)) 96 * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) 97 * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) 98 * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) 99 */ 100 101 /* 102 * Always use lock prefixes. The result is slighly less optimal for 103 * UP systems, but it matters less now, and sometimes UP is emulated 104 * over SMP. 105 * 106 * The assembly is volatilized to avoid code chunk removal by the compiler. 107 * GCC aggressively reorders operations and memory clobbering is necessary 108 * in order to avoid that for memory barriers. 109 */ 110 #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 111 static __inline void \ 112 atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 113 { \ 114 __asm __volatile("lock; " OP \ 115 : "+m" (*p) \ 116 : CONS (V) \ 117 : "cc"); \ 118 } \ 119 \ 120 static __inline void \ 121 atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 122 { \ 123 __asm __volatile("lock; " OP \ 124 : "+m" (*p) \ 125 : CONS (V) \ 126 : "memory", "cc"); \ 127 } \ 128 struct __hack 129 130 /* 131 * Atomic compare and set, used by the mutex functions. 132 * 133 * cmpset: 134 * if (*dst == expect) 135 * *dst = src 136 * 137 * fcmpset: 138 * if (*dst == *expect) 139 * *dst = src 140 * else 141 * *expect = *dst 142 * 143 * Returns 0 on failure, non-zero on success. 144 */ 145 #define ATOMIC_CMPSET(TYPE) \ 146 static __inline int \ 147 atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ 148 { \ 149 u_char res; \ 150 \ 151 __asm __volatile( \ 152 " lock; cmpxchg %3,%1 ; " \ 153 "# atomic_cmpset_" #TYPE " " \ 154 : "=@cce" (res), /* 0 */ \ 155 "+m" (*dst), /* 1 */ \ 156 "+a" (expect) /* 2 */ \ 157 : "r" (src) /* 3 */ \ 158 : "memory", "cc"); \ 159 return (res); \ 160 } \ 161 \ 162 static __inline int \ 163 atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ 164 { \ 165 u_char res; \ 166 \ 167 __asm __volatile( \ 168 " lock; cmpxchg %3,%1 ; " \ 169 "# atomic_fcmpset_" #TYPE " " \ 170 : "=@cce" (res), /* 0 */ \ 171 "+m" (*dst), /* 1 */ \ 172 "+a" (*expect) /* 2 */ \ 173 : "r" (src) /* 3 */ \ 174 : "memory", "cc"); \ 175 return (res); \ 176 } 177 178 ATOMIC_CMPSET(char); 179 ATOMIC_CMPSET(short); 180 ATOMIC_CMPSET(int); 181 ATOMIC_CMPSET(long); 182 183 /* 184 * Atomically add the value of v to the integer pointed to by p and return 185 * the previous value of *p. 186 */ 187 static __inline u_int 188 atomic_fetchadd_int(volatile u_int *p, u_int v) 189 { 190 191 __asm __volatile( 192 " lock; xaddl %0,%1 ; " 193 "# atomic_fetchadd_int" 194 : "+r" (v), /* 0 */ 195 "+m" (*p) /* 1 */ 196 : : "cc"); 197 return (v); 198 } 199 200 /* 201 * Atomically add the value of v to the long integer pointed to by p and return 202 * the previous value of *p. 203 */ 204 static __inline u_long 205 atomic_fetchadd_long(volatile u_long *p, u_long v) 206 { 207 208 __asm __volatile( 209 " lock; xaddq %0,%1 ; " 210 "# atomic_fetchadd_long" 211 : "+r" (v), /* 0 */ 212 "+m" (*p) /* 1 */ 213 : : "cc"); 214 return (v); 215 } 216 217 static __inline int 218 atomic_testandset_int(volatile u_int *p, u_int v) 219 { 220 u_char res; 221 222 __asm __volatile( 223 " lock; btsl %2,%1 ; " 224 "# atomic_testandset_int" 225 : "=@ccc" (res), /* 0 */ 226 "+m" (*p) /* 1 */ 227 : "Ir" (v & 0x1f) /* 2 */ 228 : "cc"); 229 return (res); 230 } 231 232 static __inline int 233 atomic_testandset_long(volatile u_long *p, u_int v) 234 { 235 u_char res; 236 237 __asm __volatile( 238 " lock; btsq %2,%1 ; " 239 "# atomic_testandset_long" 240 : "=@ccc" (res), /* 0 */ 241 "+m" (*p) /* 1 */ 242 : "Jr" ((u_long)(v & 0x3f)) /* 2 */ 243 : "cc"); 244 return (res); 245 } 246 247 static __inline int 248 atomic_testandclear_int(volatile u_int *p, u_int v) 249 { 250 u_char res; 251 252 __asm __volatile( 253 " lock; btrl %2,%1 ; " 254 "# atomic_testandclear_int" 255 : "=@ccc" (res), /* 0 */ 256 "+m" (*p) /* 1 */ 257 : "Ir" (v & 0x1f) /* 2 */ 258 : "cc"); 259 return (res); 260 } 261 262 static __inline int 263 atomic_testandclear_long(volatile u_long *p, u_int v) 264 { 265 u_char res; 266 267 __asm __volatile( 268 " lock; btrq %2,%1 ; " 269 "# atomic_testandclear_long" 270 : "=@ccc" (res), /* 0 */ 271 "+m" (*p) /* 1 */ 272 : "Jr" ((u_long)(v & 0x3f)) /* 2 */ 273 : "cc"); 274 return (res); 275 } 276 277 /* 278 * We assume that a = b will do atomic loads and stores. Due to the 279 * IA32 memory model, a simple store guarantees release semantics. 280 * 281 * However, a load may pass a store if they are performed on distinct 282 * addresses, so we need a Store/Load barrier for sequentially 283 * consistent fences in SMP kernels. We use "lock addl $0,mem" for a 284 * Store/Load barrier, as recommended by the AMD Software Optimization 285 * Guide, and not mfence. To avoid false data dependencies, we use a 286 * special address for "mem". In the kernel, we use a private per-cpu 287 * cache line. In user space, we use a word in the stack's red zone 288 * (-8(%rsp)). 289 */ 290 291 static __inline void 292 __storeload_barrier(void) 293 { 294 #if defined(_KERNEL) 295 __asm __volatile("lock; addl $0,%%gs:%0" 296 : "+m" (*(u_int *)OFFSETOF_MONITORBUF) : : "memory", "cc"); 297 #else /* !_KERNEL */ 298 __asm __volatile("lock; addl $0,-8(%%rsp)" : : : "memory", "cc"); 299 #endif /* _KERNEL*/ 300 } 301 302 #define ATOMIC_LOAD(TYPE) \ 303 static __inline u_##TYPE \ 304 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 305 { \ 306 u_##TYPE res; \ 307 \ 308 res = *p; \ 309 __compiler_membar(); \ 310 return (res); \ 311 } \ 312 struct __hack 313 314 #define ATOMIC_STORE(TYPE) \ 315 static __inline void \ 316 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 317 { \ 318 \ 319 __compiler_membar(); \ 320 *p = v; \ 321 } \ 322 struct __hack 323 324 static __inline void 325 atomic_thread_fence_acq(void) 326 { 327 328 __compiler_membar(); 329 } 330 331 static __inline void 332 atomic_thread_fence_rel(void) 333 { 334 335 __compiler_membar(); 336 } 337 338 static __inline void 339 atomic_thread_fence_acq_rel(void) 340 { 341 342 __compiler_membar(); 343 } 344 345 static __inline void 346 atomic_thread_fence_seq_cst(void) 347 { 348 349 __storeload_barrier(); 350 } 351 352 ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); 353 ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); 354 ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); 355 ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); 356 357 ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); 358 ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); 359 ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); 360 ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); 361 362 ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); 363 ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); 364 ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); 365 ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); 366 367 ATOMIC_ASM(set, long, "orq %1,%0", "er", v); 368 ATOMIC_ASM(clear, long, "andq %1,%0", "er", ~v); 369 ATOMIC_ASM(add, long, "addq %1,%0", "er", v); 370 ATOMIC_ASM(subtract, long, "subq %1,%0", "er", v); 371 372 #define ATOMIC_LOADSTORE(TYPE) \ 373 ATOMIC_LOAD(TYPE); \ 374 ATOMIC_STORE(TYPE) 375 376 ATOMIC_LOADSTORE(char); 377 ATOMIC_LOADSTORE(short); 378 ATOMIC_LOADSTORE(int); 379 ATOMIC_LOADSTORE(long); 380 381 #undef ATOMIC_ASM 382 #undef ATOMIC_LOAD 383 #undef ATOMIC_STORE 384 #undef ATOMIC_LOADSTORE 385 #ifndef WANT_FUNCTIONS 386 387 /* Read the current value and store a new value in the destination. */ 388 static __inline u_int 389 atomic_swap_int(volatile u_int *p, u_int v) 390 { 391 392 __asm __volatile( 393 " xchgl %1,%0 ; " 394 "# atomic_swap_int" 395 : "+r" (v), /* 0 */ 396 "+m" (*p)); /* 1 */ 397 return (v); 398 } 399 400 static __inline u_long 401 atomic_swap_long(volatile u_long *p, u_long v) 402 { 403 404 __asm __volatile( 405 " xchgq %1,%0 ; " 406 "# atomic_swap_long" 407 : "+r" (v), /* 0 */ 408 "+m" (*p)); /* 1 */ 409 return (v); 410 } 411 412 #define atomic_set_acq_char atomic_set_barr_char 413 #define atomic_set_rel_char atomic_set_barr_char 414 #define atomic_clear_acq_char atomic_clear_barr_char 415 #define atomic_clear_rel_char atomic_clear_barr_char 416 #define atomic_add_acq_char atomic_add_barr_char 417 #define atomic_add_rel_char atomic_add_barr_char 418 #define atomic_subtract_acq_char atomic_subtract_barr_char 419 #define atomic_subtract_rel_char atomic_subtract_barr_char 420 #define atomic_cmpset_acq_char atomic_cmpset_char 421 #define atomic_cmpset_rel_char atomic_cmpset_char 422 #define atomic_fcmpset_acq_char atomic_fcmpset_char 423 #define atomic_fcmpset_rel_char atomic_fcmpset_char 424 425 #define atomic_set_acq_short atomic_set_barr_short 426 #define atomic_set_rel_short atomic_set_barr_short 427 #define atomic_clear_acq_short atomic_clear_barr_short 428 #define atomic_clear_rel_short atomic_clear_barr_short 429 #define atomic_add_acq_short atomic_add_barr_short 430 #define atomic_add_rel_short atomic_add_barr_short 431 #define atomic_subtract_acq_short atomic_subtract_barr_short 432 #define atomic_subtract_rel_short atomic_subtract_barr_short 433 #define atomic_cmpset_acq_short atomic_cmpset_short 434 #define atomic_cmpset_rel_short atomic_cmpset_short 435 #define atomic_fcmpset_acq_short atomic_fcmpset_short 436 #define atomic_fcmpset_rel_short atomic_fcmpset_short 437 438 #define atomic_set_acq_int atomic_set_barr_int 439 #define atomic_set_rel_int atomic_set_barr_int 440 #define atomic_clear_acq_int atomic_clear_barr_int 441 #define atomic_clear_rel_int atomic_clear_barr_int 442 #define atomic_add_acq_int atomic_add_barr_int 443 #define atomic_add_rel_int atomic_add_barr_int 444 #define atomic_subtract_acq_int atomic_subtract_barr_int 445 #define atomic_subtract_rel_int atomic_subtract_barr_int 446 #define atomic_cmpset_acq_int atomic_cmpset_int 447 #define atomic_cmpset_rel_int atomic_cmpset_int 448 #define atomic_fcmpset_acq_int atomic_fcmpset_int 449 #define atomic_fcmpset_rel_int atomic_fcmpset_int 450 451 #define atomic_set_acq_long atomic_set_barr_long 452 #define atomic_set_rel_long atomic_set_barr_long 453 #define atomic_clear_acq_long atomic_clear_barr_long 454 #define atomic_clear_rel_long atomic_clear_barr_long 455 #define atomic_add_acq_long atomic_add_barr_long 456 #define atomic_add_rel_long atomic_add_barr_long 457 #define atomic_subtract_acq_long atomic_subtract_barr_long 458 #define atomic_subtract_rel_long atomic_subtract_barr_long 459 #define atomic_cmpset_acq_long atomic_cmpset_long 460 #define atomic_cmpset_rel_long atomic_cmpset_long 461 #define atomic_fcmpset_acq_long atomic_fcmpset_long 462 #define atomic_fcmpset_rel_long atomic_fcmpset_long 463 464 #define atomic_readandclear_int(p) atomic_swap_int(p, 0) 465 #define atomic_readandclear_long(p) atomic_swap_long(p, 0) 466 #define atomic_testandset_acq_long atomic_testandset_long 467 468 /* Operations on 8-bit bytes. */ 469 #define atomic_set_8 atomic_set_char 470 #define atomic_set_acq_8 atomic_set_acq_char 471 #define atomic_set_rel_8 atomic_set_rel_char 472 #define atomic_clear_8 atomic_clear_char 473 #define atomic_clear_acq_8 atomic_clear_acq_char 474 #define atomic_clear_rel_8 atomic_clear_rel_char 475 #define atomic_add_8 atomic_add_char 476 #define atomic_add_acq_8 atomic_add_acq_char 477 #define atomic_add_rel_8 atomic_add_rel_char 478 #define atomic_subtract_8 atomic_subtract_char 479 #define atomic_subtract_acq_8 atomic_subtract_acq_char 480 #define atomic_subtract_rel_8 atomic_subtract_rel_char 481 #define atomic_load_acq_8 atomic_load_acq_char 482 #define atomic_store_rel_8 atomic_store_rel_char 483 #define atomic_cmpset_8 atomic_cmpset_char 484 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char 485 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char 486 #define atomic_fcmpset_8 atomic_fcmpset_char 487 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char 488 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char 489 490 /* Operations on 16-bit words. */ 491 #define atomic_set_16 atomic_set_short 492 #define atomic_set_acq_16 atomic_set_acq_short 493 #define atomic_set_rel_16 atomic_set_rel_short 494 #define atomic_clear_16 atomic_clear_short 495 #define atomic_clear_acq_16 atomic_clear_acq_short 496 #define atomic_clear_rel_16 atomic_clear_rel_short 497 #define atomic_add_16 atomic_add_short 498 #define atomic_add_acq_16 atomic_add_acq_short 499 #define atomic_add_rel_16 atomic_add_rel_short 500 #define atomic_subtract_16 atomic_subtract_short 501 #define atomic_subtract_acq_16 atomic_subtract_acq_short 502 #define atomic_subtract_rel_16 atomic_subtract_rel_short 503 #define atomic_load_acq_16 atomic_load_acq_short 504 #define atomic_store_rel_16 atomic_store_rel_short 505 #define atomic_cmpset_16 atomic_cmpset_short 506 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short 507 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short 508 #define atomic_fcmpset_16 atomic_fcmpset_short 509 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short 510 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short 511 512 /* Operations on 32-bit double words. */ 513 #define atomic_set_32 atomic_set_int 514 #define atomic_set_acq_32 atomic_set_acq_int 515 #define atomic_set_rel_32 atomic_set_rel_int 516 #define atomic_clear_32 atomic_clear_int 517 #define atomic_clear_acq_32 atomic_clear_acq_int 518 #define atomic_clear_rel_32 atomic_clear_rel_int 519 #define atomic_add_32 atomic_add_int 520 #define atomic_add_acq_32 atomic_add_acq_int 521 #define atomic_add_rel_32 atomic_add_rel_int 522 #define atomic_subtract_32 atomic_subtract_int 523 #define atomic_subtract_acq_32 atomic_subtract_acq_int 524 #define atomic_subtract_rel_32 atomic_subtract_rel_int 525 #define atomic_load_acq_32 atomic_load_acq_int 526 #define atomic_store_rel_32 atomic_store_rel_int 527 #define atomic_cmpset_32 atomic_cmpset_int 528 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 529 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 530 #define atomic_fcmpset_32 atomic_fcmpset_int 531 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 532 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 533 #define atomic_swap_32 atomic_swap_int 534 #define atomic_readandclear_32 atomic_readandclear_int 535 #define atomic_fetchadd_32 atomic_fetchadd_int 536 #define atomic_testandset_32 atomic_testandset_int 537 #define atomic_testandclear_32 atomic_testandclear_int 538 539 /* Operations on 64-bit quad words. */ 540 #define atomic_set_64 atomic_set_long 541 #define atomic_set_acq_64 atomic_set_acq_long 542 #define atomic_set_rel_64 atomic_set_rel_long 543 #define atomic_clear_64 atomic_clear_long 544 #define atomic_clear_acq_64 atomic_clear_acq_long 545 #define atomic_clear_rel_64 atomic_clear_rel_long 546 #define atomic_add_64 atomic_add_long 547 #define atomic_add_acq_64 atomic_add_acq_long 548 #define atomic_add_rel_64 atomic_add_rel_long 549 #define atomic_subtract_64 atomic_subtract_long 550 #define atomic_subtract_acq_64 atomic_subtract_acq_long 551 #define atomic_subtract_rel_64 atomic_subtract_rel_long 552 #define atomic_load_acq_64 atomic_load_acq_long 553 #define atomic_store_rel_64 atomic_store_rel_long 554 #define atomic_cmpset_64 atomic_cmpset_long 555 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 556 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 557 #define atomic_fcmpset_64 atomic_fcmpset_long 558 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long 559 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long 560 #define atomic_swap_64 atomic_swap_long 561 #define atomic_readandclear_64 atomic_readandclear_long 562 #define atomic_fetchadd_64 atomic_fetchadd_long 563 #define atomic_testandset_64 atomic_testandset_long 564 #define atomic_testandclear_64 atomic_testandclear_long 565 566 /* Operations on pointers. */ 567 #define atomic_set_ptr atomic_set_long 568 #define atomic_set_acq_ptr atomic_set_acq_long 569 #define atomic_set_rel_ptr atomic_set_rel_long 570 #define atomic_clear_ptr atomic_clear_long 571 #define atomic_clear_acq_ptr atomic_clear_acq_long 572 #define atomic_clear_rel_ptr atomic_clear_rel_long 573 #define atomic_add_ptr atomic_add_long 574 #define atomic_add_acq_ptr atomic_add_acq_long 575 #define atomic_add_rel_ptr atomic_add_rel_long 576 #define atomic_subtract_ptr atomic_subtract_long 577 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 578 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 579 #define atomic_load_acq_ptr atomic_load_acq_long 580 #define atomic_store_rel_ptr atomic_store_rel_long 581 #define atomic_cmpset_ptr atomic_cmpset_long 582 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 583 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 584 #define atomic_fcmpset_ptr atomic_fcmpset_long 585 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 586 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 587 #define atomic_swap_ptr atomic_swap_long 588 #define atomic_readandclear_ptr atomic_readandclear_long 589 590 #endif /* !WANT_FUNCTIONS */ 591 592 #endif /* !SAN_NEEDS_INTERCEPTORS || SAN_RUNTIME */ 593 594 #endif /* !_MACHINE_ATOMIC_H_ */ 595