1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1998 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 #ifndef _MACHINE_ATOMIC_H_ 31 #define _MACHINE_ATOMIC_H_ 32 33 #include <sys/atomic_common.h> 34 35 #ifdef _KERNEL 36 #include <machine/md_var.h> 37 #include <machine/specialreg.h> 38 #endif 39 40 #ifndef __OFFSETOF_MONITORBUF 41 /* 42 * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). 43 * 44 * The open-coded number is used instead of the symbolic expression to 45 * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. 46 * An assertion in i386/vm_machdep.c ensures that the value is correct. 47 */ 48 #define __OFFSETOF_MONITORBUF 0x80 49 50 static __inline void 51 __mbk(void) 52 { 53 54 __asm __volatile("lock; addl $0,%%fs:%0" 55 : "+m" (*(u_int *)__OFFSETOF_MONITORBUF) : : "memory", "cc"); 56 } 57 58 static __inline void 59 __mbu(void) 60 { 61 62 __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc"); 63 } 64 #endif 65 66 /* 67 * Various simple operations on memory, each of which is atomic in the 68 * presence of interrupts and multiple processors. 69 * 70 * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) 71 * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) 72 * atomic_add_char(P, V) (*(u_char *)(P) += (V)) 73 * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) 74 * 75 * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) 76 * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) 77 * atomic_add_short(P, V) (*(u_short *)(P) += (V)) 78 * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) 79 * 80 * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) 81 * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) 82 * atomic_add_int(P, V) (*(u_int *)(P) += (V)) 83 * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) 84 * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) 85 * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) 86 * 87 * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) 88 * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) 89 * atomic_add_long(P, V) (*(u_long *)(P) += (V)) 90 * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) 91 * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) 92 * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) 93 */ 94 95 /* 96 * Always use lock prefixes. The result is slighly less optimal for 97 * UP systems, but it matters less now, and sometimes UP is emulated 98 * over SMP. 99 * 100 * The assembly is volatilized to avoid code chunk removal by the compiler. 101 * GCC aggressively reorders operations and memory clobbering is necessary 102 * in order to avoid that for memory barriers. 103 */ 104 #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 105 static __inline void \ 106 atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 107 { \ 108 __asm __volatile("lock; " OP \ 109 : "+m" (*p) \ 110 : CONS (V) \ 111 : "cc"); \ 112 } \ 113 \ 114 static __inline void \ 115 atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 116 { \ 117 __asm __volatile("lock; " OP \ 118 : "+m" (*p) \ 119 : CONS (V) \ 120 : "memory", "cc"); \ 121 } \ 122 struct __hack 123 124 /* 125 * Atomic compare and set, used by the mutex functions. 126 * 127 * cmpset: 128 * if (*dst == expect) 129 * *dst = src 130 * 131 * fcmpset: 132 * if (*dst == *expect) 133 * *dst = src 134 * else 135 * *expect = *dst 136 * 137 * Returns 0 on failure, non-zero on success. 138 */ 139 #define ATOMIC_CMPSET(TYPE, CONS) \ 140 static __inline int \ 141 atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ 142 { \ 143 u_char res; \ 144 \ 145 __asm __volatile( \ 146 " lock; cmpxchg %3,%1 ; " \ 147 " sete %0 ; " \ 148 "# atomic_cmpset_" #TYPE " " \ 149 : "=q" (res), /* 0 */ \ 150 "+m" (*dst), /* 1 */ \ 151 "+a" (expect) /* 2 */ \ 152 : CONS (src) /* 3 */ \ 153 : "memory", "cc"); \ 154 return (res); \ 155 } \ 156 \ 157 static __inline int \ 158 atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ 159 { \ 160 u_char res; \ 161 \ 162 __asm __volatile( \ 163 " lock; cmpxchg %3,%1 ; " \ 164 " sete %0 ; " \ 165 "# atomic_fcmpset_" #TYPE " " \ 166 : "=q" (res), /* 0 */ \ 167 "+m" (*dst), /* 1 */ \ 168 "+a" (*expect) /* 2 */ \ 169 : CONS (src) /* 3 */ \ 170 : "memory", "cc"); \ 171 return (res); \ 172 } 173 174 ATOMIC_CMPSET(char, "q"); 175 ATOMIC_CMPSET(short, "r"); 176 ATOMIC_CMPSET(int, "r"); 177 178 /* 179 * Atomically add the value of v to the integer pointed to by p and return 180 * the previous value of *p. 181 */ 182 static __inline u_int 183 atomic_fetchadd_int(volatile u_int *p, u_int v) 184 { 185 186 __asm __volatile( 187 " lock; xaddl %0,%1 ; " 188 "# atomic_fetchadd_int" 189 : "+r" (v), /* 0 */ 190 "+m" (*p) /* 1 */ 191 : : "cc"); 192 return (v); 193 } 194 195 static __inline int 196 atomic_testandset_int(volatile u_int *p, u_int v) 197 { 198 u_char res; 199 200 __asm __volatile( 201 " lock; btsl %2,%1 ; " 202 " setc %0 ; " 203 "# atomic_testandset_int" 204 : "=q" (res), /* 0 */ 205 "+m" (*p) /* 1 */ 206 : "Ir" (v & 0x1f) /* 2 */ 207 : "cc"); 208 return (res); 209 } 210 211 static __inline int 212 atomic_testandclear_int(volatile u_int *p, u_int v) 213 { 214 u_char res; 215 216 __asm __volatile( 217 " lock; btrl %2,%1 ; " 218 " setc %0 ; " 219 "# atomic_testandclear_int" 220 : "=q" (res), /* 0 */ 221 "+m" (*p) /* 1 */ 222 : "Ir" (v & 0x1f) /* 2 */ 223 : "cc"); 224 return (res); 225 } 226 227 /* 228 * We assume that a = b will do atomic loads and stores. Due to the 229 * IA32 memory model, a simple store guarantees release semantics. 230 * 231 * However, a load may pass a store if they are performed on distinct 232 * addresses, so we need Store/Load barrier for sequentially 233 * consistent fences in SMP kernels. We use "lock addl $0,mem" for a 234 * Store/Load barrier, as recommended by the AMD Software Optimization 235 * Guide, and not mfence. In the kernel, we use a private per-cpu 236 * cache line for "mem", to avoid introducing false data 237 * dependencies. In user space, we use the word at the top of the 238 * stack. 239 * 240 * For UP kernels, however, the memory of the single processor is 241 * always consistent, so we only need to stop the compiler from 242 * reordering accesses in a way that violates the semantics of acquire 243 * and release. 244 */ 245 246 #if defined(_KERNEL) 247 #define __storeload_barrier() __mbk() 248 #else /* !_KERNEL */ 249 #define __storeload_barrier() __mbu() 250 #endif /* _KERNEL*/ 251 252 #define ATOMIC_LOAD(TYPE) \ 253 static __inline u_##TYPE \ 254 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 255 { \ 256 u_##TYPE res; \ 257 \ 258 res = *p; \ 259 __compiler_membar(); \ 260 return (res); \ 261 } \ 262 struct __hack 263 264 #define ATOMIC_STORE(TYPE) \ 265 static __inline void \ 266 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 267 { \ 268 \ 269 __compiler_membar(); \ 270 *p = v; \ 271 } \ 272 struct __hack 273 274 static __inline void 275 atomic_thread_fence_acq(void) 276 { 277 278 __compiler_membar(); 279 } 280 281 static __inline void 282 atomic_thread_fence_rel(void) 283 { 284 285 __compiler_membar(); 286 } 287 288 static __inline void 289 atomic_thread_fence_acq_rel(void) 290 { 291 292 __compiler_membar(); 293 } 294 295 static __inline void 296 atomic_thread_fence_seq_cst(void) 297 { 298 299 __storeload_barrier(); 300 } 301 302 #ifdef _KERNEL 303 304 #ifdef WANT_FUNCTIONS 305 int atomic_cmpset_64_i386(volatile uint64_t *, uint64_t, uint64_t); 306 int atomic_cmpset_64_i586(volatile uint64_t *, uint64_t, uint64_t); 307 uint64_t atomic_load_acq_64_i386(volatile uint64_t *); 308 uint64_t atomic_load_acq_64_i586(volatile uint64_t *); 309 void atomic_store_rel_64_i386(volatile uint64_t *, uint64_t); 310 void atomic_store_rel_64_i586(volatile uint64_t *, uint64_t); 311 uint64_t atomic_swap_64_i386(volatile uint64_t *, uint64_t); 312 uint64_t atomic_swap_64_i586(volatile uint64_t *, uint64_t); 313 #endif 314 315 /* I486 does not support SMP or CMPXCHG8B. */ 316 static __inline int 317 atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src) 318 { 319 volatile uint32_t *p; 320 u_char res; 321 322 p = (volatile uint32_t *)dst; 323 __asm __volatile( 324 " pushfl ; " 325 " cli ; " 326 " xorl %1,%%eax ; " 327 " xorl %2,%%edx ; " 328 " orl %%edx,%%eax ; " 329 " jne 1f ; " 330 " movl %4,%1 ; " 331 " movl %5,%2 ; " 332 "1: " 333 " sete %3 ; " 334 " popfl" 335 : "+A" (expect), /* 0 */ 336 "+m" (*p), /* 1 */ 337 "+m" (*(p + 1)), /* 2 */ 338 "=q" (res) /* 3 */ 339 : "r" ((uint32_t)src), /* 4 */ 340 "r" ((uint32_t)(src >> 32)) /* 5 */ 341 : "memory", "cc"); 342 return (res); 343 } 344 345 static __inline int 346 atomic_fcmpset_64_i386(volatile uint64_t *dst, uint64_t *expect, uint64_t src) 347 { 348 349 if (atomic_cmpset_64_i386(dst, *expect, src)) { 350 return (1); 351 } else { 352 *expect = *dst; 353 return (0); 354 } 355 } 356 357 static __inline uint64_t 358 atomic_load_acq_64_i386(volatile uint64_t *p) 359 { 360 volatile uint32_t *q; 361 uint64_t res; 362 363 q = (volatile uint32_t *)p; 364 __asm __volatile( 365 " pushfl ; " 366 " cli ; " 367 " movl %1,%%eax ; " 368 " movl %2,%%edx ; " 369 " popfl" 370 : "=&A" (res) /* 0 */ 371 : "m" (*q), /* 1 */ 372 "m" (*(q + 1)) /* 2 */ 373 : "memory"); 374 return (res); 375 } 376 377 static __inline void 378 atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) 379 { 380 volatile uint32_t *q; 381 382 q = (volatile uint32_t *)p; 383 __asm __volatile( 384 " pushfl ; " 385 " cli ; " 386 " movl %%eax,%0 ; " 387 " movl %%edx,%1 ; " 388 " popfl" 389 : "=m" (*q), /* 0 */ 390 "=m" (*(q + 1)) /* 1 */ 391 : "A" (v) /* 2 */ 392 : "memory"); 393 } 394 395 static __inline uint64_t 396 atomic_swap_64_i386(volatile uint64_t *p, uint64_t v) 397 { 398 volatile uint32_t *q; 399 uint64_t res; 400 401 q = (volatile uint32_t *)p; 402 __asm __volatile( 403 " pushfl ; " 404 " cli ; " 405 " movl %1,%%eax ; " 406 " movl %2,%%edx ; " 407 " movl %4,%2 ; " 408 " movl %3,%1 ; " 409 " popfl" 410 : "=&A" (res), /* 0 */ 411 "+m" (*q), /* 1 */ 412 "+m" (*(q + 1)) /* 2 */ 413 : "r" ((uint32_t)v), /* 3 */ 414 "r" ((uint32_t)(v >> 32))); /* 4 */ 415 return (res); 416 } 417 418 static __inline int 419 atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src) 420 { 421 u_char res; 422 423 __asm __volatile( 424 " lock; cmpxchg8b %1 ; " 425 " sete %0" 426 : "=q" (res), /* 0 */ 427 "+m" (*dst), /* 1 */ 428 "+A" (expect) /* 2 */ 429 : "b" ((uint32_t)src), /* 3 */ 430 "c" ((uint32_t)(src >> 32)) /* 4 */ 431 : "memory", "cc"); 432 return (res); 433 } 434 435 static __inline int 436 atomic_fcmpset_64_i586(volatile uint64_t *dst, uint64_t *expect, uint64_t src) 437 { 438 u_char res; 439 440 __asm __volatile( 441 " lock; cmpxchg8b %1 ; " 442 " sete %0" 443 : "=q" (res), /* 0 */ 444 "+m" (*dst), /* 1 */ 445 "+A" (*expect) /* 2 */ 446 : "b" ((uint32_t)src), /* 3 */ 447 "c" ((uint32_t)(src >> 32)) /* 4 */ 448 : "memory", "cc"); 449 return (res); 450 } 451 452 static __inline uint64_t 453 atomic_load_acq_64_i586(volatile uint64_t *p) 454 { 455 uint64_t res; 456 457 __asm __volatile( 458 " movl %%ebx,%%eax ; " 459 " movl %%ecx,%%edx ; " 460 " lock; cmpxchg8b %1" 461 : "=&A" (res), /* 0 */ 462 "+m" (*p) /* 1 */ 463 : : "memory", "cc"); 464 return (res); 465 } 466 467 static __inline void 468 atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) 469 { 470 471 __asm __volatile( 472 " movl %%eax,%%ebx ; " 473 " movl %%edx,%%ecx ; " 474 "1: " 475 " lock; cmpxchg8b %0 ; " 476 " jne 1b" 477 : "+m" (*p), /* 0 */ 478 "+A" (v) /* 1 */ 479 : : "ebx", "ecx", "memory", "cc"); 480 } 481 482 static __inline uint64_t 483 atomic_swap_64_i586(volatile uint64_t *p, uint64_t v) 484 { 485 486 __asm __volatile( 487 " movl %%eax,%%ebx ; " 488 " movl %%edx,%%ecx ; " 489 "1: " 490 " lock; cmpxchg8b %0 ; " 491 " jne 1b" 492 : "+m" (*p), /* 0 */ 493 "+A" (v) /* 1 */ 494 : : "ebx", "ecx", "memory", "cc"); 495 return (v); 496 } 497 498 static __inline int 499 atomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src) 500 { 501 502 if ((cpu_feature & CPUID_CX8) == 0) 503 return (atomic_cmpset_64_i386(dst, expect, src)); 504 else 505 return (atomic_cmpset_64_i586(dst, expect, src)); 506 } 507 508 static __inline int 509 atomic_fcmpset_64(volatile uint64_t *dst, uint64_t *expect, uint64_t src) 510 { 511 512 if ((cpu_feature & CPUID_CX8) == 0) 513 return (atomic_fcmpset_64_i386(dst, expect, src)); 514 else 515 return (atomic_fcmpset_64_i586(dst, expect, src)); 516 } 517 518 static __inline uint64_t 519 atomic_load_acq_64(volatile uint64_t *p) 520 { 521 522 if ((cpu_feature & CPUID_CX8) == 0) 523 return (atomic_load_acq_64_i386(p)); 524 else 525 return (atomic_load_acq_64_i586(p)); 526 } 527 528 static __inline void 529 atomic_store_rel_64(volatile uint64_t *p, uint64_t v) 530 { 531 532 if ((cpu_feature & CPUID_CX8) == 0) 533 atomic_store_rel_64_i386(p, v); 534 else 535 atomic_store_rel_64_i586(p, v); 536 } 537 538 static __inline uint64_t 539 atomic_swap_64(volatile uint64_t *p, uint64_t v) 540 { 541 542 if ((cpu_feature & CPUID_CX8) == 0) 543 return (atomic_swap_64_i386(p, v)); 544 else 545 return (atomic_swap_64_i586(p, v)); 546 } 547 548 static __inline uint64_t 549 atomic_fetchadd_64(volatile uint64_t *p, uint64_t v) 550 { 551 552 for (;;) { 553 uint64_t t = *p; 554 if (atomic_cmpset_64(p, t, t + v)) 555 return (t); 556 } 557 } 558 559 static __inline void 560 atomic_add_64(volatile uint64_t *p, uint64_t v) 561 { 562 uint64_t t; 563 564 for (;;) { 565 t = *p; 566 if (atomic_cmpset_64(p, t, t + v)) 567 break; 568 } 569 } 570 571 static __inline void 572 atomic_subtract_64(volatile uint64_t *p, uint64_t v) 573 { 574 uint64_t t; 575 576 for (;;) { 577 t = *p; 578 if (atomic_cmpset_64(p, t, t - v)) 579 break; 580 } 581 } 582 583 #endif /* _KERNEL */ 584 585 ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); 586 ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); 587 ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); 588 ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); 589 590 ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); 591 ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); 592 ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); 593 ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); 594 595 ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); 596 ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); 597 ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); 598 ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); 599 600 ATOMIC_ASM(set, long, "orl %1,%0", "ir", v); 601 ATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v); 602 ATOMIC_ASM(add, long, "addl %1,%0", "ir", v); 603 ATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v); 604 605 #define ATOMIC_LOADSTORE(TYPE) \ 606 ATOMIC_LOAD(TYPE); \ 607 ATOMIC_STORE(TYPE) 608 609 ATOMIC_LOADSTORE(char); 610 ATOMIC_LOADSTORE(short); 611 ATOMIC_LOADSTORE(int); 612 ATOMIC_LOADSTORE(long); 613 614 #undef ATOMIC_ASM 615 #undef ATOMIC_LOAD 616 #undef ATOMIC_STORE 617 #undef ATOMIC_LOADSTORE 618 619 #ifndef WANT_FUNCTIONS 620 621 static __inline int 622 atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) 623 { 624 625 return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect, 626 (u_int)src)); 627 } 628 629 static __inline int 630 atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src) 631 { 632 633 return (atomic_fcmpset_int((volatile u_int *)dst, (u_int *)expect, 634 (u_int)src)); 635 } 636 637 static __inline u_long 638 atomic_fetchadd_long(volatile u_long *p, u_long v) 639 { 640 641 return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); 642 } 643 644 static __inline int 645 atomic_testandset_long(volatile u_long *p, u_int v) 646 { 647 648 return (atomic_testandset_int((volatile u_int *)p, v)); 649 } 650 651 static __inline int 652 atomic_testandclear_long(volatile u_long *p, u_int v) 653 { 654 655 return (atomic_testandclear_int((volatile u_int *)p, v)); 656 } 657 658 /* Read the current value and store a new value in the destination. */ 659 static __inline u_int 660 atomic_swap_int(volatile u_int *p, u_int v) 661 { 662 663 __asm __volatile( 664 " xchgl %1,%0 ; " 665 "# atomic_swap_int" 666 : "+r" (v), /* 0 */ 667 "+m" (*p)); /* 1 */ 668 return (v); 669 } 670 671 static __inline u_long 672 atomic_swap_long(volatile u_long *p, u_long v) 673 { 674 675 return (atomic_swap_int((volatile u_int *)p, (u_int)v)); 676 } 677 678 #define atomic_set_acq_char atomic_set_barr_char 679 #define atomic_set_rel_char atomic_set_barr_char 680 #define atomic_clear_acq_char atomic_clear_barr_char 681 #define atomic_clear_rel_char atomic_clear_barr_char 682 #define atomic_add_acq_char atomic_add_barr_char 683 #define atomic_add_rel_char atomic_add_barr_char 684 #define atomic_subtract_acq_char atomic_subtract_barr_char 685 #define atomic_subtract_rel_char atomic_subtract_barr_char 686 #define atomic_cmpset_acq_char atomic_cmpset_char 687 #define atomic_cmpset_rel_char atomic_cmpset_char 688 #define atomic_fcmpset_acq_char atomic_fcmpset_char 689 #define atomic_fcmpset_rel_char atomic_fcmpset_char 690 691 #define atomic_set_acq_short atomic_set_barr_short 692 #define atomic_set_rel_short atomic_set_barr_short 693 #define atomic_clear_acq_short atomic_clear_barr_short 694 #define atomic_clear_rel_short atomic_clear_barr_short 695 #define atomic_add_acq_short atomic_add_barr_short 696 #define atomic_add_rel_short atomic_add_barr_short 697 #define atomic_subtract_acq_short atomic_subtract_barr_short 698 #define atomic_subtract_rel_short atomic_subtract_barr_short 699 #define atomic_cmpset_acq_short atomic_cmpset_short 700 #define atomic_cmpset_rel_short atomic_cmpset_short 701 #define atomic_fcmpset_acq_short atomic_fcmpset_short 702 #define atomic_fcmpset_rel_short atomic_fcmpset_short 703 704 #define atomic_set_acq_int atomic_set_barr_int 705 #define atomic_set_rel_int atomic_set_barr_int 706 #define atomic_clear_acq_int atomic_clear_barr_int 707 #define atomic_clear_rel_int atomic_clear_barr_int 708 #define atomic_add_acq_int atomic_add_barr_int 709 #define atomic_add_rel_int atomic_add_barr_int 710 #define atomic_subtract_acq_int atomic_subtract_barr_int 711 #define atomic_subtract_rel_int atomic_subtract_barr_int 712 #define atomic_cmpset_acq_int atomic_cmpset_int 713 #define atomic_cmpset_rel_int atomic_cmpset_int 714 #define atomic_fcmpset_acq_int atomic_fcmpset_int 715 #define atomic_fcmpset_rel_int atomic_fcmpset_int 716 717 #define atomic_set_acq_long atomic_set_barr_long 718 #define atomic_set_rel_long atomic_set_barr_long 719 #define atomic_clear_acq_long atomic_clear_barr_long 720 #define atomic_clear_rel_long atomic_clear_barr_long 721 #define atomic_add_acq_long atomic_add_barr_long 722 #define atomic_add_rel_long atomic_add_barr_long 723 #define atomic_subtract_acq_long atomic_subtract_barr_long 724 #define atomic_subtract_rel_long atomic_subtract_barr_long 725 #define atomic_cmpset_acq_long atomic_cmpset_long 726 #define atomic_cmpset_rel_long atomic_cmpset_long 727 #define atomic_fcmpset_acq_long atomic_fcmpset_long 728 #define atomic_fcmpset_rel_long atomic_fcmpset_long 729 730 #define atomic_readandclear_int(p) atomic_swap_int(p, 0) 731 #define atomic_readandclear_long(p) atomic_swap_long(p, 0) 732 #define atomic_testandset_acq_long atomic_testandset_long 733 734 /* Operations on 8-bit bytes. */ 735 #define atomic_set_8 atomic_set_char 736 #define atomic_set_acq_8 atomic_set_acq_char 737 #define atomic_set_rel_8 atomic_set_rel_char 738 #define atomic_clear_8 atomic_clear_char 739 #define atomic_clear_acq_8 atomic_clear_acq_char 740 #define atomic_clear_rel_8 atomic_clear_rel_char 741 #define atomic_add_8 atomic_add_char 742 #define atomic_add_acq_8 atomic_add_acq_char 743 #define atomic_add_rel_8 atomic_add_rel_char 744 #define atomic_subtract_8 atomic_subtract_char 745 #define atomic_subtract_acq_8 atomic_subtract_acq_char 746 #define atomic_subtract_rel_8 atomic_subtract_rel_char 747 #define atomic_load_acq_8 atomic_load_acq_char 748 #define atomic_store_rel_8 atomic_store_rel_char 749 #define atomic_cmpset_8 atomic_cmpset_char 750 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char 751 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char 752 #define atomic_fcmpset_8 atomic_fcmpset_char 753 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char 754 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char 755 756 /* Operations on 16-bit words. */ 757 #define atomic_set_16 atomic_set_short 758 #define atomic_set_acq_16 atomic_set_acq_short 759 #define atomic_set_rel_16 atomic_set_rel_short 760 #define atomic_clear_16 atomic_clear_short 761 #define atomic_clear_acq_16 atomic_clear_acq_short 762 #define atomic_clear_rel_16 atomic_clear_rel_short 763 #define atomic_add_16 atomic_add_short 764 #define atomic_add_acq_16 atomic_add_acq_short 765 #define atomic_add_rel_16 atomic_add_rel_short 766 #define atomic_subtract_16 atomic_subtract_short 767 #define atomic_subtract_acq_16 atomic_subtract_acq_short 768 #define atomic_subtract_rel_16 atomic_subtract_rel_short 769 #define atomic_load_acq_16 atomic_load_acq_short 770 #define atomic_store_rel_16 atomic_store_rel_short 771 #define atomic_cmpset_16 atomic_cmpset_short 772 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short 773 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short 774 #define atomic_fcmpset_16 atomic_fcmpset_short 775 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short 776 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short 777 778 /* Operations on 32-bit double words. */ 779 #define atomic_set_32 atomic_set_int 780 #define atomic_set_acq_32 atomic_set_acq_int 781 #define atomic_set_rel_32 atomic_set_rel_int 782 #define atomic_clear_32 atomic_clear_int 783 #define atomic_clear_acq_32 atomic_clear_acq_int 784 #define atomic_clear_rel_32 atomic_clear_rel_int 785 #define atomic_add_32 atomic_add_int 786 #define atomic_add_acq_32 atomic_add_acq_int 787 #define atomic_add_rel_32 atomic_add_rel_int 788 #define atomic_subtract_32 atomic_subtract_int 789 #define atomic_subtract_acq_32 atomic_subtract_acq_int 790 #define atomic_subtract_rel_32 atomic_subtract_rel_int 791 #define atomic_load_acq_32 atomic_load_acq_int 792 #define atomic_store_rel_32 atomic_store_rel_int 793 #define atomic_cmpset_32 atomic_cmpset_int 794 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 795 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 796 #define atomic_fcmpset_32 atomic_fcmpset_int 797 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 798 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 799 #define atomic_swap_32 atomic_swap_int 800 #define atomic_readandclear_32 atomic_readandclear_int 801 #define atomic_fetchadd_32 atomic_fetchadd_int 802 #define atomic_testandset_32 atomic_testandset_int 803 #define atomic_testandclear_32 atomic_testandclear_int 804 805 #ifdef _KERNEL 806 /* Operations on 64-bit quad words. */ 807 #define atomic_cmpset_acq_64 atomic_cmpset_64 808 #define atomic_cmpset_rel_64 atomic_cmpset_64 809 #define atomic_fcmpset_acq_64 atomic_fcmpset_64 810 #define atomic_fcmpset_rel_64 atomic_fcmpset_64 811 #define atomic_fetchadd_acq_64 atomic_fetchadd_64 812 #define atomic_fetchadd_rel_64 atomic_fetchadd_64 813 #define atomic_add_acq_64 atomic_add_64 814 #define atomic_add_rel_64 atomic_add_64 815 #define atomic_subtract_acq_64 atomic_subtract_64 816 #define atomic_subtract_rel_64 atomic_subtract_64 817 #define atomic_load_64 atomic_load_acq_64 818 #define atomic_store_64 atomic_store_rel_64 819 #endif 820 821 /* Operations on pointers. */ 822 #define atomic_set_ptr(p, v) \ 823 atomic_set_int((volatile u_int *)(p), (u_int)(v)) 824 #define atomic_set_acq_ptr(p, v) \ 825 atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) 826 #define atomic_set_rel_ptr(p, v) \ 827 atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) 828 #define atomic_clear_ptr(p, v) \ 829 atomic_clear_int((volatile u_int *)(p), (u_int)(v)) 830 #define atomic_clear_acq_ptr(p, v) \ 831 atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) 832 #define atomic_clear_rel_ptr(p, v) \ 833 atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) 834 #define atomic_add_ptr(p, v) \ 835 atomic_add_int((volatile u_int *)(p), (u_int)(v)) 836 #define atomic_add_acq_ptr(p, v) \ 837 atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) 838 #define atomic_add_rel_ptr(p, v) \ 839 atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) 840 #define atomic_subtract_ptr(p, v) \ 841 atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) 842 #define atomic_subtract_acq_ptr(p, v) \ 843 atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) 844 #define atomic_subtract_rel_ptr(p, v) \ 845 atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) 846 #define atomic_load_acq_ptr(p) \ 847 atomic_load_acq_int((volatile u_int *)(p)) 848 #define atomic_store_rel_ptr(p, v) \ 849 atomic_store_rel_int((volatile u_int *)(p), (v)) 850 #define atomic_cmpset_ptr(dst, old, new) \ 851 atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 852 #define atomic_cmpset_acq_ptr(dst, old, new) \ 853 atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ 854 (u_int)(new)) 855 #define atomic_cmpset_rel_ptr(dst, old, new) \ 856 atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ 857 (u_int)(new)) 858 #define atomic_fcmpset_ptr(dst, old, new) \ 859 atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old), (u_int)(new)) 860 #define atomic_fcmpset_acq_ptr(dst, old, new) \ 861 atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \ 862 (u_int)(new)) 863 #define atomic_fcmpset_rel_ptr(dst, old, new) \ 864 atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \ 865 (u_int)(new)) 866 #define atomic_swap_ptr(p, v) \ 867 atomic_swap_int((volatile u_int *)(p), (u_int)(v)) 868 #define atomic_readandclear_ptr(p) \ 869 atomic_readandclear_int((volatile u_int *)(p)) 870 871 #endif /* !WANT_FUNCTIONS */ 872 873 #if defined(_KERNEL) 874 #define mb() __mbk() 875 #define wmb() __mbk() 876 #define rmb() __mbk() 877 #else 878 #define mb() __mbu() 879 #define wmb() __mbu() 880 #define rmb() __mbu() 881 #endif 882 883 #endif /* !_MACHINE_ATOMIC_H_ */ 884