1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1998 Doug Rabson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28 #ifndef _MACHINE_ATOMIC_H_
29 #define _MACHINE_ATOMIC_H_
30
31 #include <sys/atomic_common.h>
32
33 #ifdef _KERNEL
34 #include <machine/md_var.h>
35 #include <machine/specialreg.h>
36 #endif
37
38 #ifndef __OFFSETOF_MONITORBUF
39 /*
40 * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
41 *
42 * The open-coded number is used instead of the symbolic expression to
43 * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
44 * An assertion in i386/vm_machdep.c ensures that the value is correct.
45 */
46 #define __OFFSETOF_MONITORBUF 0x80
47
48 static __inline void
__mbk(void)49 __mbk(void)
50 {
51
52 __asm __volatile("lock; addl $0,%%fs:%c0"
53 : : "i" (__OFFSETOF_MONITORBUF) : "memory", "cc");
54 }
55
56 static __inline void
__mbu(void)57 __mbu(void)
58 {
59
60 __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc");
61 }
62 #endif
63
64 /*
65 * Various simple operations on memory, each of which is atomic in the
66 * presence of interrupts and multiple processors.
67 *
68 * atomic_set_char(P, V) (*(u_char *)(P) |= (V))
69 * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V))
70 * atomic_add_char(P, V) (*(u_char *)(P) += (V))
71 * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V))
72 *
73 * atomic_set_short(P, V) (*(u_short *)(P) |= (V))
74 * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V))
75 * atomic_add_short(P, V) (*(u_short *)(P) += (V))
76 * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V))
77 *
78 * atomic_set_int(P, V) (*(u_int *)(P) |= (V))
79 * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V))
80 * atomic_add_int(P, V) (*(u_int *)(P) += (V))
81 * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V))
82 * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);)
83 * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;)
84 *
85 * atomic_set_long(P, V) (*(u_long *)(P) |= (V))
86 * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V))
87 * atomic_add_long(P, V) (*(u_long *)(P) += (V))
88 * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V))
89 * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);)
90 * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;)
91 */
92
93 /*
94 * Always use lock prefixes. The result is slightly less optimal for
95 * UP systems, but it matters less now, and sometimes UP is emulated
96 * over SMP.
97 *
98 * The assembly is volatilized to avoid code chunk removal by the compiler.
99 * GCC aggressively reorders operations and memory clobbering is necessary
100 * in order to avoid that for memory barriers.
101 */
102 #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \
103 static __inline void \
104 atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
105 { \
106 __asm __volatile("lock; " OP \
107 : "+m" (*p) \
108 : CONS (V) \
109 : "cc"); \
110 } \
111 \
112 static __inline void \
113 atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
114 { \
115 __asm __volatile("lock; " OP \
116 : "+m" (*p) \
117 : CONS (V) \
118 : "memory", "cc"); \
119 } \
120 struct __hack
121
122 /*
123 * Atomic compare and set, used by the mutex functions.
124 *
125 * cmpset:
126 * if (*dst == expect)
127 * *dst = src
128 *
129 * fcmpset:
130 * if (*dst == *expect)
131 * *dst = src
132 * else
133 * *expect = *dst
134 *
135 * Returns 0 on failure, non-zero on success.
136 */
137 #define ATOMIC_CMPSET(TYPE, CONS) \
138 static __inline int \
139 atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \
140 { \
141 u_char res; \
142 \
143 __asm __volatile( \
144 " lock; cmpxchg %3,%1 ; " \
145 " sete %0 ; " \
146 "# atomic_cmpset_" #TYPE " " \
147 : "=q" (res), /* 0 */ \
148 "+m" (*dst), /* 1 */ \
149 "+a" (expect) /* 2 */ \
150 : CONS (src) /* 3 */ \
151 : "memory", "cc"); \
152 return (res); \
153 } \
154 \
155 static __inline int \
156 atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \
157 { \
158 u_char res; \
159 \
160 __asm __volatile( \
161 " lock; cmpxchg %3,%1 ; " \
162 " sete %0 ; " \
163 "# atomic_fcmpset_" #TYPE " " \
164 : "=q" (res), /* 0 */ \
165 "+m" (*dst), /* 1 */ \
166 "+a" (*expect) /* 2 */ \
167 : CONS (src) /* 3 */ \
168 : "memory", "cc"); \
169 return (res); \
170 }
171
172 ATOMIC_CMPSET(char, "q");
173 ATOMIC_CMPSET(short, "r");
174 ATOMIC_CMPSET(int, "r");
175
176 /*
177 * Atomically add the value of v to the integer pointed to by p and return
178 * the previous value of *p.
179 */
180 static __inline u_int
atomic_fetchadd_int(volatile u_int * p,u_int v)181 atomic_fetchadd_int(volatile u_int *p, u_int v)
182 {
183
184 __asm __volatile(
185 " lock; xaddl %0,%1 ; "
186 "# atomic_fetchadd_int"
187 : "+r" (v), /* 0 */
188 "+m" (*p) /* 1 */
189 : : "cc");
190 return (v);
191 }
192
193 static __inline int
atomic_testandset_int(volatile u_int * p,u_int v)194 atomic_testandset_int(volatile u_int *p, u_int v)
195 {
196 u_char res;
197
198 __asm __volatile(
199 " lock; btsl %2,%1 ; "
200 " setc %0 ; "
201 "# atomic_testandset_int"
202 : "=q" (res), /* 0 */
203 "+m" (*p) /* 1 */
204 : "Ir" (v & 0x1f) /* 2 */
205 : "cc");
206 return (res);
207 }
208
209 static __inline int
atomic_testandclear_int(volatile u_int * p,u_int v)210 atomic_testandclear_int(volatile u_int *p, u_int v)
211 {
212 u_char res;
213
214 __asm __volatile(
215 " lock; btrl %2,%1 ; "
216 " setc %0 ; "
217 "# atomic_testandclear_int"
218 : "=q" (res), /* 0 */
219 "+m" (*p) /* 1 */
220 : "Ir" (v & 0x1f) /* 2 */
221 : "cc");
222 return (res);
223 }
224
225 /*
226 * We assume that a = b will do atomic loads and stores. Due to the
227 * IA32 memory model, a simple store guarantees release semantics.
228 *
229 * However, a load may pass a store if they are performed on distinct
230 * addresses, so we need Store/Load barrier for sequentially
231 * consistent fences in SMP kernels. We use "lock addl $0,mem" for a
232 * Store/Load barrier, as recommended by the AMD Software Optimization
233 * Guide, and not mfence. In the kernel, we use a private per-cpu
234 * cache line for "mem", to avoid introducing false data
235 * dependencies. In user space, we use the word at the top of the
236 * stack.
237 *
238 * For UP kernels, however, the memory of the single processor is
239 * always consistent, so we only need to stop the compiler from
240 * reordering accesses in a way that violates the semantics of acquire
241 * and release.
242 */
243
244 #if defined(_KERNEL)
245 #define __storeload_barrier() __mbk()
246 #else /* !_KERNEL */
247 #define __storeload_barrier() __mbu()
248 #endif /* _KERNEL*/
249
250 #define ATOMIC_LOAD(TYPE) \
251 static __inline u_##TYPE \
252 atomic_load_acq_##TYPE(const volatile u_##TYPE *p) \
253 { \
254 u_##TYPE res; \
255 \
256 res = *p; \
257 __compiler_membar(); \
258 return (res); \
259 } \
260 struct __hack
261
262 #define ATOMIC_STORE(TYPE) \
263 static __inline void \
264 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
265 { \
266 \
267 __compiler_membar(); \
268 *p = v; \
269 } \
270 struct __hack
271
272 static __inline void
atomic_thread_fence_acq(void)273 atomic_thread_fence_acq(void)
274 {
275
276 __compiler_membar();
277 }
278
279 static __inline void
atomic_thread_fence_rel(void)280 atomic_thread_fence_rel(void)
281 {
282
283 __compiler_membar();
284 }
285
286 static __inline void
atomic_thread_fence_acq_rel(void)287 atomic_thread_fence_acq_rel(void)
288 {
289
290 __compiler_membar();
291 }
292
293 static __inline void
atomic_thread_fence_seq_cst(void)294 atomic_thread_fence_seq_cst(void)
295 {
296
297 __storeload_barrier();
298 }
299
300 #ifdef _KERNEL
301
302 /* I486 does not support SMP or CMPXCHG8B. */
303 static __inline int
atomic_cmpset_64_i386(volatile uint64_t * dst,uint64_t expect,uint64_t src)304 atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src)
305 {
306 volatile uint32_t *p;
307 u_char res;
308
309 p = (volatile uint32_t *)dst;
310 __asm __volatile(
311 " pushfl ; "
312 " cli ; "
313 " xorl %1,%%eax ; "
314 " xorl %2,%%edx ; "
315 " orl %%edx,%%eax ; "
316 " jne 1f ; "
317 " movl %4,%1 ; "
318 " movl %5,%2 ; "
319 "1: "
320 " sete %3 ; "
321 " popfl"
322 : "+A" (expect), /* 0 */
323 "+m" (*p), /* 1 */
324 "+m" (*(p + 1)), /* 2 */
325 "=q" (res) /* 3 */
326 : "r" ((uint32_t)src), /* 4 */
327 "r" ((uint32_t)(src >> 32)) /* 5 */
328 : "memory", "cc");
329 return (res);
330 }
331
332 static __inline int
atomic_fcmpset_64_i386(volatile uint64_t * dst,uint64_t * expect,uint64_t src)333 atomic_fcmpset_64_i386(volatile uint64_t *dst, uint64_t *expect, uint64_t src)
334 {
335
336 if (atomic_cmpset_64_i386(dst, *expect, src)) {
337 return (1);
338 } else {
339 *expect = *dst;
340 return (0);
341 }
342 }
343
344 static __inline uint64_t
atomic_load_acq_64_i386(const volatile uint64_t * p)345 atomic_load_acq_64_i386(const volatile uint64_t *p)
346 {
347 const volatile uint32_t *q;
348 uint64_t res;
349
350 q = (const volatile uint32_t *)p;
351 __asm __volatile(
352 " pushfl ; "
353 " cli ; "
354 " movl %1,%%eax ; "
355 " movl %2,%%edx ; "
356 " popfl"
357 : "=&A" (res) /* 0 */
358 : "m" (*q), /* 1 */
359 "m" (*(q + 1)) /* 2 */
360 : "memory");
361 return (res);
362 }
363
364 static __inline void
atomic_store_rel_64_i386(volatile uint64_t * p,uint64_t v)365 atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v)
366 {
367 volatile uint32_t *q;
368
369 q = (volatile uint32_t *)p;
370 __asm __volatile(
371 " pushfl ; "
372 " cli ; "
373 " movl %%eax,%0 ; "
374 " movl %%edx,%1 ; "
375 " popfl"
376 : "=m" (*q), /* 0 */
377 "=m" (*(q + 1)) /* 1 */
378 : "A" (v) /* 2 */
379 : "memory");
380 }
381
382 static __inline uint64_t
atomic_swap_64_i386(volatile uint64_t * p,uint64_t v)383 atomic_swap_64_i386(volatile uint64_t *p, uint64_t v)
384 {
385 volatile uint32_t *q;
386 uint64_t res;
387
388 q = (volatile uint32_t *)p;
389 __asm __volatile(
390 " pushfl ; "
391 " cli ; "
392 " movl %1,%%eax ; "
393 " movl %2,%%edx ; "
394 " movl %4,%2 ; "
395 " movl %3,%1 ; "
396 " popfl"
397 : "=&A" (res), /* 0 */
398 "+m" (*q), /* 1 */
399 "+m" (*(q + 1)) /* 2 */
400 : "r" ((uint32_t)v), /* 3 */
401 "r" ((uint32_t)(v >> 32))); /* 4 */
402 return (res);
403 }
404
405 static __inline int
atomic_cmpset_64_i586(volatile uint64_t * dst,uint64_t expect,uint64_t src)406 atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src)
407 {
408 u_char res;
409
410 __asm __volatile(
411 " lock; cmpxchg8b %1 ; "
412 " sete %0"
413 : "=q" (res), /* 0 */
414 "+m" (*dst), /* 1 */
415 "+A" (expect) /* 2 */
416 : "b" ((uint32_t)src), /* 3 */
417 "c" ((uint32_t)(src >> 32)) /* 4 */
418 : "memory", "cc");
419 return (res);
420 }
421
422 static __inline int
atomic_fcmpset_64_i586(volatile uint64_t * dst,uint64_t * expect,uint64_t src)423 atomic_fcmpset_64_i586(volatile uint64_t *dst, uint64_t *expect, uint64_t src)
424 {
425 u_char res;
426
427 __asm __volatile(
428 " lock; cmpxchg8b %1 ; "
429 " sete %0"
430 : "=q" (res), /* 0 */
431 "+m" (*dst), /* 1 */
432 "+A" (*expect) /* 2 */
433 : "b" ((uint32_t)src), /* 3 */
434 "c" ((uint32_t)(src >> 32)) /* 4 */
435 : "memory", "cc");
436 return (res);
437 }
438
439 /*
440 * Architecturally always writes back some value to '*p' so will trigger
441 * a #GP(0) on read-only mappings.
442 */
443 static __inline uint64_t
atomic_load_acq_64_i586(const volatile uint64_t * p)444 atomic_load_acq_64_i586(const volatile uint64_t *p)
445 {
446 uint64_t res;
447
448 __asm __volatile(
449 " movl %%ebx,%%eax ; "
450 " movl %%ecx,%%edx ; "
451 " lock; cmpxchg8b %1"
452 : "=&A" (res) /* 0 */
453 : "m" (*p) /* 1 */
454 : "memory", "cc");
455 return (res);
456 }
457
458 static __inline void
atomic_store_rel_64_i586(volatile uint64_t * p,uint64_t v)459 atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v)
460 {
461
462 __asm __volatile(
463 " movl %%eax,%%ebx ; "
464 " movl %%edx,%%ecx ; "
465 "1: "
466 " lock; cmpxchg8b %0 ; "
467 " jne 1b"
468 : "+m" (*p), /* 0 */
469 "+A" (v) /* 1 */
470 : : "ebx", "ecx", "memory", "cc");
471 }
472
473 static __inline uint64_t
atomic_swap_64_i586(volatile uint64_t * p,uint64_t v)474 atomic_swap_64_i586(volatile uint64_t *p, uint64_t v)
475 {
476
477 __asm __volatile(
478 " movl %%eax,%%ebx ; "
479 " movl %%edx,%%ecx ; "
480 "1: "
481 " lock; cmpxchg8b %0 ; "
482 " jne 1b"
483 : "+m" (*p), /* 0 */
484 "+A" (v) /* 1 */
485 : : "ebx", "ecx", "memory", "cc");
486 return (v);
487 }
488
489 static __inline int
atomic_cmpset_64(volatile uint64_t * dst,uint64_t expect,uint64_t src)490 atomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src)
491 {
492
493 if ((cpu_feature & CPUID_CX8) == 0)
494 return (atomic_cmpset_64_i386(dst, expect, src));
495 else
496 return (atomic_cmpset_64_i586(dst, expect, src));
497 }
498
499 static __inline int
atomic_fcmpset_64(volatile uint64_t * dst,uint64_t * expect,uint64_t src)500 atomic_fcmpset_64(volatile uint64_t *dst, uint64_t *expect, uint64_t src)
501 {
502
503 if ((cpu_feature & CPUID_CX8) == 0)
504 return (atomic_fcmpset_64_i386(dst, expect, src));
505 else
506 return (atomic_fcmpset_64_i586(dst, expect, src));
507 }
508
509 static __inline uint64_t
atomic_load_acq_64(const volatile uint64_t * p)510 atomic_load_acq_64(const volatile uint64_t *p)
511 {
512
513 if ((cpu_feature & CPUID_CX8) == 0)
514 return (atomic_load_acq_64_i386(p));
515 else
516 return (atomic_load_acq_64_i586(p));
517 }
518
519 static __inline void
atomic_store_rel_64(volatile uint64_t * p,uint64_t v)520 atomic_store_rel_64(volatile uint64_t *p, uint64_t v)
521 {
522
523 if ((cpu_feature & CPUID_CX8) == 0)
524 atomic_store_rel_64_i386(p, v);
525 else
526 atomic_store_rel_64_i586(p, v);
527 }
528
529 static __inline uint64_t
atomic_swap_64(volatile uint64_t * p,uint64_t v)530 atomic_swap_64(volatile uint64_t *p, uint64_t v)
531 {
532
533 if ((cpu_feature & CPUID_CX8) == 0)
534 return (atomic_swap_64_i386(p, v));
535 else
536 return (atomic_swap_64_i586(p, v));
537 }
538
539 static __inline uint64_t
atomic_fetchadd_64(volatile uint64_t * p,uint64_t v)540 atomic_fetchadd_64(volatile uint64_t *p, uint64_t v)
541 {
542
543 for (;;) {
544 uint64_t t = *p;
545 if (atomic_cmpset_64(p, t, t + v))
546 return (t);
547 }
548 }
549
550 static __inline void
atomic_add_64(volatile uint64_t * p,uint64_t v)551 atomic_add_64(volatile uint64_t *p, uint64_t v)
552 {
553 uint64_t t;
554
555 for (;;) {
556 t = *p;
557 if (atomic_cmpset_64(p, t, t + v))
558 break;
559 }
560 }
561
562 static __inline void
atomic_subtract_64(volatile uint64_t * p,uint64_t v)563 atomic_subtract_64(volatile uint64_t *p, uint64_t v)
564 {
565 uint64_t t;
566
567 for (;;) {
568 t = *p;
569 if (atomic_cmpset_64(p, t, t - v))
570 break;
571 }
572 }
573
574 #endif /* _KERNEL */
575
576 ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v);
577 ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v);
578 ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v);
579 ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v);
580
581 ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v);
582 ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v);
583 ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v);
584 ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v);
585
586 ATOMIC_ASM(set, int, "orl %1,%0", "ir", v);
587 ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v);
588 ATOMIC_ASM(add, int, "addl %1,%0", "ir", v);
589 ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v);
590
591 ATOMIC_ASM(set, long, "orl %1,%0", "ir", v);
592 ATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v);
593 ATOMIC_ASM(add, long, "addl %1,%0", "ir", v);
594 ATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v);
595
596 #define ATOMIC_LOADSTORE(TYPE) \
597 ATOMIC_LOAD(TYPE); \
598 ATOMIC_STORE(TYPE)
599
600 ATOMIC_LOADSTORE(char);
601 ATOMIC_LOADSTORE(short);
602 ATOMIC_LOADSTORE(int);
603 ATOMIC_LOADSTORE(long);
604
605 #undef ATOMIC_ASM
606 #undef ATOMIC_LOAD
607 #undef ATOMIC_STORE
608 #undef ATOMIC_LOADSTORE
609
610 static __inline int
atomic_cmpset_long(volatile u_long * dst,u_long expect,u_long src)611 atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src)
612 {
613
614 return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect,
615 (u_int)src));
616 }
617
618 static __inline int
atomic_fcmpset_long(volatile u_long * dst,u_long * expect,u_long src)619 atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src)
620 {
621
622 return (atomic_fcmpset_int((volatile u_int *)dst, (u_int *)expect,
623 (u_int)src));
624 }
625
626 static __inline u_long
atomic_fetchadd_long(volatile u_long * p,u_long v)627 atomic_fetchadd_long(volatile u_long *p, u_long v)
628 {
629
630 return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v));
631 }
632
633 static __inline int
atomic_testandset_long(volatile u_long * p,u_int v)634 atomic_testandset_long(volatile u_long *p, u_int v)
635 {
636
637 return (atomic_testandset_int((volatile u_int *)p, v));
638 }
639
640 static __inline int
atomic_testandclear_long(volatile u_long * p,u_int v)641 atomic_testandclear_long(volatile u_long *p, u_int v)
642 {
643
644 return (atomic_testandclear_int((volatile u_int *)p, v));
645 }
646
647 /* Read the current value and store a new value in the destination. */
648 static __inline u_int
atomic_swap_int(volatile u_int * p,u_int v)649 atomic_swap_int(volatile u_int *p, u_int v)
650 {
651
652 __asm __volatile(
653 " xchgl %1,%0 ; "
654 "# atomic_swap_int"
655 : "+r" (v), /* 0 */
656 "+m" (*p)); /* 1 */
657 return (v);
658 }
659
660 static __inline u_long
atomic_swap_long(volatile u_long * p,u_long v)661 atomic_swap_long(volatile u_long *p, u_long v)
662 {
663
664 return (atomic_swap_int((volatile u_int *)p, (u_int)v));
665 }
666
667 #define atomic_set_acq_char atomic_set_barr_char
668 #define atomic_set_rel_char atomic_set_barr_char
669 #define atomic_clear_acq_char atomic_clear_barr_char
670 #define atomic_clear_rel_char atomic_clear_barr_char
671 #define atomic_add_acq_char atomic_add_barr_char
672 #define atomic_add_rel_char atomic_add_barr_char
673 #define atomic_subtract_acq_char atomic_subtract_barr_char
674 #define atomic_subtract_rel_char atomic_subtract_barr_char
675 #define atomic_cmpset_acq_char atomic_cmpset_char
676 #define atomic_cmpset_rel_char atomic_cmpset_char
677 #define atomic_fcmpset_acq_char atomic_fcmpset_char
678 #define atomic_fcmpset_rel_char atomic_fcmpset_char
679
680 #define atomic_set_acq_short atomic_set_barr_short
681 #define atomic_set_rel_short atomic_set_barr_short
682 #define atomic_clear_acq_short atomic_clear_barr_short
683 #define atomic_clear_rel_short atomic_clear_barr_short
684 #define atomic_add_acq_short atomic_add_barr_short
685 #define atomic_add_rel_short atomic_add_barr_short
686 #define atomic_subtract_acq_short atomic_subtract_barr_short
687 #define atomic_subtract_rel_short atomic_subtract_barr_short
688 #define atomic_cmpset_acq_short atomic_cmpset_short
689 #define atomic_cmpset_rel_short atomic_cmpset_short
690 #define atomic_fcmpset_acq_short atomic_fcmpset_short
691 #define atomic_fcmpset_rel_short atomic_fcmpset_short
692
693 #define atomic_set_acq_int atomic_set_barr_int
694 #define atomic_set_rel_int atomic_set_barr_int
695 #define atomic_clear_acq_int atomic_clear_barr_int
696 #define atomic_clear_rel_int atomic_clear_barr_int
697 #define atomic_add_acq_int atomic_add_barr_int
698 #define atomic_add_rel_int atomic_add_barr_int
699 #define atomic_subtract_acq_int atomic_subtract_barr_int
700 #define atomic_subtract_rel_int atomic_subtract_barr_int
701 #define atomic_cmpset_acq_int atomic_cmpset_int
702 #define atomic_cmpset_rel_int atomic_cmpset_int
703 #define atomic_fcmpset_acq_int atomic_fcmpset_int
704 #define atomic_fcmpset_rel_int atomic_fcmpset_int
705
706 #define atomic_set_acq_long atomic_set_barr_long
707 #define atomic_set_rel_long atomic_set_barr_long
708 #define atomic_clear_acq_long atomic_clear_barr_long
709 #define atomic_clear_rel_long atomic_clear_barr_long
710 #define atomic_add_acq_long atomic_add_barr_long
711 #define atomic_add_rel_long atomic_add_barr_long
712 #define atomic_subtract_acq_long atomic_subtract_barr_long
713 #define atomic_subtract_rel_long atomic_subtract_barr_long
714 #define atomic_cmpset_acq_long atomic_cmpset_long
715 #define atomic_cmpset_rel_long atomic_cmpset_long
716 #define atomic_fcmpset_acq_long atomic_fcmpset_long
717 #define atomic_fcmpset_rel_long atomic_fcmpset_long
718
719 #define atomic_readandclear_int(p) atomic_swap_int(p, 0)
720 #define atomic_readandclear_long(p) atomic_swap_long(p, 0)
721 #define atomic_testandset_acq_long atomic_testandset_long
722
723 /* Operations on 8-bit bytes. */
724 #define atomic_set_8 atomic_set_char
725 #define atomic_set_acq_8 atomic_set_acq_char
726 #define atomic_set_rel_8 atomic_set_rel_char
727 #define atomic_clear_8 atomic_clear_char
728 #define atomic_clear_acq_8 atomic_clear_acq_char
729 #define atomic_clear_rel_8 atomic_clear_rel_char
730 #define atomic_add_8 atomic_add_char
731 #define atomic_add_acq_8 atomic_add_acq_char
732 #define atomic_add_rel_8 atomic_add_rel_char
733 #define atomic_subtract_8 atomic_subtract_char
734 #define atomic_subtract_acq_8 atomic_subtract_acq_char
735 #define atomic_subtract_rel_8 atomic_subtract_rel_char
736 #define atomic_load_acq_8 atomic_load_acq_char
737 #define atomic_store_rel_8 atomic_store_rel_char
738 #define atomic_cmpset_8 atomic_cmpset_char
739 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char
740 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char
741 #define atomic_fcmpset_8 atomic_fcmpset_char
742 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
743 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
744
745 /* Operations on 16-bit words. */
746 #define atomic_set_16 atomic_set_short
747 #define atomic_set_acq_16 atomic_set_acq_short
748 #define atomic_set_rel_16 atomic_set_rel_short
749 #define atomic_clear_16 atomic_clear_short
750 #define atomic_clear_acq_16 atomic_clear_acq_short
751 #define atomic_clear_rel_16 atomic_clear_rel_short
752 #define atomic_add_16 atomic_add_short
753 #define atomic_add_acq_16 atomic_add_acq_short
754 #define atomic_add_rel_16 atomic_add_rel_short
755 #define atomic_subtract_16 atomic_subtract_short
756 #define atomic_subtract_acq_16 atomic_subtract_acq_short
757 #define atomic_subtract_rel_16 atomic_subtract_rel_short
758 #define atomic_load_acq_16 atomic_load_acq_short
759 #define atomic_store_rel_16 atomic_store_rel_short
760 #define atomic_cmpset_16 atomic_cmpset_short
761 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short
762 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short
763 #define atomic_fcmpset_16 atomic_fcmpset_short
764 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
765 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
766
767 /* Operations on 32-bit double words. */
768 #define atomic_set_32 atomic_set_int
769 #define atomic_set_acq_32 atomic_set_acq_int
770 #define atomic_set_rel_32 atomic_set_rel_int
771 #define atomic_clear_32 atomic_clear_int
772 #define atomic_clear_acq_32 atomic_clear_acq_int
773 #define atomic_clear_rel_32 atomic_clear_rel_int
774 #define atomic_add_32 atomic_add_int
775 #define atomic_add_acq_32 atomic_add_acq_int
776 #define atomic_add_rel_32 atomic_add_rel_int
777 #define atomic_subtract_32 atomic_subtract_int
778 #define atomic_subtract_acq_32 atomic_subtract_acq_int
779 #define atomic_subtract_rel_32 atomic_subtract_rel_int
780 #define atomic_load_acq_32 atomic_load_acq_int
781 #define atomic_store_rel_32 atomic_store_rel_int
782 #define atomic_cmpset_32 atomic_cmpset_int
783 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int
784 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int
785 #define atomic_fcmpset_32 atomic_fcmpset_int
786 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int
787 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int
788 #define atomic_swap_32 atomic_swap_int
789 #define atomic_readandclear_32 atomic_readandclear_int
790 #define atomic_fetchadd_32 atomic_fetchadd_int
791 #define atomic_testandset_32 atomic_testandset_int
792 #define atomic_testandclear_32 atomic_testandclear_int
793
794 #ifdef _KERNEL
795 /* Operations on 64-bit quad words. */
796 #define atomic_cmpset_acq_64 atomic_cmpset_64
797 #define atomic_cmpset_rel_64 atomic_cmpset_64
798 #define atomic_fcmpset_acq_64 atomic_fcmpset_64
799 #define atomic_fcmpset_rel_64 atomic_fcmpset_64
800 #define atomic_fetchadd_acq_64 atomic_fetchadd_64
801 #define atomic_fetchadd_rel_64 atomic_fetchadd_64
802 #define atomic_add_acq_64 atomic_add_64
803 #define atomic_add_rel_64 atomic_add_64
804 #define atomic_subtract_acq_64 atomic_subtract_64
805 #define atomic_subtract_rel_64 atomic_subtract_64
806 #define atomic_load_64 atomic_load_acq_64
807 #define atomic_store_64 atomic_store_rel_64
808 #endif
809
810 /* Operations on pointers. */
811 #define atomic_set_ptr(p, v) \
812 atomic_set_int((volatile u_int *)(p), (u_int)(v))
813 #define atomic_set_acq_ptr(p, v) \
814 atomic_set_acq_int((volatile u_int *)(p), (u_int)(v))
815 #define atomic_set_rel_ptr(p, v) \
816 atomic_set_rel_int((volatile u_int *)(p), (u_int)(v))
817 #define atomic_clear_ptr(p, v) \
818 atomic_clear_int((volatile u_int *)(p), (u_int)(v))
819 #define atomic_clear_acq_ptr(p, v) \
820 atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v))
821 #define atomic_clear_rel_ptr(p, v) \
822 atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v))
823 #define atomic_add_ptr(p, v) \
824 atomic_add_int((volatile u_int *)(p), (u_int)(v))
825 #define atomic_add_acq_ptr(p, v) \
826 atomic_add_acq_int((volatile u_int *)(p), (u_int)(v))
827 #define atomic_add_rel_ptr(p, v) \
828 atomic_add_rel_int((volatile u_int *)(p), (u_int)(v))
829 #define atomic_subtract_ptr(p, v) \
830 atomic_subtract_int((volatile u_int *)(p), (u_int)(v))
831 #define atomic_subtract_acq_ptr(p, v) \
832 atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v))
833 #define atomic_subtract_rel_ptr(p, v) \
834 atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v))
835 #define atomic_load_acq_ptr(p) \
836 atomic_load_acq_int((const volatile u_int *)(p))
837 #define atomic_store_rel_ptr(p, v) \
838 atomic_store_rel_int((volatile u_int *)(p), (v))
839 #define atomic_cmpset_ptr(dst, old, new) \
840 atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
841 #define atomic_cmpset_acq_ptr(dst, old, new) \
842 atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \
843 (u_int)(new))
844 #define atomic_cmpset_rel_ptr(dst, old, new) \
845 atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \
846 (u_int)(new))
847 #define atomic_fcmpset_ptr(dst, old, new) \
848 atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old), (u_int)(new))
849 #define atomic_fcmpset_acq_ptr(dst, old, new) \
850 atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \
851 (u_int)(new))
852 #define atomic_fcmpset_rel_ptr(dst, old, new) \
853 atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \
854 (u_int)(new))
855 #define atomic_swap_ptr(p, v) \
856 atomic_swap_int((volatile u_int *)(p), (u_int)(v))
857 #define atomic_readandclear_ptr(p) \
858 atomic_readandclear_int((volatile u_int *)(p))
859 #define atomic_testandclear_ptr(p, val) \
860 atomic_testandclear_int((volatile u_int *)(p), (val))
861 #define atomic_testandset_ptr(p, val) \
862 atomic_testandset_int((volatile u_int *)(p), (val))
863
864 #if defined(_KERNEL)
865 #define mb() __mbk()
866 #define wmb() __mbk()
867 #define rmb() __mbk()
868 #else
869 #define mb() __mbu()
870 #define wmb() __mbu()
871 #define rmb() __mbu()
872 #endif
873
874 #endif /* !_MACHINE_ATOMIC_H_ */
875