xref: /freebsd/sys/i386/include/atomic.h (revision b71805e991fb955005640bdec81618e37d3af47c)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 1998 Doug Rabson
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 #ifndef _MACHINE_ATOMIC_H_
29 #define	_MACHINE_ATOMIC_H_
30 
31 #include <sys/atomic_common.h>
32 
33 #ifdef _KERNEL
34 #include <machine/md_var.h>
35 #include <machine/specialreg.h>
36 #endif
37 
38 #ifndef __OFFSETOF_MONITORBUF
39 /*
40  * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
41  *
42  * The open-coded number is used instead of the symbolic expression to
43  * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
44  * An assertion in i386/vm_machdep.c ensures that the value is correct.
45  */
46 #define	__OFFSETOF_MONITORBUF	0x80
47 
48 static __inline void
49 __mbk(void)
50 {
51 
52 	__asm __volatile("lock; addl $0,%%fs:%c0"
53 	    : : "i" (__OFFSETOF_MONITORBUF) : "memory", "cc");
54 }
55 
56 static __inline void
57 __mbu(void)
58 {
59 
60 	__asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc");
61 }
62 #endif
63 
64 /*
65  * Various simple operations on memory, each of which is atomic in the
66  * presence of interrupts and multiple processors.
67  *
68  * atomic_set_char(P, V)	(*(u_char *)(P) |= (V))
69  * atomic_clear_char(P, V)	(*(u_char *)(P) &= ~(V))
70  * atomic_add_char(P, V)	(*(u_char *)(P) += (V))
71  * atomic_subtract_char(P, V)	(*(u_char *)(P) -= (V))
72  *
73  * atomic_set_short(P, V)	(*(u_short *)(P) |= (V))
74  * atomic_clear_short(P, V)	(*(u_short *)(P) &= ~(V))
75  * atomic_add_short(P, V)	(*(u_short *)(P) += (V))
76  * atomic_subtract_short(P, V)	(*(u_short *)(P) -= (V))
77  *
78  * atomic_set_int(P, V)		(*(u_int *)(P) |= (V))
79  * atomic_clear_int(P, V)	(*(u_int *)(P) &= ~(V))
80  * atomic_add_int(P, V)		(*(u_int *)(P) += (V))
81  * atomic_subtract_int(P, V)	(*(u_int *)(P) -= (V))
82  * atomic_swap_int(P, V)	(return (*(u_int *)(P)); *(u_int *)(P) = (V);)
83  * atomic_readandclear_int(P)	(return (*(u_int *)(P)); *(u_int *)(P) = 0;)
84  *
85  * atomic_set_long(P, V)	(*(u_long *)(P) |= (V))
86  * atomic_clear_long(P, V)	(*(u_long *)(P) &= ~(V))
87  * atomic_add_long(P, V)	(*(u_long *)(P) += (V))
88  * atomic_subtract_long(P, V)	(*(u_long *)(P) -= (V))
89  * atomic_swap_long(P, V)	(return (*(u_long *)(P)); *(u_long *)(P) = (V);)
90  * atomic_readandclear_long(P)	(return (*(u_long *)(P)); *(u_long *)(P) = 0;)
91  */
92 
93 /*
94  * Always use lock prefixes.  The result is slightly less optimal for
95  * UP systems, but it matters less now, and sometimes UP is emulated
96  * over SMP.
97  *
98  * The assembly is volatilized to avoid code chunk removal by the compiler.
99  * GCC aggressively reorders operations and memory clobbering is necessary
100  * in order to avoid that for memory barriers.
101  */
102 #define	ATOMIC_ASM(NAME, TYPE, OP, CONS, V)		\
103 static __inline void					\
104 atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
105 {							\
106 	__asm __volatile("lock; " OP			\
107 	: "+m" (*p)					\
108 	: CONS (V)					\
109 	: "cc");					\
110 }							\
111 							\
112 static __inline void					\
113 atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
114 {							\
115 	__asm __volatile("lock; " OP			\
116 	: "+m" (*p)					\
117 	: CONS (V)					\
118 	: "memory", "cc");				\
119 }							\
120 struct __hack
121 
122 /*
123  * Atomic compare and set, used by the mutex functions.
124  *
125  * cmpset:
126  *	if (*dst == expect)
127  *		*dst = src
128  *
129  * fcmpset:
130  *	if (*dst == *expect)
131  *		*dst = src
132  *	else
133  *		*expect = *dst
134  *
135  * Returns 0 on failure, non-zero on success.
136  */
137 #define	ATOMIC_CMPSET(TYPE, CONS)			\
138 static __inline int					\
139 atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \
140 {							\
141 	u_char res;					\
142 							\
143 	__asm __volatile(				\
144 	"	lock; cmpxchg	%3,%1 ;	"		\
145 	"	sete	%0 ;		"		\
146 	"# atomic_cmpset_" #TYPE "	"		\
147 	: "=q" (res),			/* 0 */		\
148 	  "+m" (*dst),			/* 1 */		\
149 	  "+a" (expect)			/* 2 */		\
150 	: CONS (src)			/* 3 */		\
151 	: "memory", "cc");				\
152 	return (res);					\
153 }							\
154 							\
155 static __inline int					\
156 atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \
157 {							\
158 	u_char res;					\
159 							\
160 	__asm __volatile(				\
161 	"	lock; cmpxchg	%3,%1 ;	"		\
162 	"	sete	%0 ;		"		\
163 	"# atomic_fcmpset_" #TYPE "	"		\
164 	: "=q" (res),			/* 0 */		\
165 	  "+m" (*dst),			/* 1 */		\
166 	  "+a" (*expect)		/* 2 */		\
167 	: CONS (src)			/* 3 */		\
168 	: "memory", "cc");				\
169 	return (res);					\
170 }
171 
172 ATOMIC_CMPSET(char, "q");
173 ATOMIC_CMPSET(short, "r");
174 ATOMIC_CMPSET(int, "r");
175 
176 /*
177  * Atomically add the value of v to the integer pointed to by p and return
178  * the previous value of *p.
179  */
180 static __inline u_int
181 atomic_fetchadd_int(volatile u_int *p, u_int v)
182 {
183 
184 	__asm __volatile(
185 	"	lock; xaddl	%0,%1 ;	"
186 	"# atomic_fetchadd_int"
187 	: "+r" (v),			/* 0 */
188 	  "+m" (*p)			/* 1 */
189 	: : "cc");
190 	return (v);
191 }
192 
193 static __inline int
194 atomic_testandset_int(volatile u_int *p, u_int v)
195 {
196 	u_char res;
197 
198 	__asm __volatile(
199 	"	lock; btsl	%2,%1 ;	"
200 	"	setc	%0 ;		"
201 	"# atomic_testandset_int"
202 	: "=q" (res),			/* 0 */
203 	  "+m" (*p)			/* 1 */
204 	: "Ir" (v & 0x1f)		/* 2 */
205 	: "cc");
206 	return (res);
207 }
208 
209 static __inline int
210 atomic_testandclear_int(volatile u_int *p, u_int v)
211 {
212 	u_char res;
213 
214 	__asm __volatile(
215 	"	lock; btrl	%2,%1 ;	"
216 	"	setc	%0 ;		"
217 	"# atomic_testandclear_int"
218 	: "=q" (res),			/* 0 */
219 	  "+m" (*p)			/* 1 */
220 	: "Ir" (v & 0x1f)		/* 2 */
221 	: "cc");
222 	return (res);
223 }
224 
225 /*
226  * We assume that a = b will do atomic loads and stores.  Due to the
227  * IA32 memory model, a simple store guarantees release semantics.
228  *
229  * However, a load may pass a store if they are performed on distinct
230  * addresses, so we need Store/Load barrier for sequentially
231  * consistent fences in SMP kernels.  We use "lock addl $0,mem" for a
232  * Store/Load barrier, as recommended by the AMD Software Optimization
233  * Guide, and not mfence.  In the kernel, we use a private per-cpu
234  * cache line for "mem", to avoid introducing false data
235  * dependencies.  In user space, we use the word at the top of the
236  * stack.
237  *
238  * For UP kernels, however, the memory of the single processor is
239  * always consistent, so we only need to stop the compiler from
240  * reordering accesses in a way that violates the semantics of acquire
241  * and release.
242  */
243 
244 #if defined(_KERNEL)
245 #define	__storeload_barrier()	__mbk()
246 #else /* !_KERNEL */
247 #define	__storeload_barrier()	__mbu()
248 #endif /* _KERNEL*/
249 
250 #define	ATOMIC_LOAD(TYPE)					\
251 static __inline u_##TYPE					\
252 atomic_load_acq_##TYPE(const volatile u_##TYPE *p)		\
253 {								\
254 	u_##TYPE res;						\
255 								\
256 	res = *p;						\
257 	__compiler_membar();					\
258 	return (res);						\
259 }								\
260 struct __hack
261 
262 #define	ATOMIC_STORE(TYPE)					\
263 static __inline void						\
264 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)	\
265 {								\
266 								\
267 	__compiler_membar();					\
268 	*p = v;							\
269 }								\
270 struct __hack
271 
272 static __inline void
273 atomic_thread_fence_acq(void)
274 {
275 
276 	__compiler_membar();
277 }
278 
279 static __inline void
280 atomic_thread_fence_rel(void)
281 {
282 
283 	__compiler_membar();
284 }
285 
286 static __inline void
287 atomic_thread_fence_acq_rel(void)
288 {
289 
290 	__compiler_membar();
291 }
292 
293 static __inline void
294 atomic_thread_fence_seq_cst(void)
295 {
296 
297 	__storeload_barrier();
298 }
299 
300 #ifdef _KERNEL
301 
302 /* I486 does not support SMP or CMPXCHG8B. */
303 static __inline int
304 atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src)
305 {
306 	volatile uint32_t *p;
307 	u_char res;
308 
309 	p = (volatile uint32_t *)dst;
310 	__asm __volatile(
311 	"	pushfl ;		"
312 	"	cli ;			"
313 	"	xorl	%1,%%eax ;	"
314 	"	xorl	%2,%%edx ;	"
315 	"	orl	%%edx,%%eax ;	"
316 	"	jne	1f ;		"
317 	"	movl	%4,%1 ;		"
318 	"	movl	%5,%2 ;		"
319 	"1:				"
320 	"	sete	%3 ;		"
321 	"	popfl"
322 	: "+A" (expect),		/* 0 */
323 	  "+m" (*p),			/* 1 */
324 	  "+m" (*(p + 1)),		/* 2 */
325 	  "=q" (res)			/* 3 */
326 	: "r" ((uint32_t)src),		/* 4 */
327 	  "r" ((uint32_t)(src >> 32))	/* 5 */
328 	: "memory", "cc");
329 	return (res);
330 }
331 
332 static __inline int
333 atomic_fcmpset_64_i386(volatile uint64_t *dst, uint64_t *expect, uint64_t src)
334 {
335 
336 	if (atomic_cmpset_64_i386(dst, *expect, src)) {
337 		return (1);
338 	} else {
339 		*expect = *dst;
340 		return (0);
341 	}
342 }
343 
344 static __inline uint64_t
345 atomic_load_acq_64_i386(const volatile uint64_t *p)
346 {
347 	const volatile uint32_t *q;
348 	uint64_t res;
349 
350 	q = (const volatile uint32_t *)p;
351 	__asm __volatile(
352 	"	pushfl ;		"
353 	"	cli ;			"
354 	"	movl	%1,%%eax ;	"
355 	"	movl	%2,%%edx ;	"
356 	"	popfl"
357 	: "=&A" (res)			/* 0 */
358 	: "m" (*q),			/* 1 */
359 	  "m" (*(q + 1))		/* 2 */
360 	: "memory");
361 	return (res);
362 }
363 
364 static __inline void
365 atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v)
366 {
367 	volatile uint32_t *q;
368 
369 	q = (volatile uint32_t *)p;
370 	__asm __volatile(
371 	"	pushfl ;		"
372 	"	cli ;			"
373 	"	movl	%%eax,%0 ;	"
374 	"	movl	%%edx,%1 ;	"
375 	"	popfl"
376 	: "=m" (*q),			/* 0 */
377 	  "=m" (*(q + 1))		/* 1 */
378 	: "A" (v)			/* 2 */
379 	: "memory");
380 }
381 
382 static __inline uint64_t
383 atomic_swap_64_i386(volatile uint64_t *p, uint64_t v)
384 {
385 	volatile uint32_t *q;
386 	uint64_t res;
387 
388 	q = (volatile uint32_t *)p;
389 	__asm __volatile(
390 	"	pushfl ;		"
391 	"	cli ;			"
392 	"	movl	%1,%%eax ;	"
393 	"	movl	%2,%%edx ;	"
394 	"	movl	%4,%2 ;		"
395 	"	movl	%3,%1 ;		"
396 	"	popfl"
397 	: "=&A" (res),			/* 0 */
398 	  "+m" (*q),			/* 1 */
399 	  "+m" (*(q + 1))		/* 2 */
400 	: "r" ((uint32_t)v),		/* 3 */
401 	  "r" ((uint32_t)(v >> 32)));	/* 4 */
402 	return (res);
403 }
404 
405 static __inline int
406 atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src)
407 {
408 	u_char res;
409 
410 	__asm __volatile(
411 	"	lock; cmpxchg8b %1 ;	"
412 	"	sete	%0"
413 	: "=q" (res),			/* 0 */
414 	  "+m" (*dst),			/* 1 */
415 	  "+A" (expect)			/* 2 */
416 	: "b" ((uint32_t)src),		/* 3 */
417 	  "c" ((uint32_t)(src >> 32))	/* 4 */
418 	: "memory", "cc");
419 	return (res);
420 }
421 
422 static __inline int
423 atomic_fcmpset_64_i586(volatile uint64_t *dst, uint64_t *expect, uint64_t src)
424 {
425 	u_char res;
426 
427 	__asm __volatile(
428 	"	lock; cmpxchg8b %1 ;	"
429 	"	sete	%0"
430 	: "=q" (res),			/* 0 */
431 	  "+m" (*dst),			/* 1 */
432 	  "+A" (*expect)		/* 2 */
433 	: "b" ((uint32_t)src),		/* 3 */
434 	  "c" ((uint32_t)(src >> 32))	/* 4 */
435 	: "memory", "cc");
436 	return (res);
437 }
438 
439 /*
440  * Architecturally always writes back some value to '*p' so will trigger
441  * a #GP(0) on read-only mappings.
442  */
443 static __inline uint64_t
444 atomic_load_acq_64_i586(const volatile uint64_t *p)
445 {
446 	uint64_t res;
447 
448 	__asm __volatile(
449 	"	movl	%%ebx,%%eax ;	"
450 	"	movl	%%ecx,%%edx ;	"
451 	"	lock; cmpxchg8b %1"
452 	: "=&A" (res)			/* 0 */
453 	: "m" (*p)			/* 1 */
454 	: "memory", "cc");
455 	return (res);
456 }
457 
458 static __inline void
459 atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v)
460 {
461 
462 	__asm __volatile(
463 	"	movl	%%eax,%%ebx ;	"
464 	"	movl	%%edx,%%ecx ;	"
465 	"1:				"
466 	"	lock; cmpxchg8b %0 ;	"
467 	"	jne	1b"
468 	: "+m" (*p),			/* 0 */
469 	  "+A" (v)			/* 1 */
470 	: : "ebx", "ecx", "memory", "cc");
471 }
472 
473 static __inline uint64_t
474 atomic_swap_64_i586(volatile uint64_t *p, uint64_t v)
475 {
476 
477 	__asm __volatile(
478 	"	movl	%%eax,%%ebx ;	"
479 	"	movl	%%edx,%%ecx ;	"
480 	"1:				"
481 	"	lock; cmpxchg8b %0 ;	"
482 	"	jne	1b"
483 	: "+m" (*p),			/* 0 */
484 	  "+A" (v)			/* 1 */
485 	: : "ebx", "ecx", "memory", "cc");
486 	return (v);
487 }
488 
489 static __inline int
490 atomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src)
491 {
492 
493 	if ((cpu_feature & CPUID_CX8) == 0)
494 		return (atomic_cmpset_64_i386(dst, expect, src));
495 	else
496 		return (atomic_cmpset_64_i586(dst, expect, src));
497 }
498 
499 static __inline int
500 atomic_fcmpset_64(volatile uint64_t *dst, uint64_t *expect, uint64_t src)
501 {
502 
503   	if ((cpu_feature & CPUID_CX8) == 0)
504 		return (atomic_fcmpset_64_i386(dst, expect, src));
505 	else
506 		return (atomic_fcmpset_64_i586(dst, expect, src));
507 }
508 
509 static __inline uint64_t
510 atomic_load_acq_64(const volatile uint64_t *p)
511 {
512 
513 	if ((cpu_feature & CPUID_CX8) == 0)
514 		return (atomic_load_acq_64_i386(p));
515 	else
516 		return (atomic_load_acq_64_i586(p));
517 }
518 
519 static __inline void
520 atomic_store_rel_64(volatile uint64_t *p, uint64_t v)
521 {
522 
523 	if ((cpu_feature & CPUID_CX8) == 0)
524 		atomic_store_rel_64_i386(p, v);
525 	else
526 		atomic_store_rel_64_i586(p, v);
527 }
528 
529 static __inline uint64_t
530 atomic_swap_64(volatile uint64_t *p, uint64_t v)
531 {
532 
533 	if ((cpu_feature & CPUID_CX8) == 0)
534 		return (atomic_swap_64_i386(p, v));
535 	else
536 		return (atomic_swap_64_i586(p, v));
537 }
538 
539 static __inline uint64_t
540 atomic_fetchadd_64(volatile uint64_t *p, uint64_t v)
541 {
542 
543 	for (;;) {
544 		uint64_t t = *p;
545 		if (atomic_cmpset_64(p, t, t + v))
546 			return (t);
547 	}
548 }
549 
550 static __inline void
551 atomic_add_64(volatile uint64_t *p, uint64_t v)
552 {
553 	uint64_t t;
554 
555 	for (;;) {
556 		t = *p;
557 		if (atomic_cmpset_64(p, t, t + v))
558 			break;
559 	}
560 }
561 
562 static __inline void
563 atomic_subtract_64(volatile uint64_t *p, uint64_t v)
564 {
565 	uint64_t t;
566 
567 	for (;;) {
568 		t = *p;
569 		if (atomic_cmpset_64(p, t, t - v))
570 			break;
571 	}
572 }
573 
574 #endif /* _KERNEL */
575 
576 ATOMIC_ASM(set,	     char,  "orb %b1,%0",  "iq",  v);
577 ATOMIC_ASM(clear,    char,  "andb %b1,%0", "iq", ~v);
578 ATOMIC_ASM(add,	     char,  "addb %b1,%0", "iq",  v);
579 ATOMIC_ASM(subtract, char,  "subb %b1,%0", "iq",  v);
580 
581 ATOMIC_ASM(set,	     short, "orw %w1,%0",  "ir",  v);
582 ATOMIC_ASM(clear,    short, "andw %w1,%0", "ir", ~v);
583 ATOMIC_ASM(add,	     short, "addw %w1,%0", "ir",  v);
584 ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir",  v);
585 
586 ATOMIC_ASM(set,	     int,   "orl %1,%0",   "ir",  v);
587 ATOMIC_ASM(clear,    int,   "andl %1,%0",  "ir", ~v);
588 ATOMIC_ASM(add,	     int,   "addl %1,%0",  "ir",  v);
589 ATOMIC_ASM(subtract, int,   "subl %1,%0",  "ir",  v);
590 
591 ATOMIC_ASM(set,	     long,  "orl %1,%0",   "ir",  v);
592 ATOMIC_ASM(clear,    long,  "andl %1,%0",  "ir", ~v);
593 ATOMIC_ASM(add,	     long,  "addl %1,%0",  "ir",  v);
594 ATOMIC_ASM(subtract, long,  "subl %1,%0",  "ir",  v);
595 
596 #define	ATOMIC_LOADSTORE(TYPE)				\
597 	ATOMIC_LOAD(TYPE);				\
598 	ATOMIC_STORE(TYPE)
599 
600 ATOMIC_LOADSTORE(char);
601 ATOMIC_LOADSTORE(short);
602 ATOMIC_LOADSTORE(int);
603 ATOMIC_LOADSTORE(long);
604 
605 #undef ATOMIC_ASM
606 #undef ATOMIC_LOAD
607 #undef ATOMIC_STORE
608 #undef ATOMIC_LOADSTORE
609 
610 static __inline int
611 atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src)
612 {
613 
614 	return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect,
615 	    (u_int)src));
616 }
617 
618 static __inline int
619 atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src)
620 {
621 
622 	return (atomic_fcmpset_int((volatile u_int *)dst, (u_int *)expect,
623 	    (u_int)src));
624 }
625 
626 static __inline u_long
627 atomic_fetchadd_long(volatile u_long *p, u_long v)
628 {
629 
630 	return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v));
631 }
632 
633 static __inline int
634 atomic_testandset_long(volatile u_long *p, u_int v)
635 {
636 
637 	return (atomic_testandset_int((volatile u_int *)p, v));
638 }
639 
640 static __inline int
641 atomic_testandclear_long(volatile u_long *p, u_int v)
642 {
643 
644 	return (atomic_testandclear_int((volatile u_int *)p, v));
645 }
646 
647 /* Read the current value and store a new value in the destination. */
648 static __inline u_int
649 atomic_swap_int(volatile u_int *p, u_int v)
650 {
651 
652 	__asm __volatile(
653 	"	xchgl	%1,%0 ;		"
654 	"# atomic_swap_int"
655 	: "+r" (v),			/* 0 */
656 	  "+m" (*p));			/* 1 */
657 	return (v);
658 }
659 
660 static __inline u_long
661 atomic_swap_long(volatile u_long *p, u_long v)
662 {
663 
664 	return (atomic_swap_int((volatile u_int *)p, (u_int)v));
665 }
666 
667 #define	atomic_set_acq_char		atomic_set_barr_char
668 #define	atomic_set_rel_char		atomic_set_barr_char
669 #define	atomic_clear_acq_char		atomic_clear_barr_char
670 #define	atomic_clear_rel_char		atomic_clear_barr_char
671 #define	atomic_add_acq_char		atomic_add_barr_char
672 #define	atomic_add_rel_char		atomic_add_barr_char
673 #define	atomic_subtract_acq_char	atomic_subtract_barr_char
674 #define	atomic_subtract_rel_char	atomic_subtract_barr_char
675 #define	atomic_cmpset_acq_char		atomic_cmpset_char
676 #define	atomic_cmpset_rel_char		atomic_cmpset_char
677 #define	atomic_fcmpset_acq_char		atomic_fcmpset_char
678 #define	atomic_fcmpset_rel_char		atomic_fcmpset_char
679 
680 #define	atomic_set_acq_short		atomic_set_barr_short
681 #define	atomic_set_rel_short		atomic_set_barr_short
682 #define	atomic_clear_acq_short		atomic_clear_barr_short
683 #define	atomic_clear_rel_short		atomic_clear_barr_short
684 #define	atomic_add_acq_short		atomic_add_barr_short
685 #define	atomic_add_rel_short		atomic_add_barr_short
686 #define	atomic_subtract_acq_short	atomic_subtract_barr_short
687 #define	atomic_subtract_rel_short	atomic_subtract_barr_short
688 #define	atomic_cmpset_acq_short		atomic_cmpset_short
689 #define	atomic_cmpset_rel_short		atomic_cmpset_short
690 #define	atomic_fcmpset_acq_short	atomic_fcmpset_short
691 #define	atomic_fcmpset_rel_short	atomic_fcmpset_short
692 
693 #define	atomic_set_acq_int		atomic_set_barr_int
694 #define	atomic_set_rel_int		atomic_set_barr_int
695 #define	atomic_clear_acq_int		atomic_clear_barr_int
696 #define	atomic_clear_rel_int		atomic_clear_barr_int
697 #define	atomic_add_acq_int		atomic_add_barr_int
698 #define	atomic_add_rel_int		atomic_add_barr_int
699 #define	atomic_subtract_acq_int		atomic_subtract_barr_int
700 #define	atomic_subtract_rel_int		atomic_subtract_barr_int
701 #define	atomic_cmpset_acq_int		atomic_cmpset_int
702 #define	atomic_cmpset_rel_int		atomic_cmpset_int
703 #define	atomic_fcmpset_acq_int		atomic_fcmpset_int
704 #define	atomic_fcmpset_rel_int		atomic_fcmpset_int
705 
706 #define	atomic_set_acq_long		atomic_set_barr_long
707 #define	atomic_set_rel_long		atomic_set_barr_long
708 #define	atomic_clear_acq_long		atomic_clear_barr_long
709 #define	atomic_clear_rel_long		atomic_clear_barr_long
710 #define	atomic_add_acq_long		atomic_add_barr_long
711 #define	atomic_add_rel_long		atomic_add_barr_long
712 #define	atomic_subtract_acq_long	atomic_subtract_barr_long
713 #define	atomic_subtract_rel_long	atomic_subtract_barr_long
714 #define	atomic_cmpset_acq_long		atomic_cmpset_long
715 #define	atomic_cmpset_rel_long		atomic_cmpset_long
716 #define	atomic_fcmpset_acq_long		atomic_fcmpset_long
717 #define	atomic_fcmpset_rel_long		atomic_fcmpset_long
718 
719 #define	atomic_readandclear_int(p)	atomic_swap_int(p, 0)
720 #define	atomic_readandclear_long(p)	atomic_swap_long(p, 0)
721 #define	atomic_testandset_acq_long	atomic_testandset_long
722 
723 /* Operations on 8-bit bytes. */
724 #define	atomic_set_8		atomic_set_char
725 #define	atomic_set_acq_8	atomic_set_acq_char
726 #define	atomic_set_rel_8	atomic_set_rel_char
727 #define	atomic_clear_8		atomic_clear_char
728 #define	atomic_clear_acq_8	atomic_clear_acq_char
729 #define	atomic_clear_rel_8	atomic_clear_rel_char
730 #define	atomic_add_8		atomic_add_char
731 #define	atomic_add_acq_8	atomic_add_acq_char
732 #define	atomic_add_rel_8	atomic_add_rel_char
733 #define	atomic_subtract_8	atomic_subtract_char
734 #define	atomic_subtract_acq_8	atomic_subtract_acq_char
735 #define	atomic_subtract_rel_8	atomic_subtract_rel_char
736 #define	atomic_load_acq_8	atomic_load_acq_char
737 #define	atomic_store_rel_8	atomic_store_rel_char
738 #define	atomic_cmpset_8		atomic_cmpset_char
739 #define	atomic_cmpset_acq_8	atomic_cmpset_acq_char
740 #define	atomic_cmpset_rel_8	atomic_cmpset_rel_char
741 #define	atomic_fcmpset_8	atomic_fcmpset_char
742 #define	atomic_fcmpset_acq_8	atomic_fcmpset_acq_char
743 #define	atomic_fcmpset_rel_8	atomic_fcmpset_rel_char
744 
745 /* Operations on 16-bit words. */
746 #define	atomic_set_16		atomic_set_short
747 #define	atomic_set_acq_16	atomic_set_acq_short
748 #define	atomic_set_rel_16	atomic_set_rel_short
749 #define	atomic_clear_16		atomic_clear_short
750 #define	atomic_clear_acq_16	atomic_clear_acq_short
751 #define	atomic_clear_rel_16	atomic_clear_rel_short
752 #define	atomic_add_16		atomic_add_short
753 #define	atomic_add_acq_16	atomic_add_acq_short
754 #define	atomic_add_rel_16	atomic_add_rel_short
755 #define	atomic_subtract_16	atomic_subtract_short
756 #define	atomic_subtract_acq_16	atomic_subtract_acq_short
757 #define	atomic_subtract_rel_16	atomic_subtract_rel_short
758 #define	atomic_load_acq_16	atomic_load_acq_short
759 #define	atomic_store_rel_16	atomic_store_rel_short
760 #define	atomic_cmpset_16	atomic_cmpset_short
761 #define	atomic_cmpset_acq_16	atomic_cmpset_acq_short
762 #define	atomic_cmpset_rel_16	atomic_cmpset_rel_short
763 #define	atomic_fcmpset_16	atomic_fcmpset_short
764 #define	atomic_fcmpset_acq_16	atomic_fcmpset_acq_short
765 #define	atomic_fcmpset_rel_16	atomic_fcmpset_rel_short
766 
767 /* Operations on 32-bit double words. */
768 #define	atomic_set_32		atomic_set_int
769 #define	atomic_set_acq_32	atomic_set_acq_int
770 #define	atomic_set_rel_32	atomic_set_rel_int
771 #define	atomic_clear_32		atomic_clear_int
772 #define	atomic_clear_acq_32	atomic_clear_acq_int
773 #define	atomic_clear_rel_32	atomic_clear_rel_int
774 #define	atomic_add_32		atomic_add_int
775 #define	atomic_add_acq_32	atomic_add_acq_int
776 #define	atomic_add_rel_32	atomic_add_rel_int
777 #define	atomic_subtract_32	atomic_subtract_int
778 #define	atomic_subtract_acq_32	atomic_subtract_acq_int
779 #define	atomic_subtract_rel_32	atomic_subtract_rel_int
780 #define	atomic_load_acq_32	atomic_load_acq_int
781 #define	atomic_store_rel_32	atomic_store_rel_int
782 #define	atomic_cmpset_32	atomic_cmpset_int
783 #define	atomic_cmpset_acq_32	atomic_cmpset_acq_int
784 #define	atomic_cmpset_rel_32	atomic_cmpset_rel_int
785 #define	atomic_fcmpset_32	atomic_fcmpset_int
786 #define	atomic_fcmpset_acq_32	atomic_fcmpset_acq_int
787 #define	atomic_fcmpset_rel_32	atomic_fcmpset_rel_int
788 #define	atomic_swap_32		atomic_swap_int
789 #define	atomic_readandclear_32	atomic_readandclear_int
790 #define	atomic_fetchadd_32	atomic_fetchadd_int
791 #define	atomic_testandset_32	atomic_testandset_int
792 #define	atomic_testandclear_32	atomic_testandclear_int
793 
794 #ifdef _KERNEL
795 /* Operations on 64-bit quad words. */
796 #define	atomic_cmpset_acq_64 atomic_cmpset_64
797 #define	atomic_cmpset_rel_64 atomic_cmpset_64
798 #define	atomic_fcmpset_acq_64 atomic_fcmpset_64
799 #define	atomic_fcmpset_rel_64 atomic_fcmpset_64
800 #define	atomic_fetchadd_acq_64	atomic_fetchadd_64
801 #define	atomic_fetchadd_rel_64	atomic_fetchadd_64
802 #define	atomic_add_acq_64 atomic_add_64
803 #define	atomic_add_rel_64 atomic_add_64
804 #define	atomic_subtract_acq_64 atomic_subtract_64
805 #define	atomic_subtract_rel_64 atomic_subtract_64
806 #define	atomic_load_64 atomic_load_acq_64
807 #define	atomic_store_64 atomic_store_rel_64
808 #endif
809 
810 /* Operations on pointers. */
811 #define	atomic_set_ptr(p, v) \
812 	atomic_set_int((volatile u_int *)(p), (u_int)(v))
813 #define	atomic_set_acq_ptr(p, v) \
814 	atomic_set_acq_int((volatile u_int *)(p), (u_int)(v))
815 #define	atomic_set_rel_ptr(p, v) \
816 	atomic_set_rel_int((volatile u_int *)(p), (u_int)(v))
817 #define	atomic_clear_ptr(p, v) \
818 	atomic_clear_int((volatile u_int *)(p), (u_int)(v))
819 #define	atomic_clear_acq_ptr(p, v) \
820 	atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v))
821 #define	atomic_clear_rel_ptr(p, v) \
822 	atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v))
823 #define	atomic_add_ptr(p, v) \
824 	atomic_add_int((volatile u_int *)(p), (u_int)(v))
825 #define	atomic_add_acq_ptr(p, v) \
826 	atomic_add_acq_int((volatile u_int *)(p), (u_int)(v))
827 #define	atomic_add_rel_ptr(p, v) \
828 	atomic_add_rel_int((volatile u_int *)(p), (u_int)(v))
829 #define	atomic_subtract_ptr(p, v) \
830 	atomic_subtract_int((volatile u_int *)(p), (u_int)(v))
831 #define	atomic_subtract_acq_ptr(p, v) \
832 	atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v))
833 #define	atomic_subtract_rel_ptr(p, v) \
834 	atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v))
835 #define	atomic_load_acq_ptr(p) \
836 	atomic_load_acq_int((const volatile u_int *)(p))
837 #define	atomic_store_rel_ptr(p, v) \
838 	atomic_store_rel_int((volatile u_int *)(p), (v))
839 #define	atomic_cmpset_ptr(dst, old, new) \
840 	atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
841 #define	atomic_cmpset_acq_ptr(dst, old, new) \
842 	atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \
843 	    (u_int)(new))
844 #define	atomic_cmpset_rel_ptr(dst, old, new) \
845 	atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \
846 	    (u_int)(new))
847 #define	atomic_fcmpset_ptr(dst, old, new) \
848 	atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old), (u_int)(new))
849 #define	atomic_fcmpset_acq_ptr(dst, old, new) \
850 	atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \
851 	    (u_int)(new))
852 #define	atomic_fcmpset_rel_ptr(dst, old, new) \
853 	atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \
854 	    (u_int)(new))
855 #define	atomic_swap_ptr(p, v) \
856 	atomic_swap_int((volatile u_int *)(p), (u_int)(v))
857 #define	atomic_readandclear_ptr(p) \
858 	atomic_readandclear_int((volatile u_int *)(p))
859 #define	atomic_testandclear_ptr(p, val) \
860 	atomic_testandclear_int((volatile u_int *)(p), (val))
861 #define	atomic_testandset_ptr(p, val) \
862 	atomic_testandset_int((volatile u_int *)(p), (val))
863 
864 #if defined(_KERNEL)
865 #define	mb()	__mbk()
866 #define	wmb()	__mbk()
867 #define	rmb()	__mbk()
868 #else
869 #define	mb()	__mbu()
870 #define	wmb()	__mbu()
871 #define	rmb()	__mbu()
872 #endif
873 
874 #endif /* !_MACHINE_ATOMIC_H_ */
875