xref: /freebsd/sys/i386/include/atomic.h (revision 4c5aee92a79d1b794c5f1ff76a79242421427e98)
1069e9bc1SDoug Rabson /*-
2069e9bc1SDoug Rabson  * Copyright (c) 1998 Doug Rabson
3069e9bc1SDoug Rabson  * All rights reserved.
4069e9bc1SDoug Rabson  *
5069e9bc1SDoug Rabson  * Redistribution and use in source and binary forms, with or without
6069e9bc1SDoug Rabson  * modification, are permitted provided that the following conditions
7069e9bc1SDoug Rabson  * are met:
8069e9bc1SDoug Rabson  * 1. Redistributions of source code must retain the above copyright
9069e9bc1SDoug Rabson  *    notice, this list of conditions and the following disclaimer.
10069e9bc1SDoug Rabson  * 2. Redistributions in binary form must reproduce the above copyright
11069e9bc1SDoug Rabson  *    notice, this list of conditions and the following disclaimer in the
12069e9bc1SDoug Rabson  *    documentation and/or other materials provided with the distribution.
13069e9bc1SDoug Rabson  *
14069e9bc1SDoug Rabson  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15069e9bc1SDoug Rabson  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16069e9bc1SDoug Rabson  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17069e9bc1SDoug Rabson  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18069e9bc1SDoug Rabson  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19069e9bc1SDoug Rabson  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20069e9bc1SDoug Rabson  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21069e9bc1SDoug Rabson  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22069e9bc1SDoug Rabson  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23069e9bc1SDoug Rabson  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24069e9bc1SDoug Rabson  * SUCH DAMAGE.
25069e9bc1SDoug Rabson  *
26c3aac50fSPeter Wemm  * $FreeBSD$
27069e9bc1SDoug Rabson  */
28069e9bc1SDoug Rabson #ifndef _MACHINE_ATOMIC_H_
29069e9bc1SDoug Rabson #define _MACHINE_ATOMIC_H_
30069e9bc1SDoug Rabson 
31069e9bc1SDoug Rabson /*
32069e9bc1SDoug Rabson  * Various simple arithmetic on memory which is atomic in the presence
3347b8bc92SAlan Cox  * of interrupts and multiple processors.
34069e9bc1SDoug Rabson  *
3547b8bc92SAlan Cox  * atomic_set_char(P, V)	(*(u_char*)(P) |= (V))
3647b8bc92SAlan Cox  * atomic_clear_char(P, V)	(*(u_char*)(P) &= ~(V))
3747b8bc92SAlan Cox  * atomic_add_char(P, V)	(*(u_char*)(P) += (V))
3847b8bc92SAlan Cox  * atomic_subtract_char(P, V)	(*(u_char*)(P) -= (V))
3947b8bc92SAlan Cox  *
4047b8bc92SAlan Cox  * atomic_set_short(P, V)	(*(u_short*)(P) |= (V))
4147b8bc92SAlan Cox  * atomic_clear_short(P, V)	(*(u_short*)(P) &= ~(V))
4247b8bc92SAlan Cox  * atomic_add_short(P, V)	(*(u_short*)(P) += (V))
4347b8bc92SAlan Cox  * atomic_subtract_short(P, V)	(*(u_short*)(P) -= (V))
4447b8bc92SAlan Cox  *
4547b8bc92SAlan Cox  * atomic_set_int(P, V)		(*(u_int*)(P) |= (V))
4647b8bc92SAlan Cox  * atomic_clear_int(P, V)	(*(u_int*)(P) &= ~(V))
4747b8bc92SAlan Cox  * atomic_add_int(P, V)		(*(u_int*)(P) += (V))
4847b8bc92SAlan Cox  * atomic_subtract_int(P, V)	(*(u_int*)(P) -= (V))
49b4645202SJohn Baldwin  * atomic_readandclear_int(P)	(return  *(u_int*)P; *(u_int*)P = 0;)
5047b8bc92SAlan Cox  *
5147b8bc92SAlan Cox  * atomic_set_long(P, V)	(*(u_long*)(P) |= (V))
5247b8bc92SAlan Cox  * atomic_clear_long(P, V)	(*(u_long*)(P) &= ~(V))
5347b8bc92SAlan Cox  * atomic_add_long(P, V)	(*(u_long*)(P) += (V))
5447b8bc92SAlan Cox  * atomic_subtract_long(P, V)	(*(u_long*)(P) -= (V))
55b4645202SJohn Baldwin  * atomic_readandclear_long(P)	(return  *(u_long*)P; *(u_long*)P = 0;)
56069e9bc1SDoug Rabson  */
57069e9bc1SDoug Rabson 
5847b8bc92SAlan Cox /*
5908c40841SAlan Cox  * The above functions are expanded inline in the statically-linked
6008c40841SAlan Cox  * kernel.  Lock prefixes are generated if an SMP kernel is being
6108c40841SAlan Cox  * built.
6208c40841SAlan Cox  *
6308c40841SAlan Cox  * Kernel modules call real functions which are built into the kernel.
6408c40841SAlan Cox  * This allows kernel modules to be portable between UP and SMP systems.
6547b8bc92SAlan Cox  */
6608c40841SAlan Cox #if defined(KLD_MODULE)
67e4e991e1SJohn Baldwin #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V)			\
688306a37bSMark Murray void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
6908c40841SAlan Cox 
70b4645202SJohn Baldwin int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src);
71819e370cSPoul-Henning Kamp 
728a6b1c8fSJohn Baldwin #define	ATOMIC_STORE_LOAD(TYPE, LOP, SOP)			\
738a6b1c8fSJohn Baldwin u_##TYPE	atomic_load_acq_##TYPE(volatile u_##TYPE *p);	\
748306a37bSMark Murray void		atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
758a6b1c8fSJohn Baldwin 
7608c40841SAlan Cox #else /* !KLD_MODULE */
77d888fc4eSMark Murray 
784c5aee92SMark Murray #ifdef __GNUC__
794c5aee92SMark Murray 
802a89a48fSJohn Baldwin /*
812a89a48fSJohn Baldwin  * For userland, assume the SMP case and use lock prefixes so that
822a89a48fSJohn Baldwin  * the binaries will run on both types of systems.
832a89a48fSJohn Baldwin  */
842a89a48fSJohn Baldwin #if defined(SMP) || !defined(_KERNEL)
8571acb247SBosko Milekic #define MPLOCKED	lock ;
86d2f22d70SBruce Evans #else
8747b8bc92SAlan Cox #define MPLOCKED
88d2f22d70SBruce Evans #endif
89069e9bc1SDoug Rabson 
9047b8bc92SAlan Cox /*
9147b8bc92SAlan Cox  * The assembly is volatilized to demark potential before-and-after side
9247b8bc92SAlan Cox  * effects if an interrupt or SMP collision were to occur.
9347b8bc92SAlan Cox  */
94e4e991e1SJohn Baldwin #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V)		\
9547b8bc92SAlan Cox static __inline void					\
9603e3bc8eSAlan Cox atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
9747b8bc92SAlan Cox {							\
9871acb247SBosko Milekic 	__asm __volatile(__XSTRING(MPLOCKED) OP		\
9920a2016aSJohn Baldwin 			 : "+m" (*p)			\
100e4e991e1SJohn Baldwin 			 : CONS (V));			\
10147b8bc92SAlan Cox }
1024c5aee92SMark Murray 
1038306a37bSMark Murray #else /* !__GNUC__ */
1044c5aee92SMark Murray 
1058306a37bSMark Murray #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V)				\
1064c5aee92SMark Murray extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
1074c5aee92SMark Murray 
1088306a37bSMark Murray #endif /* __GNUC__ */
109693612ebSPeter Wemm 
110819e370cSPoul-Henning Kamp /*
111819e370cSPoul-Henning Kamp  * Atomic compare and set, used by the mutex functions
112819e370cSPoul-Henning Kamp  *
113819e370cSPoul-Henning Kamp  * if (*dst == exp) *dst = src (all 32 bit words)
114819e370cSPoul-Henning Kamp  *
115819e370cSPoul-Henning Kamp  * Returns 0 on failure, non-zero on success
116819e370cSPoul-Henning Kamp  */
117819e370cSPoul-Henning Kamp 
1188306a37bSMark Murray #if defined(__GNUC__)
1194c5aee92SMark Murray 
120819e370cSPoul-Henning Kamp #if defined(I386_CPU)
1214c5aee92SMark Murray 
122819e370cSPoul-Henning Kamp static __inline int
123819e370cSPoul-Henning Kamp atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src)
124819e370cSPoul-Henning Kamp {
125819e370cSPoul-Henning Kamp 	int res = exp;
126819e370cSPoul-Henning Kamp 
127819e370cSPoul-Henning Kamp 	__asm __volatile(
128819e370cSPoul-Henning Kamp 	"	pushfl ;		"
129819e370cSPoul-Henning Kamp 	"	cli ;			"
130d9009094SJohn Baldwin 	"	cmpl	%0,%2 ;		"
131819e370cSPoul-Henning Kamp 	"	jne	1f ;		"
132d9009094SJohn Baldwin 	"	movl	%1,%2 ;		"
133819e370cSPoul-Henning Kamp 	"1:				"
134819e370cSPoul-Henning Kamp 	"       sete	%%al;		"
135819e370cSPoul-Henning Kamp 	"	movzbl	%%al,%0 ;	"
136819e370cSPoul-Henning Kamp 	"	popfl ;			"
137819e370cSPoul-Henning Kamp 	"# atomic_cmpset_int"
138d9009094SJohn Baldwin 	: "+a" (res)			/* 0 (result) */
139d9009094SJohn Baldwin 	: "r" (src),			/* 1 */
140d9009094SJohn Baldwin 	  "m" (*(dst))			/* 2 */
141819e370cSPoul-Henning Kamp 	: "memory");
142819e370cSPoul-Henning Kamp 
143819e370cSPoul-Henning Kamp 	return (res);
144819e370cSPoul-Henning Kamp }
1454c5aee92SMark Murray 
146819e370cSPoul-Henning Kamp #else /* defined(I386_CPU) */
1474c5aee92SMark Murray 
148819e370cSPoul-Henning Kamp static __inline int
149819e370cSPoul-Henning Kamp atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src)
150819e370cSPoul-Henning Kamp {
151819e370cSPoul-Henning Kamp 	int res = exp;
152819e370cSPoul-Henning Kamp 
153819e370cSPoul-Henning Kamp 	__asm __volatile (
15471acb247SBosko Milekic 	"	" __XSTRING(MPLOCKED) "	"
155d9009094SJohn Baldwin 	"	cmpxchgl %1,%2 ;	"
156819e370cSPoul-Henning Kamp 	"       setz	%%al ;		"
157819e370cSPoul-Henning Kamp 	"	movzbl	%%al,%0 ;	"
158819e370cSPoul-Henning Kamp 	"1:				"
159819e370cSPoul-Henning Kamp 	"# atomic_cmpset_int"
160d9009094SJohn Baldwin 	: "+a" (res)			/* 0 (result) */
161d9009094SJohn Baldwin 	: "r" (src),			/* 1 */
162d9009094SJohn Baldwin 	  "m" (*(dst))			/* 2 */
163819e370cSPoul-Henning Kamp 	: "memory");
164819e370cSPoul-Henning Kamp 
165819e370cSPoul-Henning Kamp 	return (res);
166819e370cSPoul-Henning Kamp }
1674c5aee92SMark Murray 
168819e370cSPoul-Henning Kamp #endif /* defined(I386_CPU) */
1694c5aee92SMark Murray 
1708306a37bSMark Murray #endif /* defined(__GNUC__) */
171819e370cSPoul-Henning Kamp 
1728306a37bSMark Murray #if defined(__GNUC__)
1734c5aee92SMark Murray 
1749d979d89SJohn Baldwin #if defined(I386_CPU)
1754c5aee92SMark Murray 
176ccbdd9eeSJohn Baldwin /*
177ccbdd9eeSJohn Baldwin  * We assume that a = b will do atomic loads and stores.
1789d979d89SJohn Baldwin  *
1799d979d89SJohn Baldwin  * XXX: This is _NOT_ safe on a P6 or higher because it does not guarantee
1809d979d89SJohn Baldwin  * memory ordering.  These should only be used on a 386.
181ccbdd9eeSJohn Baldwin  */
1829d979d89SJohn Baldwin #define ATOMIC_STORE_LOAD(TYPE, LOP, SOP)		\
183ccbdd9eeSJohn Baldwin static __inline u_##TYPE				\
184ccbdd9eeSJohn Baldwin atomic_load_acq_##TYPE(volatile u_##TYPE *p)		\
185ccbdd9eeSJohn Baldwin {							\
186ccbdd9eeSJohn Baldwin 	return (*p);					\
187ccbdd9eeSJohn Baldwin }							\
188ccbdd9eeSJohn Baldwin 							\
189ccbdd9eeSJohn Baldwin static __inline void					\
190ccbdd9eeSJohn Baldwin atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
191ccbdd9eeSJohn Baldwin {							\
192ccbdd9eeSJohn Baldwin 	*p = v;						\
193ccbdd9eeSJohn Baldwin 	__asm __volatile("" : : : "memory");		\
194ccbdd9eeSJohn Baldwin }
1954c5aee92SMark Murray 
1968306a37bSMark Murray #else /* !defined(I386_CPU) */
197ccbdd9eeSJohn Baldwin 
1989d979d89SJohn Baldwin #define ATOMIC_STORE_LOAD(TYPE, LOP, SOP)		\
1999d979d89SJohn Baldwin static __inline u_##TYPE				\
2009d979d89SJohn Baldwin atomic_load_acq_##TYPE(volatile u_##TYPE *p)		\
2019d979d89SJohn Baldwin {							\
2029d979d89SJohn Baldwin 	u_##TYPE res;					\
2039d979d89SJohn Baldwin 							\
20471acb247SBosko Milekic 	__asm __volatile(__XSTRING(MPLOCKED) LOP	\
205324fffaeSJohn Baldwin 	: "=a" (res),			/* 0 (result) */\
2069d979d89SJohn Baldwin 	  "+m" (*p)			/* 1 */		\
207e4e991e1SJohn Baldwin 	: : "memory");				 	\
2089d979d89SJohn Baldwin 							\
2099d979d89SJohn Baldwin 	return (res);					\
2109d979d89SJohn Baldwin }							\
2119d979d89SJohn Baldwin 							\
2129d979d89SJohn Baldwin /*							\
2139d979d89SJohn Baldwin  * The XCHG instruction asserts LOCK automagically.	\
2149d979d89SJohn Baldwin  */							\
2159d979d89SJohn Baldwin static __inline void					\
2169d979d89SJohn Baldwin atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
2179d979d89SJohn Baldwin {							\
2189d979d89SJohn Baldwin 	__asm __volatile(SOP				\
2199d979d89SJohn Baldwin 	: "+m" (*p),			/* 0 */		\
2209d979d89SJohn Baldwin 	  "+r" (v)			/* 1 */		\
2219d979d89SJohn Baldwin 	: : "memory");				 	\
2229d979d89SJohn Baldwin }
2234c5aee92SMark Murray 
2249d979d89SJohn Baldwin #endif	/* defined(I386_CPU) */
2254c5aee92SMark Murray 
2268306a37bSMark Murray #else /* !defined(__GNUC__) */
2278306a37bSMark Murray 
2284c5aee92SMark Murray extern int atomic_cmpset_int(volatile u_int *, u_int, u_int);
2294c5aee92SMark Murray 
2308306a37bSMark Murray #define ATOMIC_STORE_LOAD(TYPE, LOP, SOP)				\
2314c5aee92SMark Murray extern u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p);		\
2324c5aee92SMark Murray extern void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
2338306a37bSMark Murray 
2348306a37bSMark Murray #endif /* defined(__GNUC__) */
2354c5aee92SMark Murray 
2368a6b1c8fSJohn Baldwin #endif /* KLD_MODULE */
2378a6b1c8fSJohn Baldwin 
2388306a37bSMark Murray ATOMIC_ASM(set,	     char,  "orb %b1,%0",  "iq",  v);
2398306a37bSMark Murray ATOMIC_ASM(clear,    char,  "andb %b1,%0", "iq", ~v);
2408306a37bSMark Murray ATOMIC_ASM(add,	     char,  "addb %b1,%0", "iq",  v);
2418306a37bSMark Murray ATOMIC_ASM(subtract, char,  "subb %b1,%0", "iq",  v);
2428a6b1c8fSJohn Baldwin 
2438306a37bSMark Murray ATOMIC_ASM(set,	     short, "orw %w1,%0",  "ir",  v);
2448306a37bSMark Murray ATOMIC_ASM(clear,    short, "andw %w1,%0", "ir", ~v);
2458306a37bSMark Murray ATOMIC_ASM(add,	     short, "addw %w1,%0", "ir",  v);
2468306a37bSMark Murray ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir",  v);
2478a6b1c8fSJohn Baldwin 
2488306a37bSMark Murray ATOMIC_ASM(set,	     int,   "orl %1,%0",   "ir",  v);
2498306a37bSMark Murray ATOMIC_ASM(clear,    int,   "andl %1,%0",  "ir", ~v);
2508306a37bSMark Murray ATOMIC_ASM(add,	     int,   "addl %1,%0",  "ir",  v);
2518306a37bSMark Murray ATOMIC_ASM(subtract, int,   "subl %1,%0",  "ir",  v);
2528a6b1c8fSJohn Baldwin 
2538306a37bSMark Murray ATOMIC_ASM(set,	     long,  "orl %1,%0",   "ir",  v);
2548306a37bSMark Murray ATOMIC_ASM(clear,    long,  "andl %1,%0",  "ir", ~v);
2558306a37bSMark Murray ATOMIC_ASM(add,	     long,  "addl %1,%0",  "ir",  v);
2568306a37bSMark Murray ATOMIC_ASM(subtract, long,  "subl %1,%0",  "ir",  v);
2579d979d89SJohn Baldwin 
2588306a37bSMark Murray ATOMIC_STORE_LOAD(char,	"cmpxchgb %b0,%1", "xchgb %b1,%0");
2598306a37bSMark Murray ATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0");
2608306a37bSMark Murray ATOMIC_STORE_LOAD(int,	"cmpxchgl %0,%1",  "xchgl %1,%0");
2618306a37bSMark Murray ATOMIC_STORE_LOAD(long,	"cmpxchgl %0,%1",  "xchgl %1,%0");
262ccbdd9eeSJohn Baldwin 
2638a6b1c8fSJohn Baldwin #undef ATOMIC_ASM
264ccbdd9eeSJohn Baldwin #undef ATOMIC_STORE_LOAD
265ccbdd9eeSJohn Baldwin 
2668a6b1c8fSJohn Baldwin #define	atomic_set_acq_char		atomic_set_char
2678a6b1c8fSJohn Baldwin #define	atomic_set_rel_char		atomic_set_char
2688a6b1c8fSJohn Baldwin #define	atomic_clear_acq_char		atomic_clear_char
2698a6b1c8fSJohn Baldwin #define	atomic_clear_rel_char		atomic_clear_char
2708a6b1c8fSJohn Baldwin #define	atomic_add_acq_char		atomic_add_char
2718a6b1c8fSJohn Baldwin #define	atomic_add_rel_char		atomic_add_char
2728a6b1c8fSJohn Baldwin #define	atomic_subtract_acq_char	atomic_subtract_char
2738a6b1c8fSJohn Baldwin #define	atomic_subtract_rel_char	atomic_subtract_char
2748a6b1c8fSJohn Baldwin 
2758a6b1c8fSJohn Baldwin #define	atomic_set_acq_short		atomic_set_short
2768a6b1c8fSJohn Baldwin #define	atomic_set_rel_short		atomic_set_short
2778a6b1c8fSJohn Baldwin #define	atomic_clear_acq_short		atomic_clear_short
2788a6b1c8fSJohn Baldwin #define	atomic_clear_rel_short		atomic_clear_short
2798a6b1c8fSJohn Baldwin #define	atomic_add_acq_short		atomic_add_short
2808a6b1c8fSJohn Baldwin #define	atomic_add_rel_short		atomic_add_short
2818a6b1c8fSJohn Baldwin #define	atomic_subtract_acq_short	atomic_subtract_short
2828a6b1c8fSJohn Baldwin #define	atomic_subtract_rel_short	atomic_subtract_short
2838a6b1c8fSJohn Baldwin 
2848a6b1c8fSJohn Baldwin #define	atomic_set_acq_int		atomic_set_int
2858a6b1c8fSJohn Baldwin #define	atomic_set_rel_int		atomic_set_int
2868a6b1c8fSJohn Baldwin #define	atomic_clear_acq_int		atomic_clear_int
2878a6b1c8fSJohn Baldwin #define	atomic_clear_rel_int		atomic_clear_int
2888a6b1c8fSJohn Baldwin #define	atomic_add_acq_int		atomic_add_int
2898a6b1c8fSJohn Baldwin #define	atomic_add_rel_int		atomic_add_int
2908a6b1c8fSJohn Baldwin #define	atomic_subtract_acq_int		atomic_subtract_int
2918a6b1c8fSJohn Baldwin #define	atomic_subtract_rel_int		atomic_subtract_int
2928a6b1c8fSJohn Baldwin #define atomic_cmpset_acq_int		atomic_cmpset_int
2938a6b1c8fSJohn Baldwin #define atomic_cmpset_rel_int		atomic_cmpset_int
2948a6b1c8fSJohn Baldwin 
2958a6b1c8fSJohn Baldwin #define	atomic_set_acq_long		atomic_set_long
2968a6b1c8fSJohn Baldwin #define	atomic_set_rel_long		atomic_set_long
2978a6b1c8fSJohn Baldwin #define	atomic_clear_acq_long		atomic_clear_long
2988a6b1c8fSJohn Baldwin #define	atomic_clear_rel_long		atomic_clear_long
2998a6b1c8fSJohn Baldwin #define	atomic_add_acq_long		atomic_add_long
3008a6b1c8fSJohn Baldwin #define	atomic_add_rel_long		atomic_add_long
3018a6b1c8fSJohn Baldwin #define	atomic_subtract_acq_long	atomic_subtract_long
3028a6b1c8fSJohn Baldwin #define	atomic_subtract_rel_long	atomic_subtract_long
3038a6b1c8fSJohn Baldwin #define	atomic_cmpset_long		atomic_cmpset_int
3048a6b1c8fSJohn Baldwin #define	atomic_cmpset_acq_long		atomic_cmpset_acq_int
3058a6b1c8fSJohn Baldwin #define	atomic_cmpset_rel_long		atomic_cmpset_rel_int
3068a6b1c8fSJohn Baldwin 
3078a6b1c8fSJohn Baldwin #define atomic_cmpset_acq_ptr		atomic_cmpset_ptr
3088a6b1c8fSJohn Baldwin #define atomic_cmpset_rel_ptr		atomic_cmpset_ptr
3098a6b1c8fSJohn Baldwin 
3108a6b1c8fSJohn Baldwin #define	atomic_set_8		atomic_set_char
3118a6b1c8fSJohn Baldwin #define	atomic_set_acq_8	atomic_set_acq_char
3128a6b1c8fSJohn Baldwin #define	atomic_set_rel_8	atomic_set_rel_char
3138a6b1c8fSJohn Baldwin #define	atomic_clear_8		atomic_clear_char
3148a6b1c8fSJohn Baldwin #define	atomic_clear_acq_8	atomic_clear_acq_char
3158a6b1c8fSJohn Baldwin #define	atomic_clear_rel_8	atomic_clear_rel_char
3168a6b1c8fSJohn Baldwin #define	atomic_add_8		atomic_add_char
3178a6b1c8fSJohn Baldwin #define	atomic_add_acq_8	atomic_add_acq_char
3188a6b1c8fSJohn Baldwin #define	atomic_add_rel_8	atomic_add_rel_char
3198a6b1c8fSJohn Baldwin #define	atomic_subtract_8	atomic_subtract_char
3208a6b1c8fSJohn Baldwin #define	atomic_subtract_acq_8	atomic_subtract_acq_char
3218a6b1c8fSJohn Baldwin #define	atomic_subtract_rel_8	atomic_subtract_rel_char
3228a6b1c8fSJohn Baldwin #define	atomic_load_acq_8	atomic_load_acq_char
3238a6b1c8fSJohn Baldwin #define	atomic_store_rel_8	atomic_store_rel_char
3248a6b1c8fSJohn Baldwin 
3258a6b1c8fSJohn Baldwin #define	atomic_set_16		atomic_set_short
3268a6b1c8fSJohn Baldwin #define	atomic_set_acq_16	atomic_set_acq_short
3278a6b1c8fSJohn Baldwin #define	atomic_set_rel_16	atomic_set_rel_short
3288a6b1c8fSJohn Baldwin #define	atomic_clear_16		atomic_clear_short
3298a6b1c8fSJohn Baldwin #define	atomic_clear_acq_16	atomic_clear_acq_short
3308a6b1c8fSJohn Baldwin #define	atomic_clear_rel_16	atomic_clear_rel_short
3318a6b1c8fSJohn Baldwin #define	atomic_add_16		atomic_add_short
3328a6b1c8fSJohn Baldwin #define	atomic_add_acq_16	atomic_add_acq_short
3338a6b1c8fSJohn Baldwin #define	atomic_add_rel_16	atomic_add_rel_short
3348a6b1c8fSJohn Baldwin #define	atomic_subtract_16	atomic_subtract_short
3358a6b1c8fSJohn Baldwin #define	atomic_subtract_acq_16	atomic_subtract_acq_short
3368a6b1c8fSJohn Baldwin #define	atomic_subtract_rel_16	atomic_subtract_rel_short
3378a6b1c8fSJohn Baldwin #define	atomic_load_acq_16	atomic_load_acq_short
3388a6b1c8fSJohn Baldwin #define	atomic_store_rel_16	atomic_store_rel_short
3398a6b1c8fSJohn Baldwin 
3408a6b1c8fSJohn Baldwin #define	atomic_set_32		atomic_set_int
3418a6b1c8fSJohn Baldwin #define	atomic_set_acq_32	atomic_set_acq_int
3428a6b1c8fSJohn Baldwin #define	atomic_set_rel_32	atomic_set_rel_int
3438a6b1c8fSJohn Baldwin #define	atomic_clear_32		atomic_clear_int
3448a6b1c8fSJohn Baldwin #define	atomic_clear_acq_32	atomic_clear_acq_int
3458a6b1c8fSJohn Baldwin #define	atomic_clear_rel_32	atomic_clear_rel_int
3468a6b1c8fSJohn Baldwin #define	atomic_add_32		atomic_add_int
3478a6b1c8fSJohn Baldwin #define	atomic_add_acq_32	atomic_add_acq_int
3488a6b1c8fSJohn Baldwin #define	atomic_add_rel_32	atomic_add_rel_int
3498a6b1c8fSJohn Baldwin #define	atomic_subtract_32	atomic_subtract_int
3508a6b1c8fSJohn Baldwin #define	atomic_subtract_acq_32	atomic_subtract_acq_int
3518a6b1c8fSJohn Baldwin #define	atomic_subtract_rel_32	atomic_subtract_rel_int
3528a6b1c8fSJohn Baldwin #define	atomic_load_acq_32	atomic_load_acq_int
3538a6b1c8fSJohn Baldwin #define	atomic_store_rel_32	atomic_store_rel_int
3548a6b1c8fSJohn Baldwin #define	atomic_cmpset_32	atomic_cmpset_int
3558a6b1c8fSJohn Baldwin #define	atomic_cmpset_acq_32	atomic_cmpset_acq_int
3568a6b1c8fSJohn Baldwin #define	atomic_cmpset_rel_32	atomic_cmpset_rel_int
3578a6b1c8fSJohn Baldwin #define	atomic_readandclear_32	atomic_readandclear_int
3588a6b1c8fSJohn Baldwin 
3598a6b1c8fSJohn Baldwin #if !defined(WANT_FUNCTIONS)
360819e370cSPoul-Henning Kamp static __inline int
361819e370cSPoul-Henning Kamp atomic_cmpset_ptr(volatile void *dst, void *exp, void *src)
362819e370cSPoul-Henning Kamp {
363819e370cSPoul-Henning Kamp 
3648a6b1c8fSJohn Baldwin 	return (atomic_cmpset_int((volatile u_int *)dst, (u_int)exp,
3658a6b1c8fSJohn Baldwin 	    (u_int)src));
366819e370cSPoul-Henning Kamp }
367b4645202SJohn Baldwin 
368ccbdd9eeSJohn Baldwin static __inline void *
369ccbdd9eeSJohn Baldwin atomic_load_acq_ptr(volatile void *p)
370ccbdd9eeSJohn Baldwin {
371ccbdd9eeSJohn Baldwin 	return (void *)atomic_load_acq_int((volatile u_int *)p);
372ccbdd9eeSJohn Baldwin }
373ccbdd9eeSJohn Baldwin 
374ccbdd9eeSJohn Baldwin static __inline void
375ccbdd9eeSJohn Baldwin atomic_store_rel_ptr(volatile void *p, void *v)
376ccbdd9eeSJohn Baldwin {
377ccbdd9eeSJohn Baldwin 	atomic_store_rel_int((volatile u_int *)p, (u_int)v);
378ccbdd9eeSJohn Baldwin }
379ccbdd9eeSJohn Baldwin 
380ccbdd9eeSJohn Baldwin #define ATOMIC_PTR(NAME)				\
381ccbdd9eeSJohn Baldwin static __inline void					\
382ccbdd9eeSJohn Baldwin atomic_##NAME##_ptr(volatile void *p, uintptr_t v)	\
383ccbdd9eeSJohn Baldwin {							\
384ccbdd9eeSJohn Baldwin 	atomic_##NAME##_int((volatile u_int *)p, v);	\
385ccbdd9eeSJohn Baldwin }							\
386ccbdd9eeSJohn Baldwin 							\
387ccbdd9eeSJohn Baldwin static __inline void					\
388ccbdd9eeSJohn Baldwin atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v)	\
389ccbdd9eeSJohn Baldwin {							\
390ccbdd9eeSJohn Baldwin 	atomic_##NAME##_acq_int((volatile u_int *)p, v);\
391ccbdd9eeSJohn Baldwin }							\
392ccbdd9eeSJohn Baldwin 							\
393ccbdd9eeSJohn Baldwin static __inline void					\
394ccbdd9eeSJohn Baldwin atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v)	\
395ccbdd9eeSJohn Baldwin {							\
396ccbdd9eeSJohn Baldwin 	atomic_##NAME##_rel_int((volatile u_int *)p, v);\
397ccbdd9eeSJohn Baldwin }
398ccbdd9eeSJohn Baldwin 
399ccbdd9eeSJohn Baldwin ATOMIC_PTR(set)
400ccbdd9eeSJohn Baldwin ATOMIC_PTR(clear)
401ccbdd9eeSJohn Baldwin ATOMIC_PTR(add)
402ccbdd9eeSJohn Baldwin ATOMIC_PTR(subtract)
403ccbdd9eeSJohn Baldwin 
404ccbdd9eeSJohn Baldwin #undef ATOMIC_PTR
405ccbdd9eeSJohn Baldwin 
4068306a37bSMark Murray #if defined(__GNUC__)
4074c5aee92SMark Murray 
408b4645202SJohn Baldwin static __inline u_int
409b4645202SJohn Baldwin atomic_readandclear_int(volatile u_int *addr)
410b4645202SJohn Baldwin {
411b4645202SJohn Baldwin 	u_int result;
412b4645202SJohn Baldwin 
413b4645202SJohn Baldwin 	__asm __volatile (
414b4645202SJohn Baldwin 	"	xorl	%0,%0 ;		"
415b4645202SJohn Baldwin 	"	xchgl	%1,%0 ;		"
416b4645202SJohn Baldwin 	"# atomic_readandclear_int"
417b4645202SJohn Baldwin 	: "=&r" (result)		/* 0 (result) */
418b4645202SJohn Baldwin 	: "m" (*addr));			/* 1 (addr) */
419b4645202SJohn Baldwin 
420b4645202SJohn Baldwin 	return (result);
421b4645202SJohn Baldwin }
422b4645202SJohn Baldwin 
423b4645202SJohn Baldwin static __inline u_long
424b4645202SJohn Baldwin atomic_readandclear_long(volatile u_long *addr)
425b4645202SJohn Baldwin {
426b4645202SJohn Baldwin 	u_long result;
427b4645202SJohn Baldwin 
428b4645202SJohn Baldwin 	__asm __volatile (
429b4645202SJohn Baldwin 	"	xorl	%0,%0 ;		"
430b4645202SJohn Baldwin 	"	xchgl	%1,%0 ;		"
431b4645202SJohn Baldwin 	"# atomic_readandclear_int"
432b4645202SJohn Baldwin 	: "=&r" (result)		/* 0 (result) */
433b4645202SJohn Baldwin 	: "m" (*addr));			/* 1 (addr) */
434b4645202SJohn Baldwin 
435b4645202SJohn Baldwin 	return (result);
436b4645202SJohn Baldwin }
4374c5aee92SMark Murray 
4388306a37bSMark Murray #else /* !defined(__GNUC__) */
4394c5aee92SMark Murray 
4404c5aee92SMark Murray extern u_long	atomic_readandclear_long(volatile u_long *);
4414c5aee92SMark Murray extern u_int	atomic_readandclear_int(volatile u_int *);
4424c5aee92SMark Murray 
4438306a37bSMark Murray #endif /* defined(__GNUC__) */
4444c5aee92SMark Murray 
4458a6b1c8fSJohn Baldwin #endif	/* !defined(WANT_FUNCTIONS) */
446069e9bc1SDoug Rabson #endif /* ! _MACHINE_ATOMIC_H_ */
447