xref: /illumos-gate/usr/src/uts/common/sys/atomic.h (revision d48be21240dfd051b689384ce2b23479d757f2d8)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2014 Garrett D'Amore <garrett@damore.org>
24  *
25  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
26  * Use is subject to license terms.
27  */
28 
29 #ifndef	_SYS_ATOMIC_H
30 #define	_SYS_ATOMIC_H
31 
32 #include <sys/types.h>
33 #include <sys/inttypes.h>
34 
35 #ifdef	__cplusplus
36 extern "C" {
37 #endif
38 
39 #if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) && \
40 	(defined(__i386) || defined(__amd64))
41 #include <asm/atomic.h>
42 #endif
43 
44 /*
45  * Increment target.
46  */
47 extern void atomic_inc_8(volatile uint8_t *);
48 extern void atomic_inc_uchar(volatile uchar_t *);
49 extern void atomic_inc_16(volatile uint16_t *);
50 extern void atomic_inc_ushort(volatile ushort_t *);
51 extern void atomic_inc_32(volatile uint32_t *);
52 extern void atomic_inc_uint(volatile uint_t *);
53 extern void atomic_inc_ulong(volatile ulong_t *);
54 #if defined(_KERNEL) || defined(_INT64_TYPE)
55 extern void atomic_inc_64(volatile uint64_t *);
56 #endif
57 
58 /*
59  * Decrement target
60  */
61 extern void atomic_dec_8(volatile uint8_t *);
62 extern void atomic_dec_uchar(volatile uchar_t *);
63 extern void atomic_dec_16(volatile uint16_t *);
64 extern void atomic_dec_ushort(volatile ushort_t *);
65 extern void atomic_dec_32(volatile uint32_t *);
66 extern void atomic_dec_uint(volatile uint_t *);
67 extern void atomic_dec_ulong(volatile ulong_t *);
68 #if defined(_KERNEL) || defined(_INT64_TYPE)
69 extern void atomic_dec_64(volatile uint64_t *);
70 #endif
71 
72 /*
73  * Add delta to target
74  */
75 extern void atomic_add_8(volatile uint8_t *, int8_t);
76 extern void atomic_add_char(volatile uchar_t *, signed char);
77 extern void atomic_add_16(volatile uint16_t *, int16_t);
78 extern void atomic_add_short(volatile ushort_t *, short);
79 extern void atomic_add_32(volatile uint32_t *, int32_t);
80 extern void atomic_add_int(volatile uint_t *, int);
81 extern void atomic_add_ptr(volatile void *, ssize_t);
82 extern void atomic_add_long(volatile ulong_t *, long);
83 #if defined(_KERNEL) || defined(_INT64_TYPE)
84 extern void atomic_add_64(volatile uint64_t *, int64_t);
85 #endif
86 
87 /*
88  * logical OR bits with target
89  */
90 extern void atomic_or_8(volatile uint8_t *, uint8_t);
91 extern void atomic_or_uchar(volatile uchar_t *, uchar_t);
92 extern void atomic_or_16(volatile uint16_t *, uint16_t);
93 extern void atomic_or_ushort(volatile ushort_t *, ushort_t);
94 extern void atomic_or_32(volatile uint32_t *, uint32_t);
95 extern void atomic_or_uint(volatile uint_t *, uint_t);
96 extern void atomic_or_ulong(volatile ulong_t *, ulong_t);
97 #if defined(_KERNEL) || defined(_INT64_TYPE)
98 extern void atomic_or_64(volatile uint64_t *, uint64_t);
99 #endif
100 
101 /*
102  * logical AND bits with target
103  */
104 extern void atomic_and_8(volatile uint8_t *, uint8_t);
105 extern void atomic_and_uchar(volatile uchar_t *, uchar_t);
106 extern void atomic_and_16(volatile uint16_t *, uint16_t);
107 extern void atomic_and_ushort(volatile ushort_t *, ushort_t);
108 extern void atomic_and_32(volatile uint32_t *, uint32_t);
109 extern void atomic_and_uint(volatile uint_t *, uint_t);
110 extern void atomic_and_ulong(volatile ulong_t *, ulong_t);
111 #if defined(_KERNEL) || defined(_INT64_TYPE)
112 extern void atomic_and_64(volatile uint64_t *, uint64_t);
113 #endif
114 
115 /*
116  * As above, but return the new value.  Note that these _nv() variants are
117  * substantially more expensive on some platforms than the no-return-value
118  * versions above, so don't use them unless you really need to know the
119  * new value *atomically* (e.g. when decrementing a reference count and
120  * checking whether it went to zero).
121  */
122 
123 /*
124  * Increment target and return new value.
125  */
126 extern uint8_t atomic_inc_8_nv(volatile uint8_t *);
127 extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *);
128 extern uint16_t atomic_inc_16_nv(volatile uint16_t *);
129 extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *);
130 extern uint32_t atomic_inc_32_nv(volatile uint32_t *);
131 extern uint_t atomic_inc_uint_nv(volatile uint_t *);
132 extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *);
133 #if defined(_KERNEL) || defined(_INT64_TYPE)
134 extern uint64_t atomic_inc_64_nv(volatile uint64_t *);
135 #endif
136 
137 /*
138  * Decrement target and return new value.
139  */
140 extern uint8_t atomic_dec_8_nv(volatile uint8_t *);
141 extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *);
142 extern uint16_t atomic_dec_16_nv(volatile uint16_t *);
143 extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *);
144 extern uint32_t atomic_dec_32_nv(volatile uint32_t *);
145 extern uint_t atomic_dec_uint_nv(volatile uint_t *);
146 extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *);
147 #if defined(_KERNEL) || defined(_INT64_TYPE)
148 extern uint64_t atomic_dec_64_nv(volatile uint64_t *);
149 #endif
150 
151 /*
152  * Add delta to target
153  */
154 extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t);
155 extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char);
156 extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t);
157 extern ushort_t atomic_add_short_nv(volatile ushort_t *, short);
158 extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t);
159 extern uint_t atomic_add_int_nv(volatile uint_t *, int);
160 extern void *atomic_add_ptr_nv(volatile void *, ssize_t);
161 extern ulong_t atomic_add_long_nv(volatile ulong_t *, long);
162 #if defined(_KERNEL) || defined(_INT64_TYPE)
163 extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t);
164 #endif
165 
166 /*
167  * logical OR bits with target and return new value.
168  */
169 extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t);
170 extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t);
171 extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t);
172 extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t);
173 extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t);
174 extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t);
175 extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t);
176 #if defined(_KERNEL) || defined(_INT64_TYPE)
177 extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t);
178 #endif
179 
180 /*
181  * logical AND bits with target and return new value.
182  */
183 extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t);
184 extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t);
185 extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t);
186 extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t);
187 extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t);
188 extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t);
189 extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t);
190 #if defined(_KERNEL) || defined(_INT64_TYPE)
191 extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t);
192 #endif
193 
194 /*
195  * If *arg1 == arg2, set *arg1 = arg3; return old value
196  */
197 extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
198 extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t);
199 extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t);
200 extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t);
201 extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
202 extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t);
203 extern void *atomic_cas_ptr(volatile void *, void *, void *);
204 extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t);
205 #if defined(_KERNEL) || defined(_INT64_TYPE)
206 extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t);
207 #endif
208 
209 /*
210  * Swap target and return old value
211  */
212 extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t);
213 extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t);
214 extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t);
215 extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t);
216 extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t);
217 extern uint_t atomic_swap_uint(volatile uint_t *, uint_t);
218 extern void *atomic_swap_ptr(volatile void *, void *);
219 extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t);
220 #if defined(_KERNEL) || defined(_INT64_TYPE)
221 extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t);
222 #endif
223 
224 /*
225  * Perform an exclusive atomic bit set/clear on a target.
226  * Returns 0 if bit was sucessfully set/cleared, or -1
227  * if the bit was already set/cleared.
228  */
229 extern int atomic_set_long_excl(volatile ulong_t *, uint_t);
230 extern int atomic_clear_long_excl(volatile ulong_t *, uint_t);
231 
232 /*
233  * Generic memory barrier used during lock entry, placed after the
234  * memory operation that acquires the lock to guarantee that the lock
235  * protects its data.  No stores from after the memory barrier will
236  * reach visibility, and no loads from after the barrier will be
237  * resolved, before the lock acquisition reaches global visibility.
238  */
239 extern void membar_enter(void);
240 
241 /*
242  * Generic memory barrier used during lock exit, placed before the
243  * memory operation that releases the lock to guarantee that the lock
244  * protects its data.  All loads and stores issued before the barrier
245  * will be resolved before the subsequent lock update reaches visibility.
246  */
247 extern void membar_exit(void);
248 
249 /*
250  * Arrange that all stores issued before this point in the code reach
251  * global visibility before any stores that follow; useful in producer
252  * modules that update a data item, then set a flag that it is available.
253  * The memory barrier guarantees that the available flag is not visible
254  * earlier than the updated data, i.e. it imposes store ordering.
255  */
256 extern void membar_producer(void);
257 
258 /*
259  * Arrange that all loads issued before this point in the code are
260  * completed before any subsequent loads; useful in consumer modules
261  * that check to see if data is available and read the data.
262  * The memory barrier guarantees that the data is not sampled until
263  * after the available flag has been seen, i.e. it imposes load ordering.
264  */
265 extern void membar_consumer(void);
266 
267 #if defined(_KERNEL)
268 
269 #if defined(_LP64) || defined(_ILP32)
270 #define	atomic_add_ip		atomic_add_long
271 #define	atomic_add_ip_nv	atomic_add_long_nv
272 #define	casip			atomic_cas_ulong
273 #endif
274 
275 #endif	/* _KERNEL */
276 
277 #ifdef	__cplusplus
278 }
279 #endif
280 
281 #endif	/* _SYS_ATOMIC_H */
282