1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org>
5 * David Chisnall <theraven@FreeBSD.org>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #ifndef _STDATOMIC_H_
31 #define _STDATOMIC_H_
32
33 #include <sys/cdefs.h>
34 #include <sys/_types.h>
35
36 #if __ISO_C_VISIBLE >= 2023 || __BSD_VISIBLE
37 #define __STDC_VERSION_STDATOMIC_H__ 202311L
38 #endif
39
40 #if (__has_extension(c_atomic) || __has_extension(cxx_atomic)) && \
41 defined(__clang__)
42 #define __CLANG_ATOMICS
43 #elif __GNUC_PREREQ__(4, 7)
44 #define __GNUC_ATOMICS
45 #elif defined(__GNUC__)
46 #define __SYNC_ATOMICS
47 #else
48 #error "stdatomic.h does not support your compiler"
49 #endif
50
51 /*
52 * 7.17.1 Atomic lock-free macros.
53 */
54
55 #ifdef __GCC_ATOMIC_BOOL_LOCK_FREE
56 #define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE
57 #endif
58 #ifdef __GCC_ATOMIC_CHAR_LOCK_FREE
59 #define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE
60 #endif
61 #if __ISO_C_VISIBLE >= 2023 && defined(__GCC_ATOMIC_CHAR8_T_LOCK_FREE)
62 #define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE
63 #endif
64 #ifdef __GCC_ATOMIC_CHAR16_T_LOCK_FREE
65 #define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE
66 #endif
67 #ifdef __GCC_ATOMIC_CHAR32_T_LOCK_FREE
68 #define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE
69 #endif
70 #ifdef __GCC_ATOMIC_WCHAR_T_LOCK_FREE
71 #define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE
72 #endif
73 #ifdef __GCC_ATOMIC_SHORT_LOCK_FREE
74 #define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE
75 #endif
76 #ifdef __GCC_ATOMIC_INT_LOCK_FREE
77 #define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE
78 #endif
79 #ifdef __GCC_ATOMIC_LONG_LOCK_FREE
80 #define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE
81 #endif
82 #ifdef __GCC_ATOMIC_LLONG_LOCK_FREE
83 #define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE
84 #endif
85 #ifdef __GCC_ATOMIC_POINTER_LOCK_FREE
86 #define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE
87 #endif
88
89 /*
90 * 7.17.2 Initialization.
91 */
92
93 #if __BSD_VISIBLE || __ISO_C_VISIBLE < 2023 || defined(__cplusplus)
94 #if defined(__CLANG_ATOMICS)
95 #define ATOMIC_VAR_INIT(value) (value)
96 #else
97 #define ATOMIC_VAR_INIT(value) { .__val = (value) }
98 #endif
99 #endif
100
101 #if defined(__CLANG_ATOMICS)
102 #define atomic_init(obj, value) __c11_atomic_init(obj, value)
103 #elif defined(__GNUC_ATOMICS)
104 #define atomic_init(obj, value) atomic_store_explicit(obj, \
105 value, __ATOMIC_RELAXED)
106 #else
107 #define atomic_init(obj, value) ((void)((obj)->__val = (value)))
108 #endif
109
110 /*
111 * 7.17.3 Order and consistency.
112 *
113 * The memory_order_* constants that denote the barrier behaviour of the
114 * atomic operations.
115 */
116
117 #ifndef __ATOMIC_RELAXED
118 #define __ATOMIC_RELAXED 0
119 #endif
120 #ifndef __ATOMIC_CONSUME
121 #define __ATOMIC_CONSUME 1
122 #endif
123 #ifndef __ATOMIC_ACQUIRE
124 #define __ATOMIC_ACQUIRE 2
125 #endif
126 #ifndef __ATOMIC_RELEASE
127 #define __ATOMIC_RELEASE 3
128 #endif
129 #ifndef __ATOMIC_ACQ_REL
130 #define __ATOMIC_ACQ_REL 4
131 #endif
132 #ifndef __ATOMIC_SEQ_CST
133 #define __ATOMIC_SEQ_CST 5
134 #endif
135
136 typedef enum {
137 memory_order_relaxed = __ATOMIC_RELAXED,
138 memory_order_consume = __ATOMIC_CONSUME,
139 memory_order_acquire = __ATOMIC_ACQUIRE,
140 memory_order_release = __ATOMIC_RELEASE,
141 memory_order_acq_rel = __ATOMIC_ACQ_REL,
142 memory_order_seq_cst = __ATOMIC_SEQ_CST
143 } memory_order;
144
145 #define kill_dependency(y) (y)
146
147 /*
148 * 7.17.4 Fences.
149 */
150
151 static __inline void
atomic_thread_fence(memory_order __order __unused)152 atomic_thread_fence(memory_order __order __unused)
153 {
154
155 #ifdef __CLANG_ATOMICS
156 __c11_atomic_thread_fence(__order);
157 #elif defined(__GNUC_ATOMICS)
158 __atomic_thread_fence(__order);
159 #else
160 __sync_synchronize();
161 #endif
162 }
163
164 static __inline void
atomic_signal_fence(memory_order __order __unused)165 atomic_signal_fence(memory_order __order __unused)
166 {
167
168 #ifdef __CLANG_ATOMICS
169 __c11_atomic_signal_fence(__order);
170 #elif defined(__GNUC_ATOMICS)
171 __atomic_signal_fence(__order);
172 #else
173 __asm volatile ("" ::: "memory");
174 #endif
175 }
176
177 #if defined(__cplusplus) && !defined(_Bool)
178 #define _Bool bool
179 #define __bool_locally_defined
180 #endif
181
182 /*
183 * 7.17.5 Lock-free property.
184 */
185
186 #if defined(_KERNEL)
187 /* Atomics in kernelspace are always lock-free. */
188 #define atomic_is_lock_free(obj) \
189 ((void)(obj), (_Bool)1)
190 #elif defined(__CLANG_ATOMICS) || defined(__GNUC_ATOMICS)
191 #define atomic_is_lock_free(obj) \
192 __atomic_is_lock_free(sizeof(*(obj)), obj)
193 #else
194 #define atomic_is_lock_free(obj) \
195 ((void)(obj), sizeof((obj)->__val) <= sizeof(void *))
196 #endif
197
198 /*
199 * 7.17.6 Atomic integer types.
200 */
201
202 typedef _Atomic(_Bool) atomic_bool;
203 typedef _Atomic(char) atomic_char;
204 typedef _Atomic(signed char) atomic_schar;
205 typedef _Atomic(unsigned char) atomic_uchar;
206 typedef _Atomic(short) atomic_short;
207 typedef _Atomic(unsigned short) atomic_ushort;
208 typedef _Atomic(int) atomic_int;
209 typedef _Atomic(unsigned int) atomic_uint;
210 typedef _Atomic(long) atomic_long;
211 typedef _Atomic(unsigned long) atomic_ulong;
212 typedef _Atomic(long long) atomic_llong;
213 typedef _Atomic(unsigned long long) atomic_ullong;
214 #if __ISO_C_VISIBLE >= 2023
215 typedef _Atomic(unsigned char) atomic_char8_t;
216 #endif
217 typedef _Atomic(__char16_t) atomic_char16_t;
218 typedef _Atomic(__char32_t) atomic_char32_t;
219 typedef _Atomic(___wchar_t) atomic_wchar_t;
220 typedef _Atomic(__int_least8_t) atomic_int_least8_t;
221 typedef _Atomic(__uint_least8_t) atomic_uint_least8_t;
222 typedef _Atomic(__int_least16_t) atomic_int_least16_t;
223 typedef _Atomic(__uint_least16_t) atomic_uint_least16_t;
224 typedef _Atomic(__int_least32_t) atomic_int_least32_t;
225 typedef _Atomic(__uint_least32_t) atomic_uint_least32_t;
226 typedef _Atomic(__int_least64_t) atomic_int_least64_t;
227 typedef _Atomic(__uint_least64_t) atomic_uint_least64_t;
228 typedef _Atomic(__int_fast8_t) atomic_int_fast8_t;
229 typedef _Atomic(__uint_fast8_t) atomic_uint_fast8_t;
230 typedef _Atomic(__int_fast16_t) atomic_int_fast16_t;
231 typedef _Atomic(__uint_fast16_t) atomic_uint_fast16_t;
232 typedef _Atomic(__int_fast32_t) atomic_int_fast32_t;
233 typedef _Atomic(__uint_fast32_t) atomic_uint_fast32_t;
234 typedef _Atomic(__int_fast64_t) atomic_int_fast64_t;
235 typedef _Atomic(__uint_fast64_t) atomic_uint_fast64_t;
236 typedef _Atomic(__intptr_t) atomic_intptr_t;
237 typedef _Atomic(__uintptr_t) atomic_uintptr_t;
238 typedef _Atomic(__size_t) atomic_size_t;
239 typedef _Atomic(__ptrdiff_t) atomic_ptrdiff_t;
240 typedef _Atomic(__intmax_t) atomic_intmax_t;
241 typedef _Atomic(__uintmax_t) atomic_uintmax_t;
242
243 /*
244 * 7.17.7 Operations on atomic types.
245 */
246
247 /*
248 * Compiler-specific operations.
249 */
250
251 #if defined(__CLANG_ATOMICS)
252 #define atomic_store_explicit(object, desired, order) \
253 __c11_atomic_store(object, desired, order)
254 #define atomic_load_explicit(object, order) \
255 __c11_atomic_load(object, order)
256 #define atomic_exchange_explicit(object, desired, order) \
257 __c11_atomic_exchange(object, desired, order)
258 #define atomic_compare_exchange_strong_explicit(object, expected, \
259 desired, success, failure) \
260 __c11_atomic_compare_exchange_strong(object, expected, desired, \
261 success, failure)
262 #define atomic_compare_exchange_weak_explicit(object, expected, \
263 desired, success, failure) \
264 __c11_atomic_compare_exchange_weak(object, expected, desired, \
265 success, failure)
266 #define atomic_fetch_add_explicit(object, operand, order) \
267 __c11_atomic_fetch_add(object, operand, order)
268 #define atomic_fetch_sub_explicit(object, operand, order) \
269 __c11_atomic_fetch_sub(object, operand, order)
270 #define atomic_fetch_or_explicit(object, operand, order) \
271 __c11_atomic_fetch_or(object, operand, order)
272 #define atomic_fetch_xor_explicit(object, operand, order) \
273 __c11_atomic_fetch_xor(object, operand, order)
274 #define atomic_fetch_and_explicit(object, operand, order) \
275 __c11_atomic_fetch_and(object, operand, order)
276 #elif defined(__GNUC_ATOMICS)
277 #define __atomic_apply_stride(object, operand) \
278 (((__typeof__(*(object)))0) + (operand))
279 #define atomic_store_explicit(object, desired, order) \
280 __atomic_store_n(object, desired, order)
281 #define atomic_load_explicit(object, order) \
282 __atomic_load_n(object, order)
283 #define atomic_exchange_explicit(object, desired, order) \
284 __atomic_exchange_n(object, desired, order)
285 #define atomic_compare_exchange_strong_explicit(object, expected, \
286 desired, success, failure) \
287 __atomic_compare_exchange_n(object, expected, \
288 desired, 0, success, failure)
289 #define atomic_compare_exchange_weak_explicit(object, expected, \
290 desired, success, failure) \
291 __atomic_compare_exchange_n(object, expected, \
292 desired, 1, success, failure)
293 #define atomic_fetch_add_explicit(object, operand, order) \
294 __atomic_fetch_add(object, \
295 __atomic_apply_stride(object, operand), order)
296 #define atomic_fetch_sub_explicit(object, operand, order) \
297 __atomic_fetch_sub(object, \
298 __atomic_apply_stride(object, operand), order)
299 #define atomic_fetch_or_explicit(object, operand, order) \
300 __atomic_fetch_or(object, operand, order)
301 #define atomic_fetch_xor_explicit(object, operand, order) \
302 __atomic_fetch_xor(object, operand, order)
303 #define atomic_fetch_and_explicit(object, operand, order) \
304 __atomic_fetch_and(object, operand, order)
305 #else
306 #define __atomic_apply_stride(object, operand) \
307 (((__typeof__((object)->__val))0) + (operand))
308 #define atomic_store_explicit(object, desired, order) \
309 ((void)atomic_exchange_explicit(object, desired, order))
310 #define atomic_load_explicit(object, order) \
311 ((void)(order), __sync_fetch_and_add(&(object)->__val, 0))
312 #if __has_builtin(__sync_swap)
313 /* Clang provides a full-barrier atomic exchange - use it if available. */
314 #define atomic_exchange_explicit(object, desired, order) \
315 ((void)(order), __sync_swap(&(object)->__val, desired))
316 #else
317 /*
318 * __sync_lock_test_and_set() is only an acquire barrier in theory (although in
319 * practice it is usually a full barrier) so we need an explicit barrier before
320 * it.
321 */
322 #define atomic_exchange_explicit(object, desired, order) \
323 __extension__ ({ \
324 __typeof__(object) __o = (object); \
325 __typeof__(desired) __d = (desired); \
326 (void)(order); \
327 __sync_synchronize(); \
328 __sync_lock_test_and_set(&(__o)->__val, __d); \
329 })
330 #define atomic_compare_exchange_strong_explicit(object, expected, \
331 desired, success, failure) __extension__ ({ \
332 __typeof__(expected) __ep = (expected); \
333 __typeof__(*__ep) __e = *__ep; \
334 (void)(success); (void)(failure); \
335 (_Bool)((*__ep = __sync_val_compare_and_swap(&(object)->__val, \
336 __e, desired)) == __e); \
337 })
338 #define atomic_compare_exchange_weak_explicit(object, expected, \
339 desired, success, failure) \
340 atomic_compare_exchange_strong_explicit(object, expected, \
341 desired, success, failure)
342 #endif
343 #define atomic_fetch_add_explicit(object, operand, order) \
344 ((void)(order), __sync_fetch_and_add(&(object)->__val, \
345 __atomic_apply_stride(object, operand)))
346 #define atomic_fetch_sub_explicit(object, operand, order) \
347 ((void)(order), __sync_fetch_and_sub(&(object)->__val, \
348 __atomic_apply_stride(object, operand)))
349 #define atomic_fetch_or_explicit(object, operand, order) \
350 ((void)(order), __sync_fetch_and_or(&(object)->__val, operand))
351 #define atomic_fetch_xor_explicit(object, operand, order) \
352 ((void)(order), __sync_fetch_and_xor(&(object)->__val, operand))
353 #define atomic_fetch_and_explicit(object, operand, order) \
354 ((void)(order), __sync_fetch_and_and(&(object)->__val, operand))
355 #endif
356
357 /*
358 * Convenience functions.
359 *
360 * Don't provide these in kernel space. In kernel space, we should be
361 * disciplined enough to always provide explicit barriers.
362 */
363
364 #ifndef _KERNEL
365 #define atomic_store(object, desired) \
366 atomic_store_explicit(object, desired, memory_order_seq_cst)
367 #define atomic_load(object) \
368 atomic_load_explicit(object, memory_order_seq_cst)
369 #define atomic_exchange(object, desired) \
370 atomic_exchange_explicit(object, desired, memory_order_seq_cst)
371 #define atomic_compare_exchange_strong(object, expected, desired) \
372 atomic_compare_exchange_strong_explicit(object, expected, \
373 desired, memory_order_seq_cst, memory_order_seq_cst)
374 #define atomic_compare_exchange_weak(object, expected, desired) \
375 atomic_compare_exchange_weak_explicit(object, expected, \
376 desired, memory_order_seq_cst, memory_order_seq_cst)
377 #define atomic_fetch_add(object, operand) \
378 atomic_fetch_add_explicit(object, operand, memory_order_seq_cst)
379 #define atomic_fetch_sub(object, operand) \
380 atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst)
381 #define atomic_fetch_or(object, operand) \
382 atomic_fetch_or_explicit(object, operand, memory_order_seq_cst)
383 #define atomic_fetch_xor(object, operand) \
384 atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst)
385 #define atomic_fetch_and(object, operand) \
386 atomic_fetch_and_explicit(object, operand, memory_order_seq_cst)
387 #endif /* !_KERNEL */
388
389 /*
390 * 7.17.8 Atomic flag type and operations.
391 */
392
393 typedef struct {
394 #if ATOMIC_BOOL_LOCK_FREE == 2
395 atomic_bool __flag;
396 #elif ATOMIC_CHAR_LOCK_FREE == 2
397 atomic_uchar __flag;
398 #else
399 #error "atomic_flag is required to be lock-free"
400 #endif
401 } atomic_flag;
402 #if __ISO_C_VISIBLE < 2023
403 #define ATOMIC_FLAG_INIT { ATOMIC_VAR_INIT(0) }
404 #else
405 #define ATOMIC_FLAG_INIT { 0 }
406 #endif
407
408 static __inline _Bool
atomic_flag_test_and_set_explicit(volatile atomic_flag * __object,memory_order __order)409 atomic_flag_test_and_set_explicit(volatile atomic_flag *__object,
410 memory_order __order)
411 {
412
413 return (atomic_exchange_explicit(&__object->__flag, 1, __order) != 0);
414 }
415
416 static __inline void
atomic_flag_clear_explicit(volatile atomic_flag * __object,memory_order __order)417 atomic_flag_clear_explicit(volatile atomic_flag *__object, memory_order __order)
418 {
419
420 atomic_store_explicit(&__object->__flag, 0, __order);
421 }
422
423 #ifndef _KERNEL
424 static __inline _Bool
atomic_flag_test_and_set(volatile atomic_flag * __object)425 atomic_flag_test_and_set(volatile atomic_flag *__object)
426 {
427
428 return (atomic_flag_test_and_set_explicit(__object,
429 memory_order_seq_cst));
430 }
431
432 static __inline void
atomic_flag_clear(volatile atomic_flag * __object)433 atomic_flag_clear(volatile atomic_flag *__object)
434 {
435
436 atomic_flag_clear_explicit(__object, memory_order_seq_cst);
437 }
438 #endif /* !_KERNEL */
439
440 #ifdef __bool_locally_defined
441 #undef _Bool
442 #undef __bool_locally_defined
443 #endif
444
445 #endif /* !_STDATOMIC_H_ */
446