1 //===----------------------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef _LIBCPP___ATOMIC_SUPPORT_C11_H
10 #define _LIBCPP___ATOMIC_SUPPORT_C11_H
11
12 #include <__atomic/memory_order.h>
13 #include <__config>
14 #include <__cstddef/ptrdiff_t.h>
15 #include <__memory/addressof.h>
16 #include <__type_traits/remove_const.h>
17
18 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
19 # pragma GCC system_header
20 #endif
21
22 //
23 // This file implements support for C11-style atomics
24 //
25
26 _LIBCPP_BEGIN_NAMESPACE_STD
27
28 template <typename _Tp>
29 struct __cxx_atomic_base_impl {
30 _LIBCPP_HIDE_FROM_ABI
31 #ifndef _LIBCPP_CXX03_LANG
32 __cxx_atomic_base_impl() _NOEXCEPT = default;
33 #else
34 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
35 }
36 #endif // _LIBCPP_CXX03_LANG
__cxx_atomic_base_impl__cxx_atomic_base_impl37 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
38 _Atomic(_Tp) __a_value;
39 };
40
41 #define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
42
__cxx_atomic_thread_fence(memory_order __order)43 _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
44 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
45 }
46
__cxx_atomic_signal_fence(memory_order __order)47 _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
48 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
49 }
50
51 template <class _Tp>
__cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp __val)52 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
53 __c11_atomic_init(std::addressof(__a->__a_value), __val);
54 }
55 template <class _Tp>
__cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a,_Tp __val)56 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
57 __c11_atomic_init(std::addressof(__a->__a_value), __val);
58 }
59
60 template <class _Tp>
61 _LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp __val,memory_order __order)62 __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
63 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
64 }
65 template <class _Tp>
66 _LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a,_Tp __val,memory_order __order)67 __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
68 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
69 }
70
71 template <class _Tp>
72 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile * __a,memory_order __order)73 __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
74 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
75 return __c11_atomic_load(
76 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
77 }
78 template <class _Tp>
__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const * __a,memory_order __order)79 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
80 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
81 return __c11_atomic_load(
82 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
83 }
84
85 template <class _Tp>
86 _LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile * __a,_Tp * __dst,memory_order __order)87 __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
88 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
89 *__dst = __c11_atomic_load(
90 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
91 }
92 template <class _Tp>
93 _LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const * __a,_Tp * __dst,memory_order __order)94 __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
95 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
96 *__dst = __c11_atomic_load(
97 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
98 }
99
100 template <class _Tp>
101 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp __value,memory_order __order)102 __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
103 return __c11_atomic_exchange(
104 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
105 }
106 template <class _Tp>
107 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a,_Tp __value,memory_order __order)108 __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
109 return __c11_atomic_exchange(
110 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
111 }
112
__to_failure_order(memory_order __order)113 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
114 // Avoid switch statement to make this a constexpr.
115 return __order == memory_order_release
116 ? memory_order_relaxed
117 : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
118 }
119
120 template <class _Tp>
__cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp * __expected,_Tp __value,memory_order __success,memory_order __failure)121 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
122 __cxx_atomic_base_impl<_Tp> volatile* __a,
123 _Tp* __expected,
124 _Tp __value,
125 memory_order __success,
126 memory_order __failure) _NOEXCEPT {
127 return __c11_atomic_compare_exchange_strong(
128 std::addressof(__a->__a_value),
129 __expected,
130 __value,
131 static_cast<__memory_order_underlying_t>(__success),
132 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
133 }
134 template <class _Tp>
__cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a,_Tp * __expected,_Tp __value,memory_order __success,memory_order __failure)135 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
136 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
137 _NOEXCEPT {
138 return __c11_atomic_compare_exchange_strong(
139 std::addressof(__a->__a_value),
140 __expected,
141 __value,
142 static_cast<__memory_order_underlying_t>(__success),
143 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
144 }
145
146 template <class _Tp>
__cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp * __expected,_Tp __value,memory_order __success,memory_order __failure)147 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
148 __cxx_atomic_base_impl<_Tp> volatile* __a,
149 _Tp* __expected,
150 _Tp __value,
151 memory_order __success,
152 memory_order __failure) _NOEXCEPT {
153 return __c11_atomic_compare_exchange_weak(
154 std::addressof(__a->__a_value),
155 __expected,
156 __value,
157 static_cast<__memory_order_underlying_t>(__success),
158 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
159 }
160 template <class _Tp>
__cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a,_Tp * __expected,_Tp __value,memory_order __success,memory_order __failure)161 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
162 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
163 _NOEXCEPT {
164 return __c11_atomic_compare_exchange_weak(
165 std::addressof(__a->__a_value),
166 __expected,
167 __value,
168 static_cast<__memory_order_underlying_t>(__success),
169 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
170 }
171
172 template <class _Tp>
173 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp __delta,memory_order __order)174 __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
175 return __c11_atomic_fetch_add(
176 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
177 }
178 template <class _Tp>
179 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a,_Tp __delta,memory_order __order)180 __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
181 return __c11_atomic_fetch_add(
182 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
183 }
184
185 template <class _Tp>
186 _LIBCPP_HIDE_FROM_ABI _Tp*
__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp * > volatile * __a,ptrdiff_t __delta,memory_order __order)187 __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
188 return __c11_atomic_fetch_add(
189 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
190 }
191 template <class _Tp>
192 _LIBCPP_HIDE_FROM_ABI _Tp*
__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp * > * __a,ptrdiff_t __delta,memory_order __order)193 __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
194 return __c11_atomic_fetch_add(
195 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
196 }
197
198 template <class _Tp>
199 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp __delta,memory_order __order)200 __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
201 return __c11_atomic_fetch_sub(
202 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
203 }
204 template <class _Tp>
205 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a,_Tp __delta,memory_order __order)206 __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
207 return __c11_atomic_fetch_sub(
208 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
209 }
210 template <class _Tp>
211 _LIBCPP_HIDE_FROM_ABI _Tp*
__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp * > volatile * __a,ptrdiff_t __delta,memory_order __order)212 __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
213 return __c11_atomic_fetch_sub(
214 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
215 }
216 template <class _Tp>
217 _LIBCPP_HIDE_FROM_ABI _Tp*
__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp * > * __a,ptrdiff_t __delta,memory_order __order)218 __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
219 return __c11_atomic_fetch_sub(
220 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
221 }
222
223 template <class _Tp>
224 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp __pattern,memory_order __order)225 __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
226 return __c11_atomic_fetch_and(
227 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
228 }
229 template <class _Tp>
230 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a,_Tp __pattern,memory_order __order)231 __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
232 return __c11_atomic_fetch_and(
233 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
234 }
235
236 template <class _Tp>
237 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp __pattern,memory_order __order)238 __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
239 return __c11_atomic_fetch_or(
240 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
241 }
242 template <class _Tp>
243 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a,_Tp __pattern,memory_order __order)244 __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
245 return __c11_atomic_fetch_or(
246 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
247 }
248
249 template <class _Tp>
250 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile * __a,_Tp __pattern,memory_order __order)251 __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
252 return __c11_atomic_fetch_xor(
253 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
254 }
255 template <class _Tp>
256 _LIBCPP_HIDE_FROM_ABI _Tp
__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a,_Tp __pattern,memory_order __order)257 __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
258 return __c11_atomic_fetch_xor(
259 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
260 }
261
262 _LIBCPP_END_NAMESPACE_STD
263
264 #endif // _LIBCPP___ATOMIC_SUPPORT_C11_H
265