1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef _LIBCPP___ATOMIC_ATOMIC_BASE_H 10 #define _LIBCPP___ATOMIC_ATOMIC_BASE_H 11 12 #include <__atomic/atomic_sync.h> 13 #include <__atomic/check_memory_order.h> 14 #include <__atomic/cxx_atomic_impl.h> 15 #include <__atomic/is_always_lock_free.h> 16 #include <__atomic/memory_order.h> 17 #include <__config> 18 #include <__memory/addressof.h> 19 #include <__type_traits/is_integral.h> 20 #include <__type_traits/is_nothrow_constructible.h> 21 #include <__type_traits/is_same.h> 22 #include <version> 23 24 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 25 # pragma GCC system_header 26 #endif 27 28 _LIBCPP_BEGIN_NAMESPACE_STD 29 30 template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 31 struct __atomic_base // false 32 { 33 mutable __cxx_atomic_impl<_Tp> __a_; 34 35 #if _LIBCPP_STD_VER >= 17 36 static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value; 37 #endif 38 is_lock_free__atomic_base39 _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT { 40 return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>)); 41 } is_lock_free__atomic_base42 _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT { 43 return static_cast<__atomic_base const volatile*>(this)->is_lock_free(); 44 } 45 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT _LIBCPP_CHECK_STORE_MEMORY_ORDER__atomic_base46 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { 47 std::__cxx_atomic_store(std::addressof(__a_), __d, __m); 48 } 49 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT _LIBCPP_CHECK_STORE_MEMORY_ORDER__atomic_base50 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { 51 std::__cxx_atomic_store(std::addressof(__a_), __d, __m); 52 } 53 _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER__atomic_base54 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { 55 return std::__cxx_atomic_load(std::addressof(__a_), __m); 56 } 57 _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER__atomic_base58 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { 59 return std::__cxx_atomic_load(std::addressof(__a_), __m); 60 } _Tp__atomic_base61 _LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); } _Tp__atomic_base62 _LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); } 63 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 64 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); 65 } 66 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 67 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); 68 } 69 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak__atomic_base70 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT 71 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 72 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); 73 } compare_exchange_weak__atomic_base74 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT 75 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 76 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); 77 } 78 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong__atomic_base79 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT 80 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 81 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); 82 } compare_exchange_strong__atomic_base83 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT 84 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { 85 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); 86 } 87 _LIBCPP_HIDE_FROM_ABI bool 88 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 89 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); 90 } 91 _LIBCPP_HIDE_FROM_ABI bool 92 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 93 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); 94 } 95 _LIBCPP_HIDE_FROM_ABI bool 96 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 97 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); 98 } 99 _LIBCPP_HIDE_FROM_ABI bool 100 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 101 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); 102 } 103 104 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const 105 volatile _NOEXCEPT { 106 std::__atomic_wait(*this, __v, __m); 107 } 108 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void 109 wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { 110 std::__atomic_wait(*this, __v, __m); 111 } notify_one__atomic_base112 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { 113 std::__atomic_notify_one(*this); 114 } notify_one__atomic_base115 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); } notify_all__atomic_base116 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { 117 std::__atomic_notify_all(*this); 118 } notify_all__atomic_base119 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); } 120 121 #if _LIBCPP_STD_VER >= 20 __atomic_base__atomic_base122 _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} 123 #else 124 _LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default; 125 #endif 126 __atomic_base__atomic_base127 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 128 129 __atomic_base(const __atomic_base&) = delete; 130 }; 131 132 // atomic<Integral> 133 134 template <class _Tp> 135 struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> { 136 using __base = __atomic_base<_Tp, false>; 137 138 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default; 139 140 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 141 142 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 143 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); 144 } 145 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 146 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); 147 } 148 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 149 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); 150 } 151 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 152 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); 153 } 154 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 155 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); 156 } 157 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 158 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); 159 } 160 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 161 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); 162 } 163 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 164 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); 165 } 166 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 167 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); 168 } 169 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 170 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); 171 } 172 173 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); } 174 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); } 175 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); } 176 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); } 177 _LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } 178 _LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } 179 _LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } 180 _LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } 181 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; } 182 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; } 183 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; } 184 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; } 185 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; } 186 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; } 187 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; } 188 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; } 189 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; } 190 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; } 191 }; 192 193 // Here we need _IsIntegral because the default template argument is not enough 194 // e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from 195 // __atomic_base<int, false> and the caller of the wait function is 196 // __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work 197 template <class _Tp, bool _IsIntegral> 198 struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > { 199 static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) { 200 return __a.load(__order); 201 } 202 203 static _LIBCPP_HIDE_FROM_ABI _Tp 204 __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) { 205 return __this.load(__order); 206 } 207 208 static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>* 209 __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) { 210 return std::addressof(__a.__a_); 211 } 212 213 static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>* 214 __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) { 215 return std::addressof(__this.__a_); 216 } 217 }; 218 219 _LIBCPP_END_NAMESPACE_STD 220 221 #endif // _LIBCPP___ATOMIC_ATOMIC_BASE_H 222