1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H 10 #define _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H 11 12 #include <__atomic/is_always_lock_free.h> 13 #include <__atomic/memory_order.h> 14 #include <__config> 15 #include <__memory/addressof.h> 16 #include <__type_traits/conditional.h> 17 #include <__type_traits/is_assignable.h> 18 #include <__type_traits/is_trivially_copyable.h> 19 #include <__type_traits/remove_const.h> 20 #include <cstddef> 21 #include <cstring> 22 23 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 24 # pragma GCC system_header 25 #endif 26 27 _LIBCPP_BEGIN_NAMESPACE_STD 28 29 #if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 30 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 31 32 // [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 33 // the default operator= in an object is not volatile, a byte-by-byte copy 34 // is required. 35 template <typename _Tp, typename _Tv> _LIBCPP_HIDE_FROM_ABI 36 typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 37 __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 38 __a_value = __val; 39 } 40 template <typename _Tp, typename _Tv> _LIBCPP_HIDE_FROM_ABI 41 typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 42 __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 43 volatile char* __to = reinterpret_cast<volatile char*>(std::addressof(__a_value)); 44 volatile char* __end = __to + sizeof(_Tp); 45 volatile const char* __from = reinterpret_cast<volatile const char*>(std::addressof(__val)); 46 while (__to != __end) 47 *__to++ = *__from++; 48 } 49 50 #endif 51 52 #if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 53 54 template <typename _Tp> 55 struct __cxx_atomic_base_impl { 56 57 _LIBCPP_HIDE_FROM_ABI 58 #ifndef _LIBCPP_CXX03_LANG 59 __cxx_atomic_base_impl() _NOEXCEPT = default; 60 #else 61 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 62 #endif // _LIBCPP_CXX03_LANG 63 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 64 : __a_value(value) {} 65 _Tp __a_value; 66 }; 67 68 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 69 // Avoid switch statement to make this a constexpr. 70 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 71 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 72 (__order == memory_order_release ? __ATOMIC_RELEASE: 73 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 74 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 75 __ATOMIC_CONSUME)))); 76 } 77 78 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 79 // Avoid switch statement to make this a constexpr. 80 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 81 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 82 (__order == memory_order_release ? __ATOMIC_RELAXED: 83 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 84 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 85 __ATOMIC_CONSUME)))); 86 } 87 88 template <typename _Tp> 89 _LIBCPP_HIDE_FROM_ABI 90 void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 91 __cxx_atomic_assign_volatile(__a->__a_value, __val); 92 } 93 94 template <typename _Tp> 95 _LIBCPP_HIDE_FROM_ABI 96 void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 97 __a->__a_value = __val; 98 } 99 100 _LIBCPP_HIDE_FROM_ABI inline 101 void __cxx_atomic_thread_fence(memory_order __order) { 102 __atomic_thread_fence(__to_gcc_order(__order)); 103 } 104 105 _LIBCPP_HIDE_FROM_ABI inline 106 void __cxx_atomic_signal_fence(memory_order __order) { 107 __atomic_signal_fence(__to_gcc_order(__order)); 108 } 109 110 template <typename _Tp> 111 _LIBCPP_HIDE_FROM_ABI 112 void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 113 memory_order __order) { 114 __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order)); 115 } 116 117 template <typename _Tp> 118 _LIBCPP_HIDE_FROM_ABI 119 void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 120 memory_order __order) { 121 __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order)); 122 } 123 124 template <typename _Tp> 125 _LIBCPP_HIDE_FROM_ABI 126 _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 127 memory_order __order) { 128 _Tp __ret; 129 __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order)); 130 return __ret; 131 } 132 133 template <typename _Tp> 134 _LIBCPP_HIDE_FROM_ABI 135 _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 136 _Tp __ret; 137 __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order)); 138 return __ret; 139 } 140 141 template <typename _Tp> 142 _LIBCPP_HIDE_FROM_ABI 143 _Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 144 _Tp __value, memory_order __order) { 145 _Tp __ret; 146 __atomic_exchange( 147 std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order)); 148 return __ret; 149 } 150 151 template <typename _Tp> 152 _LIBCPP_HIDE_FROM_ABI 153 _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 154 memory_order __order) { 155 _Tp __ret; 156 __atomic_exchange( 157 std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order)); 158 return __ret; 159 } 160 161 template <typename _Tp> 162 _LIBCPP_HIDE_FROM_ABI 163 bool __cxx_atomic_compare_exchange_strong( 164 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 165 memory_order __success, memory_order __failure) { 166 return __atomic_compare_exchange( 167 std::addressof(__a->__a_value), 168 __expected, 169 std::addressof(__value), 170 false, 171 __to_gcc_order(__success), 172 __to_gcc_failure_order(__failure)); 173 } 174 175 template <typename _Tp> 176 _LIBCPP_HIDE_FROM_ABI 177 bool __cxx_atomic_compare_exchange_strong( 178 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 179 memory_order __failure) { 180 return __atomic_compare_exchange( 181 std::addressof(__a->__a_value), 182 __expected, 183 std::addressof(__value), 184 false, 185 __to_gcc_order(__success), 186 __to_gcc_failure_order(__failure)); 187 } 188 189 template <typename _Tp> 190 _LIBCPP_HIDE_FROM_ABI 191 bool __cxx_atomic_compare_exchange_weak( 192 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 193 memory_order __success, memory_order __failure) { 194 return __atomic_compare_exchange( 195 std::addressof(__a->__a_value), 196 __expected, 197 std::addressof(__value), 198 true, 199 __to_gcc_order(__success), 200 __to_gcc_failure_order(__failure)); 201 } 202 203 template <typename _Tp> 204 _LIBCPP_HIDE_FROM_ABI 205 bool __cxx_atomic_compare_exchange_weak( 206 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 207 memory_order __failure) { 208 return __atomic_compare_exchange( 209 std::addressof(__a->__a_value), 210 __expected, 211 std::addressof(__value), 212 true, 213 __to_gcc_order(__success), 214 __to_gcc_failure_order(__failure)); 215 } 216 217 template <typename _Tp> 218 struct __skip_amt { enum {value = 1}; }; 219 220 template <typename _Tp> 221 struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 222 223 // FIXME: Haven't figured out what the spec says about using arrays with 224 // atomic_fetch_add. Force a failure rather than creating bad behavior. 225 template <typename _Tp> 226 struct __skip_amt<_Tp[]> { }; 227 template <typename _Tp, int n> 228 struct __skip_amt<_Tp[n]> { }; 229 230 template <typename _Tp, typename _Td> 231 _LIBCPP_HIDE_FROM_ABI 232 _Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 233 _Td __delta, memory_order __order) { 234 return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); 235 } 236 237 template <typename _Tp, typename _Td> 238 _LIBCPP_HIDE_FROM_ABI 239 _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 240 memory_order __order) { 241 return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); 242 } 243 244 template <typename _Tp, typename _Td> 245 _LIBCPP_HIDE_FROM_ABI 246 _Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 247 _Td __delta, memory_order __order) { 248 return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); 249 } 250 251 template <typename _Tp, typename _Td> 252 _LIBCPP_HIDE_FROM_ABI 253 _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 254 memory_order __order) { 255 return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); 256 } 257 258 template <typename _Tp> 259 _LIBCPP_HIDE_FROM_ABI 260 _Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 261 _Tp __pattern, memory_order __order) { 262 return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); 263 } 264 265 template <typename _Tp> 266 _LIBCPP_HIDE_FROM_ABI 267 _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 268 _Tp __pattern, memory_order __order) { 269 return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); 270 } 271 272 template <typename _Tp> 273 _LIBCPP_HIDE_FROM_ABI 274 _Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 275 _Tp __pattern, memory_order __order) { 276 return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); 277 } 278 279 template <typename _Tp> 280 _LIBCPP_HIDE_FROM_ABI 281 _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 282 memory_order __order) { 283 return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); 284 } 285 286 template <typename _Tp> 287 _LIBCPP_HIDE_FROM_ABI 288 _Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 289 _Tp __pattern, memory_order __order) { 290 return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); 291 } 292 293 template <typename _Tp> 294 _LIBCPP_HIDE_FROM_ABI 295 _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 296 memory_order __order) { 297 return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); 298 } 299 300 #define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 301 302 #elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 303 304 template <typename _Tp> 305 struct __cxx_atomic_base_impl { 306 307 _LIBCPP_HIDE_FROM_ABI 308 #ifndef _LIBCPP_CXX03_LANG 309 __cxx_atomic_base_impl() _NOEXCEPT = default; 310 #else 311 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 312 #endif // _LIBCPP_CXX03_LANG 313 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT 314 : __a_value(__value) {} 315 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 316 }; 317 318 #define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 319 320 _LIBCPP_HIDE_FROM_ABI inline 321 void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 322 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 323 } 324 325 _LIBCPP_HIDE_FROM_ABI inline 326 void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 327 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 328 } 329 330 template<class _Tp> 331 _LIBCPP_HIDE_FROM_ABI 332 void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 333 __c11_atomic_init(std::addressof(__a->__a_value), __val); 334 } 335 template<class _Tp> 336 _LIBCPP_HIDE_FROM_ABI 337 void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 338 __c11_atomic_init(std::addressof(__a->__a_value), __val); 339 } 340 341 template<class _Tp> 342 _LIBCPP_HIDE_FROM_ABI 343 void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 344 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); 345 } 346 template<class _Tp> 347 _LIBCPP_HIDE_FROM_ABI 348 void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 349 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); 350 } 351 352 template<class _Tp> 353 _LIBCPP_HIDE_FROM_ABI 354 _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 355 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; 356 return __c11_atomic_load( 357 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); 358 } 359 template<class _Tp> 360 _LIBCPP_HIDE_FROM_ABI 361 _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 362 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; 363 return __c11_atomic_load( 364 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); 365 } 366 367 template<class _Tp> 368 _LIBCPP_HIDE_FROM_ABI 369 _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 370 return __c11_atomic_exchange( 371 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); 372 } 373 template<class _Tp> 374 _LIBCPP_HIDE_FROM_ABI 375 _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 376 return __c11_atomic_exchange( 377 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); 378 } 379 380 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { 381 // Avoid switch statement to make this a constexpr. 382 return __order == memory_order_release ? memory_order_relaxed: 383 (__order == memory_order_acq_rel ? memory_order_acquire: 384 __order); 385 } 386 387 template<class _Tp> 388 _LIBCPP_HIDE_FROM_ABI 389 bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 390 return __c11_atomic_compare_exchange_strong( 391 std::addressof(__a->__a_value), 392 __expected, 393 __value, 394 static_cast<__memory_order_underlying_t>(__success), 395 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 396 } 397 template<class _Tp> 398 _LIBCPP_HIDE_FROM_ABI 399 bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 400 return __c11_atomic_compare_exchange_strong( 401 std::addressof(__a->__a_value), 402 __expected, 403 __value, 404 static_cast<__memory_order_underlying_t>(__success), 405 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 406 } 407 408 template<class _Tp> 409 _LIBCPP_HIDE_FROM_ABI 410 bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 411 return __c11_atomic_compare_exchange_weak( 412 std::addressof(__a->__a_value), 413 __expected, 414 __value, 415 static_cast<__memory_order_underlying_t>(__success), 416 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 417 } 418 template<class _Tp> 419 _LIBCPP_HIDE_FROM_ABI 420 bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 421 return __c11_atomic_compare_exchange_weak( 422 std::addressof(__a->__a_value), 423 __expected, 424 __value, 425 static_cast<__memory_order_underlying_t>(__success), 426 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 427 } 428 429 template<class _Tp> 430 _LIBCPP_HIDE_FROM_ABI 431 _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 432 return __c11_atomic_fetch_add( 433 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); 434 } 435 template<class _Tp> 436 _LIBCPP_HIDE_FROM_ABI 437 _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 438 return __c11_atomic_fetch_add( 439 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); 440 } 441 442 template<class _Tp> 443 _LIBCPP_HIDE_FROM_ABI 444 _Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 445 return __c11_atomic_fetch_add( 446 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); 447 } 448 template<class _Tp> 449 _LIBCPP_HIDE_FROM_ABI 450 _Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 451 return __c11_atomic_fetch_add( 452 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); 453 } 454 455 template<class _Tp> 456 _LIBCPP_HIDE_FROM_ABI 457 _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 458 return __c11_atomic_fetch_sub( 459 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); 460 } 461 template<class _Tp> 462 _LIBCPP_HIDE_FROM_ABI 463 _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 464 return __c11_atomic_fetch_sub( 465 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); 466 } 467 template<class _Tp> 468 _LIBCPP_HIDE_FROM_ABI 469 _Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 470 return __c11_atomic_fetch_sub( 471 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); 472 } 473 template<class _Tp> 474 _LIBCPP_HIDE_FROM_ABI 475 _Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 476 return __c11_atomic_fetch_sub( 477 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); 478 } 479 480 template<class _Tp> 481 _LIBCPP_HIDE_FROM_ABI 482 _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 483 return __c11_atomic_fetch_and( 484 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); 485 } 486 template<class _Tp> 487 _LIBCPP_HIDE_FROM_ABI 488 _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 489 return __c11_atomic_fetch_and( 490 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); 491 } 492 493 template<class _Tp> 494 _LIBCPP_HIDE_FROM_ABI 495 _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 496 return __c11_atomic_fetch_or( 497 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); 498 } 499 template<class _Tp> 500 _LIBCPP_HIDE_FROM_ABI 501 _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 502 return __c11_atomic_fetch_or( 503 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); 504 } 505 506 template<class _Tp> 507 _LIBCPP_HIDE_FROM_ABI 508 _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 509 return __c11_atomic_fetch_xor( 510 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); 511 } 512 template<class _Tp> 513 _LIBCPP_HIDE_FROM_ABI 514 _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 515 return __c11_atomic_fetch_xor( 516 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); 517 } 518 519 #endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 520 521 #ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 522 523 template<typename _Tp> 524 struct __cxx_atomic_lock_impl { 525 526 _LIBCPP_HIDE_FROM_ABI 527 __cxx_atomic_lock_impl() _NOEXCEPT 528 : __a_value(), __a_lock(0) {} 529 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit 530 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 531 : __a_value(value), __a_lock(0) {} 532 533 _Tp __a_value; 534 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 535 536 _LIBCPP_HIDE_FROM_ABI void __lock() const volatile { 537 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 538 /*spin*/; 539 } 540 _LIBCPP_HIDE_FROM_ABI void __lock() const { 541 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 542 /*spin*/; 543 } 544 _LIBCPP_HIDE_FROM_ABI void __unlock() const volatile { 545 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 546 } 547 _LIBCPP_HIDE_FROM_ABI void __unlock() const { 548 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 549 } 550 _LIBCPP_HIDE_FROM_ABI _Tp __read() const volatile { 551 __lock(); 552 _Tp __old; 553 __cxx_atomic_assign_volatile(__old, __a_value); 554 __unlock(); 555 return __old; 556 } 557 _LIBCPP_HIDE_FROM_ABI _Tp __read() const { 558 __lock(); 559 _Tp __old = __a_value; 560 __unlock(); 561 return __old; 562 } 563 }; 564 565 template <typename _Tp> 566 _LIBCPP_HIDE_FROM_ABI 567 void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 568 __cxx_atomic_assign_volatile(__a->__a_value, __val); 569 } 570 template <typename _Tp> 571 _LIBCPP_HIDE_FROM_ABI 572 void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 573 __a->__a_value = __val; 574 } 575 576 template <typename _Tp> 577 _LIBCPP_HIDE_FROM_ABI 578 void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 579 __a->__lock(); 580 __cxx_atomic_assign_volatile(__a->__a_value, __val); 581 __a->__unlock(); 582 } 583 template <typename _Tp> 584 _LIBCPP_HIDE_FROM_ABI 585 void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 586 __a->__lock(); 587 __a->__a_value = __val; 588 __a->__unlock(); 589 } 590 591 template <typename _Tp> 592 _LIBCPP_HIDE_FROM_ABI 593 _Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 594 return __a->__read(); 595 } 596 template <typename _Tp> 597 _LIBCPP_HIDE_FROM_ABI 598 _Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 599 return __a->__read(); 600 } 601 602 template <typename _Tp> 603 _LIBCPP_HIDE_FROM_ABI 604 _Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 605 __a->__lock(); 606 _Tp __old; 607 __cxx_atomic_assign_volatile(__old, __a->__a_value); 608 __cxx_atomic_assign_volatile(__a->__a_value, __value); 609 __a->__unlock(); 610 return __old; 611 } 612 template <typename _Tp> 613 _LIBCPP_HIDE_FROM_ABI 614 _Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 615 __a->__lock(); 616 _Tp __old = __a->__a_value; 617 __a->__a_value = __value; 618 __a->__unlock(); 619 return __old; 620 } 621 622 template <typename _Tp> 623 _LIBCPP_HIDE_FROM_ABI 624 bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 625 _Tp* __expected, _Tp __value, memory_order, memory_order) { 626 _Tp __temp; 627 __a->__lock(); 628 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 629 bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 630 if(__ret) 631 __cxx_atomic_assign_volatile(__a->__a_value, __value); 632 else 633 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 634 __a->__unlock(); 635 return __ret; 636 } 637 template <typename _Tp> 638 _LIBCPP_HIDE_FROM_ABI 639 bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 640 _Tp* __expected, _Tp __value, memory_order, memory_order) { 641 __a->__lock(); 642 bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 643 if(__ret) 644 std::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 645 else 646 std::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 647 __a->__unlock(); 648 return __ret; 649 } 650 651 template <typename _Tp> 652 _LIBCPP_HIDE_FROM_ABI 653 bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 654 _Tp* __expected, _Tp __value, memory_order, memory_order) { 655 _Tp __temp; 656 __a->__lock(); 657 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 658 bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 659 if(__ret) 660 __cxx_atomic_assign_volatile(__a->__a_value, __value); 661 else 662 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 663 __a->__unlock(); 664 return __ret; 665 } 666 template <typename _Tp> 667 _LIBCPP_HIDE_FROM_ABI 668 bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 669 _Tp* __expected, _Tp __value, memory_order, memory_order) { 670 __a->__lock(); 671 bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 672 if(__ret) 673 std::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 674 else 675 std::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 676 __a->__unlock(); 677 return __ret; 678 } 679 680 template <typename _Tp, typename _Td> 681 _LIBCPP_HIDE_FROM_ABI 682 _Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 683 _Td __delta, memory_order) { 684 __a->__lock(); 685 _Tp __old; 686 __cxx_atomic_assign_volatile(__old, __a->__a_value); 687 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 688 __a->__unlock(); 689 return __old; 690 } 691 template <typename _Tp, typename _Td> 692 _LIBCPP_HIDE_FROM_ABI 693 _Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 694 _Td __delta, memory_order) { 695 __a->__lock(); 696 _Tp __old = __a->__a_value; 697 __a->__a_value += __delta; 698 __a->__unlock(); 699 return __old; 700 } 701 702 template <typename _Tp, typename _Td> 703 _LIBCPP_HIDE_FROM_ABI 704 _Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 705 ptrdiff_t __delta, memory_order) { 706 __a->__lock(); 707 _Tp* __old; 708 __cxx_atomic_assign_volatile(__old, __a->__a_value); 709 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 710 __a->__unlock(); 711 return __old; 712 } 713 template <typename _Tp, typename _Td> 714 _LIBCPP_HIDE_FROM_ABI 715 _Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 716 ptrdiff_t __delta, memory_order) { 717 __a->__lock(); 718 _Tp* __old = __a->__a_value; 719 __a->__a_value += __delta; 720 __a->__unlock(); 721 return __old; 722 } 723 724 template <typename _Tp, typename _Td> 725 _LIBCPP_HIDE_FROM_ABI 726 _Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 727 _Td __delta, memory_order) { 728 __a->__lock(); 729 _Tp __old; 730 __cxx_atomic_assign_volatile(__old, __a->__a_value); 731 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 732 __a->__unlock(); 733 return __old; 734 } 735 template <typename _Tp, typename _Td> 736 _LIBCPP_HIDE_FROM_ABI 737 _Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 738 _Td __delta, memory_order) { 739 __a->__lock(); 740 _Tp __old = __a->__a_value; 741 __a->__a_value -= __delta; 742 __a->__unlock(); 743 return __old; 744 } 745 746 template <typename _Tp> 747 _LIBCPP_HIDE_FROM_ABI 748 _Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 749 _Tp __pattern, memory_order) { 750 __a->__lock(); 751 _Tp __old; 752 __cxx_atomic_assign_volatile(__old, __a->__a_value); 753 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 754 __a->__unlock(); 755 return __old; 756 } 757 template <typename _Tp> 758 _LIBCPP_HIDE_FROM_ABI 759 _Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 760 _Tp __pattern, memory_order) { 761 __a->__lock(); 762 _Tp __old = __a->__a_value; 763 __a->__a_value &= __pattern; 764 __a->__unlock(); 765 return __old; 766 } 767 768 template <typename _Tp> 769 _LIBCPP_HIDE_FROM_ABI 770 _Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 771 _Tp __pattern, memory_order) { 772 __a->__lock(); 773 _Tp __old; 774 __cxx_atomic_assign_volatile(__old, __a->__a_value); 775 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 776 __a->__unlock(); 777 return __old; 778 } 779 template <typename _Tp> 780 _LIBCPP_HIDE_FROM_ABI 781 _Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 782 _Tp __pattern, memory_order) { 783 __a->__lock(); 784 _Tp __old = __a->__a_value; 785 __a->__a_value |= __pattern; 786 __a->__unlock(); 787 return __old; 788 } 789 790 template <typename _Tp> 791 _LIBCPP_HIDE_FROM_ABI 792 _Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 793 _Tp __pattern, memory_order) { 794 __a->__lock(); 795 _Tp __old; 796 __cxx_atomic_assign_volatile(__old, __a->__a_value); 797 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 798 __a->__unlock(); 799 return __old; 800 } 801 template <typename _Tp> 802 _LIBCPP_HIDE_FROM_ABI 803 _Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 804 _Tp __pattern, memory_order) { 805 __a->__lock(); 806 _Tp __old = __a->__a_value; 807 __a->__a_value ^= __pattern; 808 __a->__unlock(); 809 return __old; 810 } 811 812 template <typename _Tp, 813 typename _Base = typename conditional<__libcpp_is_always_lock_free<_Tp>::__value, 814 __cxx_atomic_base_impl<_Tp>, 815 __cxx_atomic_lock_impl<_Tp> >::type> 816 #else 817 template <typename _Tp, 818 typename _Base = __cxx_atomic_base_impl<_Tp> > 819 #endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 820 struct __cxx_atomic_impl : public _Base { 821 static_assert(is_trivially_copyable<_Tp>::value, 822 "std::atomic<T> requires that 'T' be a trivially copyable type"); 823 824 _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default; 825 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT 826 : _Base(__value) {} 827 }; 828 829 _LIBCPP_END_NAMESPACE_STD 830 831 #endif // _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H 832