1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H 10 #define _LIBCPP___ATOMIC_ATOMIC_SYNC_H 11 12 #include <__atomic/contention_t.h> 13 #include <__atomic/cxx_atomic_impl.h> 14 #include <__atomic/memory_order.h> 15 #include <__availability> 16 #include <__chrono/duration.h> 17 #include <__config> 18 #include <__memory/addressof.h> 19 #include <__thread/poll_with_backoff.h> 20 #include <__threading_support> 21 #include <__type_traits/decay.h> 22 #include <cstring> 23 24 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 25 # pragma GCC system_header 26 #endif 27 28 _LIBCPP_BEGIN_NAMESPACE_STD 29 30 #ifndef _LIBCPP_HAS_NO_THREADS 31 32 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 33 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 34 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 35 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 36 37 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void 38 __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 39 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void 40 __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 41 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t 42 __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 43 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void 44 __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 45 46 template <class _Atp, class _Fn> 47 struct __libcpp_atomic_wait_backoff_impl { 48 _Atp* __a; 49 _Fn __test_fn; 50 _LIBCPP_AVAILABILITY_SYNC 51 _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const { 52 if (__elapsed > chrono::microseconds(64)) { 53 auto const __monitor = std::__libcpp_atomic_monitor(__a); 54 if (__test_fn()) 55 return true; 56 std::__libcpp_atomic_wait(__a, __monitor); 57 } else if (__elapsed > chrono::microseconds(4)) 58 __libcpp_thread_yield(); 59 else { 60 } // poll 61 return false; 62 } 63 }; 64 65 template <class _Atp, class _Fn> 66 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp* __a, _Fn&& __test_fn) { 67 __libcpp_atomic_wait_backoff_impl<_Atp, __decay_t<_Fn> > __backoff_fn = {__a, __test_fn}; 68 return std::__libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 69 } 70 71 #else // _LIBCPP_HAS_NO_THREADS 72 73 template <class _Tp> 74 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) {} 75 template <class _Tp> 76 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) {} 77 template <class _Atp, class _Fn> 78 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp*, _Fn&& __test_fn) { 79 return std::__libcpp_thread_poll_with_backoff(__test_fn, __spinning_backoff_policy()); 80 } 81 82 #endif // _LIBCPP_HAS_NO_THREADS 83 84 template <typename _Tp> 85 _LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 86 return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0; 87 } 88 89 template <class _Atp, class _Tp> 90 struct __cxx_atomic_wait_test_fn_impl { 91 _Atp* __a; 92 _Tp __val; 93 memory_order __order; 94 _LIBCPP_HIDE_FROM_ABI bool operator()() const { 95 return !std::__cxx_nonatomic_compare_equal(std::__cxx_atomic_load(__a, __order), __val); 96 } 97 }; 98 99 template <class _Atp, class _Tp> 100 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool 101 __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) { 102 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 103 return std::__cxx_atomic_wait(__a, __test_fn); 104 } 105 106 _LIBCPP_END_NAMESPACE_STD 107 108 #endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_H 109