1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H 10 #define _LIBCPP___ATOMIC_ATOMIC_SYNC_H 11 12 #include <__atomic/contention_t.h> 13 #include <__atomic/cxx_atomic_impl.h> 14 #include <__atomic/memory_order.h> 15 #include <__availability> 16 #include <__chrono/duration.h> 17 #include <__config> 18 #include <__memory/addressof.h> 19 #include <__thread/poll_with_backoff.h> 20 #include <__threading_support> 21 #include <__type_traits/decay.h> 22 #include <cstring> 23 24 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 25 # pragma GCC system_header 26 #endif 27 28 _LIBCPP_BEGIN_NAMESPACE_STD 29 30 #ifndef _LIBCPP_HAS_NO_THREADS 31 32 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 33 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 34 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 35 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 36 37 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 38 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 39 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 40 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 41 42 template <class _Atp, class _Fn> 43 struct __libcpp_atomic_wait_backoff_impl { 44 _Atp* __a; 45 _Fn __test_fn; 46 _LIBCPP_AVAILABILITY_SYNC 47 _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const 48 { 49 if(__elapsed > chrono::microseconds(64)) 50 { 51 auto const __monitor = std::__libcpp_atomic_monitor(__a); 52 if(__test_fn()) 53 return true; 54 std::__libcpp_atomic_wait(__a, __monitor); 55 } 56 else if(__elapsed > chrono::microseconds(4)) 57 __libcpp_thread_yield(); 58 else 59 {} // poll 60 return false; 61 } 62 }; 63 64 template <class _Atp, class _Fn> 65 _LIBCPP_AVAILABILITY_SYNC 66 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 67 { 68 __libcpp_atomic_wait_backoff_impl<_Atp, __decay_t<_Fn> > __backoff_fn = {__a, __test_fn}; 69 return std::__libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 70 } 71 72 #else // _LIBCPP_HAS_NO_THREADS 73 74 template <class _Tp> 75 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 76 template <class _Tp> 77 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 78 template <class _Atp, class _Fn> 79 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 80 { 81 return std::__libcpp_thread_poll_with_backoff(__test_fn, __spinning_backoff_policy()); 82 } 83 84 #endif // _LIBCPP_HAS_NO_THREADS 85 86 template <typename _Tp> _LIBCPP_HIDE_FROM_ABI 87 bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 88 return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0; 89 } 90 91 template <class _Atp, class _Tp> 92 struct __cxx_atomic_wait_test_fn_impl { 93 _Atp* __a; 94 _Tp __val; 95 memory_order __order; 96 _LIBCPP_HIDE_FROM_ABI bool operator()() const 97 { 98 return !std::__cxx_nonatomic_compare_equal(std::__cxx_atomic_load(__a, __order), __val); 99 } 100 }; 101 102 template <class _Atp, class _Tp> 103 _LIBCPP_AVAILABILITY_SYNC 104 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 105 { 106 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 107 return std::__cxx_atomic_wait(__a, __test_fn); 108 } 109 110 _LIBCPP_END_NAMESPACE_STD 111 112 #endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_H 113