1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include <__config> 10 #ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS 11 # define _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS 12 #endif 13 14 #include <memory> 15 16 #ifndef _LIBCPP_HAS_NO_THREADS 17 # include <mutex> 18 # include <thread> 19 # if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB) 20 # pragma comment(lib, "pthread") 21 # endif 22 #endif 23 24 #include "include/atomic_support.h" 25 26 _LIBCPP_BEGIN_NAMESPACE_STD 27 28 const allocator_arg_t allocator_arg = allocator_arg_t(); 29 30 bad_weak_ptr::~bad_weak_ptr() noexcept {} 31 32 const char* 33 bad_weak_ptr::what() const noexcept 34 { 35 return "bad_weak_ptr"; 36 } 37 38 __shared_count::~__shared_count() 39 { 40 } 41 42 __shared_weak_count::~__shared_weak_count() 43 { 44 } 45 46 #if defined(_LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS) 47 void 48 __shared_count::__add_shared() noexcept 49 { 50 __libcpp_atomic_refcount_increment(__shared_owners_); 51 } 52 53 bool 54 __shared_count::__release_shared() noexcept 55 { 56 if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) 57 { 58 __on_zero_shared(); 59 return true; 60 } 61 return false; 62 } 63 64 void 65 __shared_weak_count::__add_shared() noexcept 66 { 67 __shared_count::__add_shared(); 68 } 69 70 void 71 __shared_weak_count::__add_weak() noexcept 72 { 73 __libcpp_atomic_refcount_increment(__shared_weak_owners_); 74 } 75 76 void 77 __shared_weak_count::__release_shared() noexcept 78 { 79 if (__shared_count::__release_shared()) 80 __release_weak(); 81 } 82 #endif // _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS 83 84 void 85 __shared_weak_count::__release_weak() noexcept 86 { 87 // NOTE: The acquire load here is an optimization of the very 88 // common case where a shared pointer is being destructed while 89 // having no other contended references. 90 // 91 // BENEFIT: We avoid expensive atomic stores like XADD and STREX 92 // in a common case. Those instructions are slow and do nasty 93 // things to caches. 94 // 95 // IS THIS SAFE? Yes. During weak destruction, if we see that we 96 // are the last reference, we know that no-one else is accessing 97 // us. If someone were accessing us, then they would be doing so 98 // while the last shared / weak_ptr was being destructed, and 99 // that's undefined anyway. 100 // 101 // If we see anything other than a 0, then we have possible 102 // contention, and need to use an atomicrmw primitive. 103 // The same arguments don't apply for increment, where it is legal 104 // (though inadvisable) to share shared_ptr references between 105 // threads, and have them all get copied at once. The argument 106 // also doesn't apply for __release_shared, because an outstanding 107 // weak_ptr::lock() could read / modify the shared count. 108 if (__libcpp_atomic_load(&__shared_weak_owners_, _AO_Acquire) == 0) 109 { 110 // no need to do this store, because we are about 111 // to destroy everything. 112 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release); 113 __on_zero_shared_weak(); 114 } 115 else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1) 116 __on_zero_shared_weak(); 117 } 118 119 __shared_weak_count* 120 __shared_weak_count::lock() noexcept 121 { 122 long object_owners = __libcpp_atomic_load(&__shared_owners_); 123 while (object_owners != -1) 124 { 125 if (__libcpp_atomic_compare_exchange(&__shared_owners_, 126 &object_owners, 127 object_owners+1)) 128 return this; 129 } 130 return nullptr; 131 } 132 133 const void* 134 __shared_weak_count::__get_deleter(const type_info&) const noexcept 135 { 136 return nullptr; 137 } 138 139 #if !defined(_LIBCPP_HAS_NO_THREADS) 140 141 static constexpr std::size_t __sp_mut_count = 32; 142 static constinit __libcpp_mutex_t mut_back[__sp_mut_count] = 143 { 144 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 145 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 146 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 147 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 148 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 149 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 150 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 151 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER 152 }; 153 154 _LIBCPP_CONSTEXPR __sp_mut::__sp_mut(void* p) noexcept 155 : __lx(p) 156 { 157 } 158 159 void 160 __sp_mut::lock() noexcept 161 { 162 auto m = static_cast<__libcpp_mutex_t*>(__lx); 163 __libcpp_mutex_lock(m); 164 } 165 166 void 167 __sp_mut::unlock() noexcept 168 { 169 __libcpp_mutex_unlock(static_cast<__libcpp_mutex_t*>(__lx)); 170 } 171 172 __sp_mut& 173 __get_sp_mut(const void* p) 174 { 175 static constinit __sp_mut muts[__sp_mut_count] = { 176 &mut_back[ 0], &mut_back[ 1], &mut_back[ 2], &mut_back[ 3], 177 &mut_back[ 4], &mut_back[ 5], &mut_back[ 6], &mut_back[ 7], 178 &mut_back[ 8], &mut_back[ 9], &mut_back[10], &mut_back[11], 179 &mut_back[12], &mut_back[13], &mut_back[14], &mut_back[15], 180 &mut_back[16], &mut_back[17], &mut_back[18], &mut_back[19], 181 &mut_back[20], &mut_back[21], &mut_back[22], &mut_back[23], 182 &mut_back[24], &mut_back[25], &mut_back[26], &mut_back[27], 183 &mut_back[28], &mut_back[29], &mut_back[30], &mut_back[31] 184 }; 185 return muts[hash<const void*>()(p) & (__sp_mut_count-1)]; 186 } 187 188 #endif // !defined(_LIBCPP_HAS_NO_THREADS) 189 190 void* 191 align(size_t alignment, size_t size, void*& ptr, size_t& space) 192 { 193 void* r = nullptr; 194 if (size <= space) 195 { 196 char* p1 = static_cast<char*>(ptr); 197 char* p2 = reinterpret_cast<char*>(reinterpret_cast<size_t>(p1 + (alignment - 1)) & -alignment); 198 size_t d = static_cast<size_t>(p2 - p1); 199 if (d <= space - size) 200 { 201 r = p2; 202 ptr = r; 203 space -= d; 204 } 205 } 206 return r; 207 } 208 209 _LIBCPP_END_NAMESPACE_STD 210