1*349cc55cSDimitry Andric //===----------------------------------------------------------------------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric 90b57cec5SDimitry Andric #include "memory" 100b57cec5SDimitry Andric #ifndef _LIBCPP_HAS_NO_THREADS 110b57cec5SDimitry Andric # include "mutex" 120b57cec5SDimitry Andric # include "thread" 13480093f4SDimitry Andric # if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB) 140b57cec5SDimitry Andric # pragma comment(lib, "pthread") 150b57cec5SDimitry Andric # endif 160b57cec5SDimitry Andric #endif 170b57cec5SDimitry Andric #include "include/atomic_support.h" 180b57cec5SDimitry Andric 190b57cec5SDimitry Andric _LIBCPP_BEGIN_NAMESPACE_STD 200b57cec5SDimitry Andric 210b57cec5SDimitry Andric const allocator_arg_t allocator_arg = allocator_arg_t(); 220b57cec5SDimitry Andric 23fe6060f1SDimitry Andric bad_weak_ptr::~bad_weak_ptr() noexcept {} 240b57cec5SDimitry Andric 250b57cec5SDimitry Andric const char* 26fe6060f1SDimitry Andric bad_weak_ptr::what() const noexcept 270b57cec5SDimitry Andric { 280b57cec5SDimitry Andric return "bad_weak_ptr"; 290b57cec5SDimitry Andric } 300b57cec5SDimitry Andric 310b57cec5SDimitry Andric __shared_count::~__shared_count() 320b57cec5SDimitry Andric { 330b57cec5SDimitry Andric } 340b57cec5SDimitry Andric 350b57cec5SDimitry Andric __shared_weak_count::~__shared_weak_count() 360b57cec5SDimitry Andric { 370b57cec5SDimitry Andric } 380b57cec5SDimitry Andric 390b57cec5SDimitry Andric #if defined(_LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS) 400b57cec5SDimitry Andric void 41fe6060f1SDimitry Andric __shared_count::__add_shared() noexcept 420b57cec5SDimitry Andric { 430b57cec5SDimitry Andric __libcpp_atomic_refcount_increment(__shared_owners_); 440b57cec5SDimitry Andric } 450b57cec5SDimitry Andric 460b57cec5SDimitry Andric bool 47fe6060f1SDimitry Andric __shared_count::__release_shared() noexcept 480b57cec5SDimitry Andric { 490b57cec5SDimitry Andric if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) 500b57cec5SDimitry Andric { 510b57cec5SDimitry Andric __on_zero_shared(); 520b57cec5SDimitry Andric return true; 530b57cec5SDimitry Andric } 540b57cec5SDimitry Andric return false; 550b57cec5SDimitry Andric } 560b57cec5SDimitry Andric 570b57cec5SDimitry Andric void 58fe6060f1SDimitry Andric __shared_weak_count::__add_shared() noexcept 590b57cec5SDimitry Andric { 600b57cec5SDimitry Andric __shared_count::__add_shared(); 610b57cec5SDimitry Andric } 620b57cec5SDimitry Andric 630b57cec5SDimitry Andric void 64fe6060f1SDimitry Andric __shared_weak_count::__add_weak() noexcept 650b57cec5SDimitry Andric { 660b57cec5SDimitry Andric __libcpp_atomic_refcount_increment(__shared_weak_owners_); 670b57cec5SDimitry Andric } 680b57cec5SDimitry Andric 690b57cec5SDimitry Andric void 70fe6060f1SDimitry Andric __shared_weak_count::__release_shared() noexcept 710b57cec5SDimitry Andric { 720b57cec5SDimitry Andric if (__shared_count::__release_shared()) 730b57cec5SDimitry Andric __release_weak(); 740b57cec5SDimitry Andric } 750b57cec5SDimitry Andric 760b57cec5SDimitry Andric #endif // _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS 770b57cec5SDimitry Andric 780b57cec5SDimitry Andric void 79fe6060f1SDimitry Andric __shared_weak_count::__release_weak() noexcept 800b57cec5SDimitry Andric { 810b57cec5SDimitry Andric // NOTE: The acquire load here is an optimization of the very 820b57cec5SDimitry Andric // common case where a shared pointer is being destructed while 830b57cec5SDimitry Andric // having no other contended references. 840b57cec5SDimitry Andric // 850b57cec5SDimitry Andric // BENEFIT: We avoid expensive atomic stores like XADD and STREX 860b57cec5SDimitry Andric // in a common case. Those instructions are slow and do nasty 870b57cec5SDimitry Andric // things to caches. 880b57cec5SDimitry Andric // 890b57cec5SDimitry Andric // IS THIS SAFE? Yes. During weak destruction, if we see that we 900b57cec5SDimitry Andric // are the last reference, we know that no-one else is accessing 910b57cec5SDimitry Andric // us. If someone were accessing us, then they would be doing so 920b57cec5SDimitry Andric // while the last shared / weak_ptr was being destructed, and 930b57cec5SDimitry Andric // that's undefined anyway. 940b57cec5SDimitry Andric // 950b57cec5SDimitry Andric // If we see anything other than a 0, then we have possible 960b57cec5SDimitry Andric // contention, and need to use an atomicrmw primitive. 970b57cec5SDimitry Andric // The same arguments don't apply for increment, where it is legal 980b57cec5SDimitry Andric // (though inadvisable) to share shared_ptr references between 990b57cec5SDimitry Andric // threads, and have them all get copied at once. The argument 1000b57cec5SDimitry Andric // also doesn't apply for __release_shared, because an outstanding 1010b57cec5SDimitry Andric // weak_ptr::lock() could read / modify the shared count. 1020b57cec5SDimitry Andric if (__libcpp_atomic_load(&__shared_weak_owners_, _AO_Acquire) == 0) 1030b57cec5SDimitry Andric { 1040b57cec5SDimitry Andric // no need to do this store, because we are about 1050b57cec5SDimitry Andric // to destroy everything. 1060b57cec5SDimitry Andric //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release); 1070b57cec5SDimitry Andric __on_zero_shared_weak(); 1080b57cec5SDimitry Andric } 1090b57cec5SDimitry Andric else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1) 1100b57cec5SDimitry Andric __on_zero_shared_weak(); 1110b57cec5SDimitry Andric } 1120b57cec5SDimitry Andric 1130b57cec5SDimitry Andric __shared_weak_count* 114fe6060f1SDimitry Andric __shared_weak_count::lock() noexcept 1150b57cec5SDimitry Andric { 1160b57cec5SDimitry Andric long object_owners = __libcpp_atomic_load(&__shared_owners_); 1170b57cec5SDimitry Andric while (object_owners != -1) 1180b57cec5SDimitry Andric { 1190b57cec5SDimitry Andric if (__libcpp_atomic_compare_exchange(&__shared_owners_, 1200b57cec5SDimitry Andric &object_owners, 1210b57cec5SDimitry Andric object_owners+1)) 1220b57cec5SDimitry Andric return this; 1230b57cec5SDimitry Andric } 1240b57cec5SDimitry Andric return nullptr; 1250b57cec5SDimitry Andric } 1260b57cec5SDimitry Andric 1270b57cec5SDimitry Andric const void* 128fe6060f1SDimitry Andric __shared_weak_count::__get_deleter(const type_info&) const noexcept 1290b57cec5SDimitry Andric { 1300b57cec5SDimitry Andric return nullptr; 1310b57cec5SDimitry Andric } 1320b57cec5SDimitry Andric 133*349cc55cSDimitry Andric #if !defined(_LIBCPP_HAS_NO_THREADS) 1340b57cec5SDimitry Andric 1350b57cec5SDimitry Andric _LIBCPP_SAFE_STATIC static const std::size_t __sp_mut_count = 16; 1360b57cec5SDimitry Andric _LIBCPP_SAFE_STATIC static __libcpp_mutex_t mut_back[__sp_mut_count] = 1370b57cec5SDimitry Andric { 1380b57cec5SDimitry Andric _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 1390b57cec5SDimitry Andric _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 1400b57cec5SDimitry Andric _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 1410b57cec5SDimitry Andric _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER 1420b57cec5SDimitry Andric }; 1430b57cec5SDimitry Andric 144fe6060f1SDimitry Andric _LIBCPP_CONSTEXPR __sp_mut::__sp_mut(void* p) noexcept 1450b57cec5SDimitry Andric : __lx(p) 1460b57cec5SDimitry Andric { 1470b57cec5SDimitry Andric } 1480b57cec5SDimitry Andric 1490b57cec5SDimitry Andric void 150fe6060f1SDimitry Andric __sp_mut::lock() noexcept 1510b57cec5SDimitry Andric { 1520b57cec5SDimitry Andric auto m = static_cast<__libcpp_mutex_t*>(__lx); 1530b57cec5SDimitry Andric unsigned count = 0; 1540b57cec5SDimitry Andric while (!__libcpp_mutex_trylock(m)) 1550b57cec5SDimitry Andric { 1560b57cec5SDimitry Andric if (++count > 16) 1570b57cec5SDimitry Andric { 1580b57cec5SDimitry Andric __libcpp_mutex_lock(m); 1590b57cec5SDimitry Andric break; 1600b57cec5SDimitry Andric } 1610b57cec5SDimitry Andric this_thread::yield(); 1620b57cec5SDimitry Andric } 1630b57cec5SDimitry Andric } 1640b57cec5SDimitry Andric 1650b57cec5SDimitry Andric void 166fe6060f1SDimitry Andric __sp_mut::unlock() noexcept 1670b57cec5SDimitry Andric { 1680b57cec5SDimitry Andric __libcpp_mutex_unlock(static_cast<__libcpp_mutex_t*>(__lx)); 1690b57cec5SDimitry Andric } 1700b57cec5SDimitry Andric 1710b57cec5SDimitry Andric __sp_mut& 1720b57cec5SDimitry Andric __get_sp_mut(const void* p) 1730b57cec5SDimitry Andric { 1740b57cec5SDimitry Andric static __sp_mut muts[__sp_mut_count] 1750b57cec5SDimitry Andric { 1760b57cec5SDimitry Andric &mut_back[ 0], &mut_back[ 1], &mut_back[ 2], &mut_back[ 3], 1770b57cec5SDimitry Andric &mut_back[ 4], &mut_back[ 5], &mut_back[ 6], &mut_back[ 7], 1780b57cec5SDimitry Andric &mut_back[ 8], &mut_back[ 9], &mut_back[10], &mut_back[11], 1790b57cec5SDimitry Andric &mut_back[12], &mut_back[13], &mut_back[14], &mut_back[15] 1800b57cec5SDimitry Andric }; 1810b57cec5SDimitry Andric return muts[hash<const void*>()(p) & (__sp_mut_count-1)]; 1820b57cec5SDimitry Andric } 1830b57cec5SDimitry Andric 184*349cc55cSDimitry Andric #endif // !defined(_LIBCPP_HAS_NO_THREADS) 1850b57cec5SDimitry Andric 1860b57cec5SDimitry Andric void* 1870b57cec5SDimitry Andric align(size_t alignment, size_t size, void*& ptr, size_t& space) 1880b57cec5SDimitry Andric { 1890b57cec5SDimitry Andric void* r = nullptr; 1900b57cec5SDimitry Andric if (size <= space) 1910b57cec5SDimitry Andric { 1920b57cec5SDimitry Andric char* p1 = static_cast<char*>(ptr); 1930b57cec5SDimitry Andric char* p2 = reinterpret_cast<char*>(reinterpret_cast<size_t>(p1 + (alignment - 1)) & -alignment); 1940b57cec5SDimitry Andric size_t d = static_cast<size_t>(p2 - p1); 1950b57cec5SDimitry Andric if (d <= space - size) 1960b57cec5SDimitry Andric { 1970b57cec5SDimitry Andric r = p2; 1980b57cec5SDimitry Andric ptr = r; 1990b57cec5SDimitry Andric space -= d; 2000b57cec5SDimitry Andric } 2010b57cec5SDimitry Andric } 2020b57cec5SDimitry Andric return r; 2030b57cec5SDimitry Andric } 2040b57cec5SDimitry Andric 2050b57cec5SDimitry Andric _LIBCPP_END_NAMESPACE_STD 206