xref: /freebsd/contrib/llvm-project/libcxx/src/memory.cpp (revision 349cc55c9796c4596a5b9904cd3281af295f878f)
1*349cc55cSDimitry Andric //===----------------------------------------------------------------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric 
90b57cec5SDimitry Andric #include "memory"
100b57cec5SDimitry Andric #ifndef _LIBCPP_HAS_NO_THREADS
110b57cec5SDimitry Andric #   include "mutex"
120b57cec5SDimitry Andric #   include "thread"
13480093f4SDimitry Andric #   if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB)
140b57cec5SDimitry Andric #       pragma comment(lib, "pthread")
150b57cec5SDimitry Andric #   endif
160b57cec5SDimitry Andric #endif
170b57cec5SDimitry Andric #include "include/atomic_support.h"
180b57cec5SDimitry Andric 
190b57cec5SDimitry Andric _LIBCPP_BEGIN_NAMESPACE_STD
200b57cec5SDimitry Andric 
210b57cec5SDimitry Andric const allocator_arg_t allocator_arg = allocator_arg_t();
220b57cec5SDimitry Andric 
23fe6060f1SDimitry Andric bad_weak_ptr::~bad_weak_ptr() noexcept {}
240b57cec5SDimitry Andric 
250b57cec5SDimitry Andric const char*
26fe6060f1SDimitry Andric bad_weak_ptr::what() const noexcept
270b57cec5SDimitry Andric {
280b57cec5SDimitry Andric     return "bad_weak_ptr";
290b57cec5SDimitry Andric }
300b57cec5SDimitry Andric 
310b57cec5SDimitry Andric __shared_count::~__shared_count()
320b57cec5SDimitry Andric {
330b57cec5SDimitry Andric }
340b57cec5SDimitry Andric 
350b57cec5SDimitry Andric __shared_weak_count::~__shared_weak_count()
360b57cec5SDimitry Andric {
370b57cec5SDimitry Andric }
380b57cec5SDimitry Andric 
390b57cec5SDimitry Andric #if defined(_LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS)
400b57cec5SDimitry Andric void
41fe6060f1SDimitry Andric __shared_count::__add_shared() noexcept
420b57cec5SDimitry Andric {
430b57cec5SDimitry Andric     __libcpp_atomic_refcount_increment(__shared_owners_);
440b57cec5SDimitry Andric }
450b57cec5SDimitry Andric 
460b57cec5SDimitry Andric bool
47fe6060f1SDimitry Andric __shared_count::__release_shared() noexcept
480b57cec5SDimitry Andric {
490b57cec5SDimitry Andric     if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1)
500b57cec5SDimitry Andric     {
510b57cec5SDimitry Andric         __on_zero_shared();
520b57cec5SDimitry Andric         return true;
530b57cec5SDimitry Andric     }
540b57cec5SDimitry Andric     return false;
550b57cec5SDimitry Andric }
560b57cec5SDimitry Andric 
570b57cec5SDimitry Andric void
58fe6060f1SDimitry Andric __shared_weak_count::__add_shared() noexcept
590b57cec5SDimitry Andric {
600b57cec5SDimitry Andric     __shared_count::__add_shared();
610b57cec5SDimitry Andric }
620b57cec5SDimitry Andric 
630b57cec5SDimitry Andric void
64fe6060f1SDimitry Andric __shared_weak_count::__add_weak() noexcept
650b57cec5SDimitry Andric {
660b57cec5SDimitry Andric     __libcpp_atomic_refcount_increment(__shared_weak_owners_);
670b57cec5SDimitry Andric }
680b57cec5SDimitry Andric 
690b57cec5SDimitry Andric void
70fe6060f1SDimitry Andric __shared_weak_count::__release_shared() noexcept
710b57cec5SDimitry Andric {
720b57cec5SDimitry Andric     if (__shared_count::__release_shared())
730b57cec5SDimitry Andric         __release_weak();
740b57cec5SDimitry Andric }
750b57cec5SDimitry Andric 
760b57cec5SDimitry Andric #endif // _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS
770b57cec5SDimitry Andric 
780b57cec5SDimitry Andric void
79fe6060f1SDimitry Andric __shared_weak_count::__release_weak() noexcept
800b57cec5SDimitry Andric {
810b57cec5SDimitry Andric     // NOTE: The acquire load here is an optimization of the very
820b57cec5SDimitry Andric     // common case where a shared pointer is being destructed while
830b57cec5SDimitry Andric     // having no other contended references.
840b57cec5SDimitry Andric     //
850b57cec5SDimitry Andric     // BENEFIT: We avoid expensive atomic stores like XADD and STREX
860b57cec5SDimitry Andric     // in a common case.  Those instructions are slow and do nasty
870b57cec5SDimitry Andric     // things to caches.
880b57cec5SDimitry Andric     //
890b57cec5SDimitry Andric     // IS THIS SAFE?  Yes.  During weak destruction, if we see that we
900b57cec5SDimitry Andric     // are the last reference, we know that no-one else is accessing
910b57cec5SDimitry Andric     // us. If someone were accessing us, then they would be doing so
920b57cec5SDimitry Andric     // while the last shared / weak_ptr was being destructed, and
930b57cec5SDimitry Andric     // that's undefined anyway.
940b57cec5SDimitry Andric     //
950b57cec5SDimitry Andric     // If we see anything other than a 0, then we have possible
960b57cec5SDimitry Andric     // contention, and need to use an atomicrmw primitive.
970b57cec5SDimitry Andric     // The same arguments don't apply for increment, where it is legal
980b57cec5SDimitry Andric     // (though inadvisable) to share shared_ptr references between
990b57cec5SDimitry Andric     // threads, and have them all get copied at once.  The argument
1000b57cec5SDimitry Andric     // also doesn't apply for __release_shared, because an outstanding
1010b57cec5SDimitry Andric     // weak_ptr::lock() could read / modify the shared count.
1020b57cec5SDimitry Andric     if (__libcpp_atomic_load(&__shared_weak_owners_, _AO_Acquire) == 0)
1030b57cec5SDimitry Andric     {
1040b57cec5SDimitry Andric         // no need to do this store, because we are about
1050b57cec5SDimitry Andric         // to destroy everything.
1060b57cec5SDimitry Andric         //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
1070b57cec5SDimitry Andric         __on_zero_shared_weak();
1080b57cec5SDimitry Andric     }
1090b57cec5SDimitry Andric     else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1)
1100b57cec5SDimitry Andric         __on_zero_shared_weak();
1110b57cec5SDimitry Andric }
1120b57cec5SDimitry Andric 
1130b57cec5SDimitry Andric __shared_weak_count*
114fe6060f1SDimitry Andric __shared_weak_count::lock() noexcept
1150b57cec5SDimitry Andric {
1160b57cec5SDimitry Andric     long object_owners = __libcpp_atomic_load(&__shared_owners_);
1170b57cec5SDimitry Andric     while (object_owners != -1)
1180b57cec5SDimitry Andric     {
1190b57cec5SDimitry Andric         if (__libcpp_atomic_compare_exchange(&__shared_owners_,
1200b57cec5SDimitry Andric                                              &object_owners,
1210b57cec5SDimitry Andric                                              object_owners+1))
1220b57cec5SDimitry Andric             return this;
1230b57cec5SDimitry Andric     }
1240b57cec5SDimitry Andric     return nullptr;
1250b57cec5SDimitry Andric }
1260b57cec5SDimitry Andric 
1270b57cec5SDimitry Andric const void*
128fe6060f1SDimitry Andric __shared_weak_count::__get_deleter(const type_info&) const noexcept
1290b57cec5SDimitry Andric {
1300b57cec5SDimitry Andric     return nullptr;
1310b57cec5SDimitry Andric }
1320b57cec5SDimitry Andric 
133*349cc55cSDimitry Andric #if !defined(_LIBCPP_HAS_NO_THREADS)
1340b57cec5SDimitry Andric 
1350b57cec5SDimitry Andric _LIBCPP_SAFE_STATIC static const std::size_t __sp_mut_count = 16;
1360b57cec5SDimitry Andric _LIBCPP_SAFE_STATIC static __libcpp_mutex_t mut_back[__sp_mut_count] =
1370b57cec5SDimitry Andric {
1380b57cec5SDimitry Andric     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
1390b57cec5SDimitry Andric     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
1400b57cec5SDimitry Andric     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
1410b57cec5SDimitry Andric     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER
1420b57cec5SDimitry Andric };
1430b57cec5SDimitry Andric 
144fe6060f1SDimitry Andric _LIBCPP_CONSTEXPR __sp_mut::__sp_mut(void* p) noexcept
1450b57cec5SDimitry Andric    : __lx(p)
1460b57cec5SDimitry Andric {
1470b57cec5SDimitry Andric }
1480b57cec5SDimitry Andric 
1490b57cec5SDimitry Andric void
150fe6060f1SDimitry Andric __sp_mut::lock() noexcept
1510b57cec5SDimitry Andric {
1520b57cec5SDimitry Andric     auto m = static_cast<__libcpp_mutex_t*>(__lx);
1530b57cec5SDimitry Andric     unsigned count = 0;
1540b57cec5SDimitry Andric     while (!__libcpp_mutex_trylock(m))
1550b57cec5SDimitry Andric     {
1560b57cec5SDimitry Andric         if (++count > 16)
1570b57cec5SDimitry Andric         {
1580b57cec5SDimitry Andric             __libcpp_mutex_lock(m);
1590b57cec5SDimitry Andric             break;
1600b57cec5SDimitry Andric         }
1610b57cec5SDimitry Andric         this_thread::yield();
1620b57cec5SDimitry Andric     }
1630b57cec5SDimitry Andric }
1640b57cec5SDimitry Andric 
1650b57cec5SDimitry Andric void
166fe6060f1SDimitry Andric __sp_mut::unlock() noexcept
1670b57cec5SDimitry Andric {
1680b57cec5SDimitry Andric     __libcpp_mutex_unlock(static_cast<__libcpp_mutex_t*>(__lx));
1690b57cec5SDimitry Andric }
1700b57cec5SDimitry Andric 
1710b57cec5SDimitry Andric __sp_mut&
1720b57cec5SDimitry Andric __get_sp_mut(const void* p)
1730b57cec5SDimitry Andric {
1740b57cec5SDimitry Andric     static __sp_mut muts[__sp_mut_count]
1750b57cec5SDimitry Andric     {
1760b57cec5SDimitry Andric         &mut_back[ 0], &mut_back[ 1], &mut_back[ 2], &mut_back[ 3],
1770b57cec5SDimitry Andric         &mut_back[ 4], &mut_back[ 5], &mut_back[ 6], &mut_back[ 7],
1780b57cec5SDimitry Andric         &mut_back[ 8], &mut_back[ 9], &mut_back[10], &mut_back[11],
1790b57cec5SDimitry Andric         &mut_back[12], &mut_back[13], &mut_back[14], &mut_back[15]
1800b57cec5SDimitry Andric     };
1810b57cec5SDimitry Andric     return muts[hash<const void*>()(p) & (__sp_mut_count-1)];
1820b57cec5SDimitry Andric }
1830b57cec5SDimitry Andric 
184*349cc55cSDimitry Andric #endif // !defined(_LIBCPP_HAS_NO_THREADS)
1850b57cec5SDimitry Andric 
1860b57cec5SDimitry Andric void*
1870b57cec5SDimitry Andric align(size_t alignment, size_t size, void*& ptr, size_t& space)
1880b57cec5SDimitry Andric {
1890b57cec5SDimitry Andric     void* r = nullptr;
1900b57cec5SDimitry Andric     if (size <= space)
1910b57cec5SDimitry Andric     {
1920b57cec5SDimitry Andric         char* p1 = static_cast<char*>(ptr);
1930b57cec5SDimitry Andric         char* p2 = reinterpret_cast<char*>(reinterpret_cast<size_t>(p1 + (alignment - 1)) & -alignment);
1940b57cec5SDimitry Andric         size_t d = static_cast<size_t>(p2 - p1);
1950b57cec5SDimitry Andric         if (d <= space - size)
1960b57cec5SDimitry Andric         {
1970b57cec5SDimitry Andric             r = p2;
1980b57cec5SDimitry Andric             ptr = r;
1990b57cec5SDimitry Andric             space -= d;
2000b57cec5SDimitry Andric         }
2010b57cec5SDimitry Andric     }
2020b57cec5SDimitry Andric     return r;
2030b57cec5SDimitry Andric }
2040b57cec5SDimitry Andric 
2050b57cec5SDimitry Andric _LIBCPP_END_NAMESPACE_STD
206