1 //===-- tsd_exclusive.h -----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef SCUDO_TSD_EXCLUSIVE_H_ 10 #define SCUDO_TSD_EXCLUSIVE_H_ 11 12 #include "tsd.h" 13 14 namespace scudo { 15 16 struct ThreadState { 17 bool DisableMemInit : 1; 18 enum { 19 NotInitialized = 0, 20 Initialized, 21 TornDown, 22 } InitState : 2; 23 }; 24 25 template <class Allocator> void teardownThread(void *Ptr); 26 27 template <class Allocator> struct TSDRegistryExT { 28 void initLinkerInitialized(Allocator *Instance) { 29 Instance->initLinkerInitialized(); 30 CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0); 31 FallbackTSD.initLinkerInitialized(Instance); 32 Initialized = true; 33 } 34 void init(Allocator *Instance) { 35 memset(this, 0, sizeof(*this)); 36 initLinkerInitialized(Instance); 37 } 38 39 void unmapTestOnly() {} 40 41 ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) { 42 if (LIKELY(State.InitState != ThreadState::NotInitialized)) 43 return; 44 initThread(Instance, MinimalInit); 45 } 46 47 ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) { 48 if (LIKELY(State.InitState == ThreadState::Initialized && 49 !atomic_load(&Disabled, memory_order_acquire))) { 50 *UnlockRequired = false; 51 return &ThreadTSD; 52 } 53 FallbackTSD.lock(); 54 *UnlockRequired = true; 55 return &FallbackTSD; 56 } 57 58 // To disable the exclusive TSD registry, we effectively lock the fallback TSD 59 // and force all threads to attempt to use it instead of their local one. 60 void disable() { 61 Mutex.lock(); 62 FallbackTSD.lock(); 63 atomic_store(&Disabled, 1U, memory_order_release); 64 } 65 66 void enable() { 67 atomic_store(&Disabled, 0U, memory_order_release); 68 FallbackTSD.unlock(); 69 Mutex.unlock(); 70 } 71 72 bool setOption(Option O, UNUSED sptr Value) { 73 if (O == Option::ThreadDisableMemInit) 74 State.DisableMemInit = Value; 75 if (O == Option::MaxTSDsCount) 76 return false; 77 return true; 78 } 79 80 bool getDisableMemInit() { return State.DisableMemInit; } 81 82 private: 83 void initOnceMaybe(Allocator *Instance) { 84 ScopedLock L(Mutex); 85 if (LIKELY(Initialized)) 86 return; 87 initLinkerInitialized(Instance); // Sets Initialized. 88 } 89 90 // Using minimal initialization allows for global initialization while keeping 91 // the thread specific structure untouched. The fallback structure will be 92 // used instead. 93 NOINLINE void initThread(Allocator *Instance, bool MinimalInit) { 94 initOnceMaybe(Instance); 95 if (UNLIKELY(MinimalInit)) 96 return; 97 CHECK_EQ( 98 pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0); 99 ThreadTSD.initLinkerInitialized(Instance); 100 State.InitState = ThreadState::Initialized; 101 Instance->callPostInitCallback(); 102 } 103 104 pthread_key_t PThreadKey; 105 bool Initialized; 106 atomic_u8 Disabled; 107 TSD<Allocator> FallbackTSD; 108 HybridMutex Mutex; 109 static thread_local ThreadState State; 110 static thread_local TSD<Allocator> ThreadTSD; 111 112 friend void teardownThread<Allocator>(void *Ptr); 113 }; 114 115 template <class Allocator> 116 thread_local TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD; 117 template <class Allocator> 118 thread_local ThreadState TSDRegistryExT<Allocator>::State; 119 120 template <class Allocator> void teardownThread(void *Ptr) { 121 typedef TSDRegistryExT<Allocator> TSDRegistryT; 122 Allocator *Instance = reinterpret_cast<Allocator *>(Ptr); 123 // The glibc POSIX thread-local-storage deallocation routine calls user 124 // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS. 125 // We want to be called last since other destructors might call free and the 126 // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the 127 // quarantine and swallowing the cache. 128 if (TSDRegistryT::ThreadTSD.DestructorIterations > 1) { 129 TSDRegistryT::ThreadTSD.DestructorIterations--; 130 // If pthread_setspecific fails, we will go ahead with the teardown. 131 if (LIKELY(pthread_setspecific(Instance->getTSDRegistry()->PThreadKey, 132 Ptr) == 0)) 133 return; 134 } 135 TSDRegistryT::ThreadTSD.commitBack(Instance); 136 TSDRegistryT::State.InitState = ThreadState::TornDown; 137 } 138 139 } // namespace scudo 140 141 #endif // SCUDO_TSD_EXCLUSIVE_H_ 142