1 //===-- tsd_exclusive.h -----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef SCUDO_TSD_EXCLUSIVE_H_ 10 #define SCUDO_TSD_EXCLUSIVE_H_ 11 12 #include "tsd.h" 13 14 namespace scudo { 15 16 struct ThreadState { 17 bool DisableMemInit : 1; 18 enum : unsigned { 19 NotInitialized = 0, 20 Initialized, 21 TornDown, 22 } InitState : 2; 23 }; 24 25 template <class Allocator> void teardownThread(void *Ptr); 26 27 template <class Allocator> struct TSDRegistryExT { 28 void init(Allocator *Instance) { 29 DCHECK(!Initialized); 30 Instance->init(); 31 CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0); 32 FallbackTSD.init(Instance); 33 Initialized = true; 34 } 35 36 void initOnceMaybe(Allocator *Instance) { 37 ScopedLock L(Mutex); 38 if (LIKELY(Initialized)) 39 return; 40 init(Instance); // Sets Initialized. 41 } 42 43 void unmapTestOnly(Allocator *Instance) { 44 DCHECK(Instance); 45 if (reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey))) { 46 DCHECK_EQ(reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey)), 47 Instance); 48 ThreadTSD.commitBack(Instance); 49 ThreadTSD = {}; 50 } 51 CHECK_EQ(pthread_key_delete(PThreadKey), 0); 52 PThreadKey = {}; 53 FallbackTSD.commitBack(Instance); 54 FallbackTSD = {}; 55 State = {}; 56 Initialized = false; 57 } 58 59 ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) { 60 if (LIKELY(State.InitState != ThreadState::NotInitialized)) 61 return; 62 initThread(Instance, MinimalInit); 63 } 64 65 ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) { 66 if (LIKELY(State.InitState == ThreadState::Initialized && 67 !atomic_load(&Disabled, memory_order_acquire))) { 68 *UnlockRequired = false; 69 return &ThreadTSD; 70 } 71 FallbackTSD.lock(); 72 *UnlockRequired = true; 73 return &FallbackTSD; 74 } 75 76 // To disable the exclusive TSD registry, we effectively lock the fallback TSD 77 // and force all threads to attempt to use it instead of their local one. 78 void disable() { 79 Mutex.lock(); 80 FallbackTSD.lock(); 81 atomic_store(&Disabled, 1U, memory_order_release); 82 } 83 84 void enable() { 85 atomic_store(&Disabled, 0U, memory_order_release); 86 FallbackTSD.unlock(); 87 Mutex.unlock(); 88 } 89 90 bool setOption(Option O, sptr Value) { 91 if (O == Option::ThreadDisableMemInit) 92 State.DisableMemInit = Value; 93 if (O == Option::MaxTSDsCount) 94 return false; 95 return true; 96 } 97 98 bool getDisableMemInit() { return State.DisableMemInit; } 99 100 private: 101 // Using minimal initialization allows for global initialization while keeping 102 // the thread specific structure untouched. The fallback structure will be 103 // used instead. 104 NOINLINE void initThread(Allocator *Instance, bool MinimalInit) { 105 initOnceMaybe(Instance); 106 if (UNLIKELY(MinimalInit)) 107 return; 108 CHECK_EQ( 109 pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0); 110 ThreadTSD.init(Instance); 111 State.InitState = ThreadState::Initialized; 112 Instance->callPostInitCallback(); 113 } 114 115 pthread_key_t PThreadKey = {}; 116 bool Initialized = false; 117 atomic_u8 Disabled = {}; 118 TSD<Allocator> FallbackTSD; 119 HybridMutex Mutex; 120 static thread_local ThreadState State; 121 static thread_local TSD<Allocator> ThreadTSD; 122 123 friend void teardownThread<Allocator>(void *Ptr); 124 }; 125 126 template <class Allocator> 127 thread_local TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD; 128 template <class Allocator> 129 thread_local ThreadState TSDRegistryExT<Allocator>::State; 130 131 template <class Allocator> void teardownThread(void *Ptr) { 132 typedef TSDRegistryExT<Allocator> TSDRegistryT; 133 Allocator *Instance = reinterpret_cast<Allocator *>(Ptr); 134 // The glibc POSIX thread-local-storage deallocation routine calls user 135 // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS. 136 // We want to be called last since other destructors might call free and the 137 // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the 138 // quarantine and swallowing the cache. 139 if (TSDRegistryT::ThreadTSD.DestructorIterations > 1) { 140 TSDRegistryT::ThreadTSD.DestructorIterations--; 141 // If pthread_setspecific fails, we will go ahead with the teardown. 142 if (LIKELY(pthread_setspecific(Instance->getTSDRegistry()->PThreadKey, 143 Ptr) == 0)) 144 return; 145 } 146 TSDRegistryT::ThreadTSD.commitBack(Instance); 147 TSDRegistryT::State.InitState = ThreadState::TornDown; 148 } 149 150 } // namespace scudo 151 152 #endif // SCUDO_TSD_EXCLUSIVE_H_ 153