1 #define JEMALLOC_MUTEX_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) 5 #include <dlfcn.h> 6 #endif 7 8 #ifndef _CRT_SPINCOUNT 9 #define _CRT_SPINCOUNT 4000 10 #endif 11 12 /******************************************************************************/ 13 /* Data. */ 14 15 #ifdef JEMALLOC_LAZY_LOCK 16 bool isthreaded = false; 17 #endif 18 #ifdef JEMALLOC_MUTEX_INIT_CB 19 static bool postpone_init = true; 20 static malloc_mutex_t *postponed_mutexes = NULL; 21 #endif 22 23 #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) 24 static void pthread_create_once(void); 25 #endif 26 27 /******************************************************************************/ 28 /* 29 * We intercept pthread_create() calls in order to toggle isthreaded if the 30 * process goes multi-threaded. 31 */ 32 33 #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) 34 static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, 35 void *(*)(void *), void *__restrict); 36 37 static void 38 pthread_create_once(void) 39 { 40 41 pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); 42 if (pthread_create_fptr == NULL) { 43 malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, " 44 "\"pthread_create\")\n"); 45 abort(); 46 } 47 48 isthreaded = true; 49 } 50 51 JEMALLOC_EXPORT int 52 pthread_create(pthread_t *__restrict thread, 53 const pthread_attr_t *__restrict attr, void *(*start_routine)(void *), 54 void *__restrict arg) 55 { 56 static pthread_once_t once_control = PTHREAD_ONCE_INIT; 57 58 pthread_once(&once_control, pthread_create_once); 59 60 return (pthread_create_fptr(thread, attr, start_routine, arg)); 61 } 62 #endif 63 64 /******************************************************************************/ 65 66 #ifdef JEMALLOC_MUTEX_INIT_CB 67 JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 68 void *(calloc_cb)(size_t, size_t)); 69 70 #pragma weak _pthread_mutex_init_calloc_cb 71 int 72 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 73 void *(calloc_cb)(size_t, size_t)) 74 { 75 76 return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t))) 77 __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex, 78 calloc_cb)); 79 } 80 #endif 81 82 bool 83 malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank) 84 { 85 86 #ifdef _WIN32 87 # if _WIN32_WINNT >= 0x0600 88 InitializeSRWLock(&mutex->lock); 89 # else 90 if (!InitializeCriticalSectionAndSpinCount(&mutex->lock, 91 _CRT_SPINCOUNT)) 92 return (true); 93 # endif 94 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) 95 mutex->lock = OS_UNFAIR_LOCK_INIT; 96 #elif (defined(JEMALLOC_OSSPIN)) 97 mutex->lock = 0; 98 #elif (defined(JEMALLOC_MUTEX_INIT_CB)) 99 if (postpone_init) { 100 mutex->postponed_next = postponed_mutexes; 101 postponed_mutexes = mutex; 102 } else { 103 if (_pthread_mutex_init_calloc_cb(&mutex->lock, 104 bootstrap_calloc) != 0) 105 return (true); 106 } 107 #else 108 pthread_mutexattr_t attr; 109 110 if (pthread_mutexattr_init(&attr) != 0) 111 return (true); 112 pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE); 113 if (pthread_mutex_init(&mutex->lock, &attr) != 0) { 114 pthread_mutexattr_destroy(&attr); 115 return (true); 116 } 117 pthread_mutexattr_destroy(&attr); 118 #endif 119 if (config_debug) 120 witness_init(&mutex->witness, name, rank, NULL); 121 return (false); 122 } 123 124 void 125 malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex) 126 { 127 128 malloc_mutex_lock(tsdn, mutex); 129 } 130 131 void 132 malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex) 133 { 134 135 malloc_mutex_unlock(tsdn, mutex); 136 } 137 138 void 139 malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex) 140 { 141 142 #ifdef JEMALLOC_MUTEX_INIT_CB 143 malloc_mutex_unlock(tsdn, mutex); 144 #else 145 if (malloc_mutex_init(mutex, mutex->witness.name, 146 mutex->witness.rank)) { 147 malloc_printf("<jemalloc>: Error re-initializing mutex in " 148 "child\n"); 149 if (opt_abort) 150 abort(); 151 } 152 #endif 153 } 154 155 bool 156 malloc_mutex_first_thread(void) 157 { 158 159 #ifdef JEMALLOC_MUTEX_INIT_CB 160 postpone_init = false; 161 while (postponed_mutexes != NULL) { 162 if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock, 163 bootstrap_calloc) != 0) 164 return (true); 165 postponed_mutexes = postponed_mutexes->postponed_next; 166 } 167 #endif 168 return (false); 169 } 170 171 bool 172 malloc_mutex_boot(void) 173 { 174 175 #ifndef JEMALLOC_MUTEX_INIT_CB 176 return (malloc_mutex_first_thread()); 177 #else 178 return (false); 179 #endif 180 } 181