1 #include "jemalloc/internal/jemalloc_preamble.h" 2 3 #include "jemalloc/internal/hook.h" 4 5 #include "jemalloc/internal/atomic.h" 6 #include "jemalloc/internal/mutex.h" 7 #include "jemalloc/internal/seq.h" 8 9 typedef struct hooks_internal_s hooks_internal_t; 10 struct hooks_internal_s { 11 hooks_t hooks; 12 bool in_use; 13 }; 14 15 seq_define(hooks_internal_t, hooks) 16 17 static atomic_u_t nhooks = ATOMIC_INIT(0); 18 static seq_hooks_t hooks[HOOK_MAX]; 19 static malloc_mutex_t hooks_mu; 20 21 bool hook_boot()22 hook_boot() { 23 return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK, 24 malloc_mutex_rank_exclusive); 25 } 26 27 static void * hook_install_locked(hooks_t * to_install)28 hook_install_locked(hooks_t *to_install) { 29 hooks_internal_t hooks_internal; 30 for (int i = 0; i < HOOK_MAX; i++) { 31 bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]); 32 /* We hold mu; no concurrent access. */ 33 assert(success); 34 if (!hooks_internal.in_use) { 35 hooks_internal.hooks = *to_install; 36 hooks_internal.in_use = true; 37 seq_store_hooks(&hooks[i], &hooks_internal); 38 atomic_store_u(&nhooks, 39 atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1, 40 ATOMIC_RELAXED); 41 return &hooks[i]; 42 } 43 } 44 return NULL; 45 } 46 47 void * hook_install(tsdn_t * tsdn,hooks_t * to_install)48 hook_install(tsdn_t *tsdn, hooks_t *to_install) { 49 malloc_mutex_lock(tsdn, &hooks_mu); 50 void *ret = hook_install_locked(to_install); 51 if (ret != NULL) { 52 tsd_global_slow_inc(tsdn); 53 } 54 malloc_mutex_unlock(tsdn, &hooks_mu); 55 return ret; 56 } 57 58 static void hook_remove_locked(seq_hooks_t * to_remove)59 hook_remove_locked(seq_hooks_t *to_remove) { 60 hooks_internal_t hooks_internal; 61 bool success = seq_try_load_hooks(&hooks_internal, to_remove); 62 /* We hold mu; no concurrent access. */ 63 assert(success); 64 /* Should only remove hooks that were added. */ 65 assert(hooks_internal.in_use); 66 hooks_internal.in_use = false; 67 seq_store_hooks(to_remove, &hooks_internal); 68 atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1, 69 ATOMIC_RELAXED); 70 } 71 72 void hook_remove(tsdn_t * tsdn,void * opaque)73 hook_remove(tsdn_t *tsdn, void *opaque) { 74 if (config_debug) { 75 char *hooks_begin = (char *)&hooks[0]; 76 char *hooks_end = (char *)&hooks[HOOK_MAX]; 77 char *hook = (char *)opaque; 78 assert(hooks_begin <= hook && hook < hooks_end 79 && (hook - hooks_begin) % sizeof(seq_hooks_t) == 0); 80 } 81 malloc_mutex_lock(tsdn, &hooks_mu); 82 hook_remove_locked((seq_hooks_t *)opaque); 83 tsd_global_slow_dec(tsdn); 84 malloc_mutex_unlock(tsdn, &hooks_mu); 85 } 86 87 #define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \ 88 for (int for_each_hook_counter = 0; \ 89 for_each_hook_counter < HOOK_MAX; \ 90 for_each_hook_counter++) { \ 91 bool for_each_hook_success = seq_try_load_hooks( \ 92 (hooks_internal_ptr), &hooks[for_each_hook_counter]); \ 93 if (!for_each_hook_success) { \ 94 continue; \ 95 } \ 96 if (!(hooks_internal_ptr)->in_use) { \ 97 continue; \ 98 } 99 #define FOR_EACH_HOOK_END \ 100 } 101 102 static bool * hook_reentrantp()103 hook_reentrantp() { 104 /* 105 * We prevent user reentrancy within hooks. This is basically just a 106 * thread-local bool that triggers an early-exit. 107 * 108 * We don't fold in_hook into reentrancy. There are two reasons for 109 * this: 110 * - Right now, we turn on reentrancy during things like extent hook 111 * execution. Allocating during extent hooks is not officially 112 * supported, but we don't want to break it for the time being. These 113 * sorts of allocations should probably still be hooked, though. 114 * - If a hook allocates, we may want it to be relatively fast (after 115 * all, it executes on every allocator operation). Turning on 116 * reentrancy is a fairly heavyweight mode (disabling tcache, 117 * redirecting to arena 0, etc.). It's possible we may one day want 118 * to turn on reentrant mode here, if it proves too difficult to keep 119 * this working. But that's fairly easy for us to see; OTOH, people 120 * not using hooks because they're too slow is easy for us to miss. 121 * 122 * The tricky part is 123 * that this code might get invoked even if we don't have access to tsd. 124 * This function mimics getting a pointer to thread-local data, except 125 * that it might secretly return a pointer to some global data if we 126 * know that the caller will take the early-exit path. 127 * If we return a bool that indicates that we are reentrant, then the 128 * caller will go down the early exit path, leaving the global 129 * untouched. 130 */ 131 static bool in_hook_global = true; 132 tsdn_t *tsdn = tsdn_fetch(); 133 tcache_t *tcache = tsdn_tcachep_get(tsdn); 134 if (tcache != NULL) { 135 return &tcache->in_hook; 136 } 137 return &in_hook_global; 138 } 139 140 #define HOOK_PROLOGUE \ 141 if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) { \ 142 return; \ 143 } \ 144 bool *in_hook = hook_reentrantp(); \ 145 if (*in_hook) { \ 146 return; \ 147 } \ 148 *in_hook = true; 149 150 #define HOOK_EPILOGUE \ 151 *in_hook = false; 152 153 void hook_invoke_alloc(hook_alloc_t type,void * result,uintptr_t result_raw,uintptr_t args_raw[3])154 hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, 155 uintptr_t args_raw[3]) { 156 HOOK_PROLOGUE 157 158 hooks_internal_t hook; 159 FOR_EACH_HOOK_BEGIN(&hook) 160 hook_alloc h = hook.hooks.alloc_hook; 161 if (h != NULL) { 162 h(hook.hooks.extra, type, result, result_raw, args_raw); 163 } 164 FOR_EACH_HOOK_END 165 166 HOOK_EPILOGUE 167 } 168 169 void hook_invoke_dalloc(hook_dalloc_t type,void * address,uintptr_t args_raw[3])170 hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) { 171 HOOK_PROLOGUE 172 hooks_internal_t hook; 173 FOR_EACH_HOOK_BEGIN(&hook) 174 hook_dalloc h = hook.hooks.dalloc_hook; 175 if (h != NULL) { 176 h(hook.hooks.extra, type, address, args_raw); 177 } 178 FOR_EACH_HOOK_END 179 HOOK_EPILOGUE 180 } 181 182 void hook_invoke_expand(hook_expand_t type,void * address,size_t old_usize,size_t new_usize,uintptr_t result_raw,uintptr_t args_raw[4])183 hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, 184 size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) { 185 HOOK_PROLOGUE 186 hooks_internal_t hook; 187 FOR_EACH_HOOK_BEGIN(&hook) 188 hook_expand h = hook.hooks.expand_hook; 189 if (h != NULL) { 190 h(hook.hooks.extra, type, address, old_usize, new_usize, 191 result_raw, args_raw); 192 } 193 FOR_EACH_HOOK_END 194 HOOK_EPILOGUE 195 } 196