1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/assert.h" 5 #include "jemalloc/internal/emap.h" 6 #include "jemalloc/internal/extent_mmap.h" 7 #include "jemalloc/internal/mutex.h" 8 #include "jemalloc/internal/prof_recent.h" 9 #include "jemalloc/internal/util.h" 10 11 /******************************************************************************/ 12 13 void * 14 large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) { 15 assert(usize == sz_s2u(usize)); 16 17 return large_palloc(tsdn, arena, usize, CACHELINE, zero); 18 } 19 20 void * 21 large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, 22 bool zero) { 23 size_t ausize; 24 edata_t *edata; 25 UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false); 26 27 assert(!tsdn_null(tsdn) || arena != NULL); 28 29 ausize = sz_sa2u(usize, alignment); 30 if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) { 31 return NULL; 32 } 33 34 if (likely(!tsdn_null(tsdn))) { 35 arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize); 36 } 37 if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn, 38 arena, usize, alignment, zero)) == NULL) { 39 return NULL; 40 } 41 42 /* See comments in arena_bin_slabs_full_insert(). */ 43 if (!arena_is_auto(arena)) { 44 /* Insert edata into large. */ 45 malloc_mutex_lock(tsdn, &arena->large_mtx); 46 edata_list_active_append(&arena->large, edata); 47 malloc_mutex_unlock(tsdn, &arena->large_mtx); 48 } 49 50 arena_decay_tick(tsdn, arena); 51 return edata_addr_get(edata); 52 } 53 54 static bool 55 large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) { 56 arena_t *arena = arena_get_from_edata(edata); 57 ehooks_t *ehooks = arena_get_ehooks(arena); 58 size_t old_size = edata_size_get(edata); 59 size_t old_usize = edata_usize_get(edata); 60 61 assert(old_usize > usize); 62 63 if (ehooks_split_will_fail(ehooks)) { 64 return true; 65 } 66 67 bool deferred_work_generated = false; 68 bool err = pa_shrink(tsdn, &arena->pa_shard, edata, old_size, 69 usize + sz_large_pad, sz_size2index(usize), 70 &deferred_work_generated); 71 if (err) { 72 return true; 73 } 74 if (deferred_work_generated) { 75 arena_handle_deferred_work(tsdn, arena); 76 } 77 arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize); 78 79 return false; 80 } 81 82 static bool 83 large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize, 84 bool zero) { 85 arena_t *arena = arena_get_from_edata(edata); 86 87 size_t old_size = edata_size_get(edata); 88 size_t old_usize = edata_usize_get(edata); 89 size_t new_size = usize + sz_large_pad; 90 91 szind_t szind = sz_size2index(usize); 92 93 bool deferred_work_generated = false; 94 bool err = pa_expand(tsdn, &arena->pa_shard, edata, old_size, new_size, 95 szind, zero, &deferred_work_generated); 96 97 if (deferred_work_generated) { 98 arena_handle_deferred_work(tsdn, arena); 99 } 100 101 if (err) { 102 return true; 103 } 104 105 if (zero) { 106 if (opt_cache_oblivious) { 107 assert(sz_large_pad == PAGE); 108 /* 109 * Zero the trailing bytes of the original allocation's 110 * last page, since they are in an indeterminate state. 111 * There will always be trailing bytes, because ptr's 112 * offset from the beginning of the extent is a multiple 113 * of CACHELINE in [0 .. PAGE). 114 */ 115 void *zbase = (void *) 116 ((uintptr_t)edata_addr_get(edata) + old_usize); 117 void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase + 118 PAGE)); 119 size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase; 120 assert(nzero > 0); 121 memset(zbase, 0, nzero); 122 } 123 } 124 arena_extent_ralloc_large_expand(tsdn, arena, edata, old_usize); 125 126 return false; 127 } 128 129 bool 130 large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, 131 size_t usize_max, bool zero) { 132 size_t oldusize = edata_usize_get(edata); 133 134 /* The following should have been caught by callers. */ 135 assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS); 136 /* Both allocation sizes must be large to avoid a move. */ 137 assert(oldusize >= SC_LARGE_MINCLASS 138 && usize_max >= SC_LARGE_MINCLASS); 139 140 if (usize_max > oldusize) { 141 /* Attempt to expand the allocation in-place. */ 142 if (!large_ralloc_no_move_expand(tsdn, edata, usize_max, 143 zero)) { 144 arena_decay_tick(tsdn, arena_get_from_edata(edata)); 145 return false; 146 } 147 /* Try again, this time with usize_min. */ 148 if (usize_min < usize_max && usize_min > oldusize && 149 large_ralloc_no_move_expand(tsdn, edata, usize_min, zero)) { 150 arena_decay_tick(tsdn, arena_get_from_edata(edata)); 151 return false; 152 } 153 } 154 155 /* 156 * Avoid moving the allocation if the existing extent size accommodates 157 * the new size. 158 */ 159 if (oldusize >= usize_min && oldusize <= usize_max) { 160 arena_decay_tick(tsdn, arena_get_from_edata(edata)); 161 return false; 162 } 163 164 /* Attempt to shrink the allocation in-place. */ 165 if (oldusize > usize_max) { 166 if (!large_ralloc_no_move_shrink(tsdn, edata, usize_max)) { 167 arena_decay_tick(tsdn, arena_get_from_edata(edata)); 168 return false; 169 } 170 } 171 return true; 172 } 173 174 static void * 175 large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, 176 size_t alignment, bool zero) { 177 if (alignment <= CACHELINE) { 178 return large_malloc(tsdn, arena, usize, zero); 179 } 180 return large_palloc(tsdn, arena, usize, alignment, zero); 181 } 182 183 void * 184 large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, 185 size_t alignment, bool zero, tcache_t *tcache, 186 hook_ralloc_args_t *hook_args) { 187 edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); 188 189 size_t oldusize = edata_usize_get(edata); 190 /* The following should have been caught by callers. */ 191 assert(usize > 0 && usize <= SC_LARGE_MAXCLASS); 192 /* Both allocation sizes must be large to avoid a move. */ 193 assert(oldusize >= SC_LARGE_MINCLASS 194 && usize >= SC_LARGE_MINCLASS); 195 196 /* Try to avoid moving the allocation. */ 197 if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) { 198 hook_invoke_expand(hook_args->is_realloc 199 ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize, 200 usize, (uintptr_t)ptr, hook_args->args); 201 return edata_addr_get(edata); 202 } 203 204 /* 205 * usize and old size are different enough that we need to use a 206 * different size class. In that case, fall back to allocating new 207 * space and copying. 208 */ 209 void *ret = large_ralloc_move_helper(tsdn, arena, usize, alignment, 210 zero); 211 if (ret == NULL) { 212 return NULL; 213 } 214 215 hook_invoke_alloc(hook_args->is_realloc 216 ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, 217 hook_args->args); 218 hook_invoke_dalloc(hook_args->is_realloc 219 ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); 220 221 size_t copysize = (usize < oldusize) ? usize : oldusize; 222 memcpy(ret, edata_addr_get(edata), copysize); 223 isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true); 224 return ret; 225 } 226 227 /* 228 * locked indicates whether the arena's large_mtx is currently held. 229 */ 230 static void 231 large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata, 232 bool locked) { 233 if (!locked) { 234 /* See comments in arena_bin_slabs_full_insert(). */ 235 if (!arena_is_auto(arena)) { 236 malloc_mutex_lock(tsdn, &arena->large_mtx); 237 edata_list_active_remove(&arena->large, edata); 238 malloc_mutex_unlock(tsdn, &arena->large_mtx); 239 } 240 } else { 241 /* Only hold the large_mtx if necessary. */ 242 if (!arena_is_auto(arena)) { 243 malloc_mutex_assert_owner(tsdn, &arena->large_mtx); 244 edata_list_active_remove(&arena->large, edata); 245 } 246 } 247 arena_extent_dalloc_large_prep(tsdn, arena, edata); 248 } 249 250 static void 251 large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) { 252 bool deferred_work_generated = false; 253 pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated); 254 if (deferred_work_generated) { 255 arena_handle_deferred_work(tsdn, arena); 256 } 257 } 258 259 void 260 large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata) { 261 large_dalloc_prep_impl(tsdn, arena_get_from_edata(edata), edata, true); 262 } 263 264 void 265 large_dalloc_finish(tsdn_t *tsdn, edata_t *edata) { 266 large_dalloc_finish_impl(tsdn, arena_get_from_edata(edata), edata); 267 } 268 269 void 270 large_dalloc(tsdn_t *tsdn, edata_t *edata) { 271 arena_t *arena = arena_get_from_edata(edata); 272 large_dalloc_prep_impl(tsdn, arena, edata, false); 273 large_dalloc_finish_impl(tsdn, arena, edata); 274 arena_decay_tick(tsdn, arena); 275 } 276 277 size_t 278 large_salloc(tsdn_t *tsdn, const edata_t *edata) { 279 return edata_usize_get(edata); 280 } 281 282 void 283 large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, 284 bool reset_recent) { 285 assert(prof_info != NULL); 286 287 prof_tctx_t *alloc_tctx = edata_prof_tctx_get(edata); 288 prof_info->alloc_tctx = alloc_tctx; 289 290 if ((uintptr_t)alloc_tctx > (uintptr_t)1U) { 291 nstime_copy(&prof_info->alloc_time, 292 edata_prof_alloc_time_get(edata)); 293 prof_info->alloc_size = edata_prof_alloc_size_get(edata); 294 if (reset_recent) { 295 /* 296 * Reset the pointer on the recent allocation record, 297 * so that this allocation is recorded as released. 298 */ 299 prof_recent_alloc_reset(tsd, edata); 300 } 301 } 302 } 303 304 static void 305 large_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) { 306 edata_prof_tctx_set(edata, tctx); 307 } 308 309 void 310 large_prof_tctx_reset(edata_t *edata) { 311 large_prof_tctx_set(edata, (prof_tctx_t *)(uintptr_t)1U); 312 } 313 314 void 315 large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size) { 316 nstime_t t; 317 nstime_prof_init_update(&t); 318 edata_prof_alloc_time_set(edata, &t); 319 edata_prof_alloc_size_set(edata, size); 320 edata_prof_recent_alloc_init(edata); 321 large_prof_tctx_set(edata, tctx); 322 } 323