Lines Matching full:tsd
41 tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { in tcache_event_hard() argument
55 tcache_bin_flush_small(tsd, tcache, tbin, binind, in tcache_event_hard()
68 tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached in tcache_event_hard()
134 tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, in tcache_bin_flush_small() argument
148 tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, in tcache_bin_flush_small()
152 item_extent[i] = iealloc(tsd_tsdn(tsd), in tcache_bin_flush_small()
160 arena_t *bin_arena = arena_get(tsd_tsdn(tsd), bin_arena_ind, in tcache_bin_flush_small()
167 if (arena_prof_accum(tsd_tsdn(tsd), arena, in tcache_bin_flush_small()
169 prof_idump(tsd_tsdn(tsd)); in tcache_bin_flush_small()
174 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); in tcache_bin_flush_small()
189 arena_dalloc_bin_junked_locked(tsd_tsdn(tsd), in tcache_bin_flush_small()
203 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); in tcache_bin_flush_small()
204 arena_decay_ticks(tsd_tsdn(tsd), bin_arena, nflush - ndeferred); in tcache_bin_flush_small()
213 bin_t *bin = arena_bin_choose_lock(tsd_tsdn(tsd), arena, binind, in tcache_bin_flush_small()
218 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); in tcache_bin_flush_small()
230 tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, in tcache_bin_flush_large() argument
245 item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); in tcache_bin_flush_large()
248 tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush, in tcache_bin_flush_large()
255 arena_t *locked_arena = arena_get(tsd_tsdn(tsd), in tcache_bin_flush_large()
265 malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); in tcache_bin_flush_large()
272 large_dalloc_prep_junked_locked(tsd_tsdn(tsd), in tcache_bin_flush_large()
279 idump = arena_prof_accum(tsd_tsdn(tsd), in tcache_bin_flush_large()
286 tsd_tsdn(tsd), &tcache_arena->stats, binind, in tcache_bin_flush_large()
292 malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); in tcache_bin_flush_large()
302 large_dalloc_finish(tsd_tsdn(tsd), extent); in tcache_bin_flush_large()
316 prof_idump(tsd_tsdn(tsd)); in tcache_bin_flush_large()
318 arena_decay_ticks(tsd_tsdn(tsd), locked_arena, nflush - in tcache_bin_flush_large()
327 arena_stats_large_flush_nrequests_add(tsd_tsdn(tsd), in tcache_bin_flush_large()
395 tsd_tcache_enabled_data_init(tsd_t *tsd) { in tsd_tcache_enabled_data_init() argument
396 /* Called upon tsd initialization. */ in tsd_tcache_enabled_data_init()
397 tsd_tcache_enabled_set(tsd, opt_tcache); in tsd_tcache_enabled_data_init()
398 tsd_slow_update(tsd); in tsd_tcache_enabled_data_init()
402 tsd_tcache_data_init(tsd); in tsd_tcache_enabled_data_init()
408 /* Initialize auto tcache (embedded in TSD). */
410 tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { in tcache_init() argument
442 /* Initialize auto tcache (embedded in TSD). */
444 tsd_tcache_data_init(tsd_t *tsd) { in tsd_tcache_data_init() argument
445 tcache_t *tcache = tsd_tcachep_get_unsafe(tsd); in tsd_tcache_data_init()
451 void *avail_array = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, in tsd_tcache_data_init()
457 tcache_init(tsd, tcache, avail_array); in tsd_tcache_data_init()
462 * functional tsd, and it can only rely on a0. In that case, we in tsd_tcache_data_init()
470 arena = arena_get(tsd_tsdn(tsd), 0, false); in tsd_tcache_data_init()
471 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); in tsd_tcache_data_init()
473 arena = arena_choose(tsd, NULL); in tsd_tcache_data_init()
476 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); in tsd_tcache_data_init()
486 tcache_create_explicit(tsd_t *tsd) { in tcache_create_explicit() argument
498 tcache = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, NULL, true, in tcache_create_explicit()
504 tcache_init(tsd, tcache, in tcache_create_explicit()
506 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena_ichoose(tsd, NULL)); in tcache_create_explicit()
512 tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { in tcache_flush_cache() argument
517 tcache_bin_flush_small(tsd, tcache, tbin, i, 0); in tcache_flush_cache()
525 tcache_bin_flush_large(tsd, tbin, i, 0, tcache); in tcache_flush_cache()
533 arena_prof_accum(tsd_tsdn(tsd), tcache->arena, in tcache_flush_cache()
535 prof_idump(tsd_tsdn(tsd)); in tcache_flush_cache()
540 tcache_flush(tsd_t *tsd) { in tcache_flush() argument
541 assert(tcache_available(tsd)); in tcache_flush()
542 tcache_flush_cache(tsd, tsd_tcachep_get(tsd)); in tcache_flush()
546 tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { in tcache_destroy() argument
547 tcache_flush_cache(tsd, tcache); in tcache_destroy()
549 tcache_arena_dissociate(tsd_tsdn(tsd), tcache); in tcache_destroy()
552 /* Release the avail array for the TSD embedded auto tcache. */ in tcache_destroy()
556 idalloctm(tsd_tsdn(tsd), avail_array, NULL, NULL, true, true); in tcache_destroy()
559 idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true); in tcache_destroy()
565 * tsd). Manually trigger decay to avoid pathological cases. Also in tcache_destroy()
568 arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), in tcache_destroy()
574 arena_decay(tsd_tsdn(tsd), arena, false, true); in tcache_destroy()
576 arena_decay(tsd_tsdn(tsd), arena, false, false); in tcache_destroy()
580 /* For auto tcache (embedded in TSD) only. */
582 tcache_cleanup(tsd_t *tsd) { in tcache_cleanup() argument
583 tcache_t *tcache = tsd_tcachep_get(tsd); in tcache_cleanup()
584 if (!tcache_available(tsd)) { in tcache_cleanup()
585 assert(tsd_tcache_enabled_get(tsd) == false); in tcache_cleanup()
591 assert(tsd_tcache_enabled_get(tsd)); in tcache_cleanup()
594 tcache_destroy(tsd, tcache, true); in tcache_cleanup()
625 tcaches_create_prep(tsd_t *tsd) { in tcaches_create_prep() argument
628 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); in tcaches_create_prep()
631 tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *) in tcaches_create_prep()
646 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); in tcaches_create_prep()
651 tcaches_create(tsd_t *tsd, unsigned *r_ind) { in tcaches_create() argument
652 witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0); in tcaches_create()
656 if (tcaches_create_prep(tsd)) { in tcaches_create()
661 tcache_t *tcache = tcache_create_explicit(tsd); in tcaches_create()
668 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); in tcaches_create()
680 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); in tcaches_create()
684 witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0); in tcaches_create()
689 tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) { in tcaches_elm_remove() argument
690 malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx); in tcaches_elm_remove()
709 tcaches_flush(tsd_t *tsd, unsigned ind) { in tcaches_flush() argument
710 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); in tcaches_flush()
711 tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true); in tcaches_flush()
712 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); in tcaches_flush()
715 tcache_destroy(tsd, tcache, false); in tcaches_flush()
720 tcaches_destroy(tsd_t *tsd, unsigned ind) { in tcaches_destroy() argument
721 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); in tcaches_destroy()
723 tcache_t *tcache = tcaches_elm_remove(tsd, elm, false); in tcaches_destroy()
726 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); in tcaches_destroy()
728 tcache_destroy(tsd, tcache, false); in tcaches_destroy()