Lines Matching refs:arena

34 static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena,
43 static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
50 static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
56 static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena,
63 static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena,
103 static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena,
107 static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
110 static void extent_record(tsdn_t *tsdn, arena_t *arena,
180 extent_alloc(tsdn_t *tsdn, arena_t *arena) { in extent_alloc() argument
181 malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); in extent_alloc()
182 extent_t *extent = extent_avail_first(&arena->extent_avail); in extent_alloc()
184 malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); in extent_alloc()
185 return base_alloc_extent(tsdn, arena->base); in extent_alloc()
187 extent_avail_remove(&arena->extent_avail, extent); in extent_alloc()
188 atomic_fetch_sub_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); in extent_alloc()
189 malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); in extent_alloc()
194 extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { in extent_dalloc() argument
195 malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); in extent_dalloc()
196 extent_avail_insert(&arena->extent_avail, extent); in extent_dalloc()
197 atomic_fetch_add_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); in extent_dalloc()
198 malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); in extent_dalloc()
202 extent_hooks_get(arena_t *arena) { in extent_hooks_get() argument
203 return base_extent_hooks_get(arena->base); in extent_hooks_get()
207 extent_hooks_set(tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks) { in extent_hooks_set() argument
210 info = arena_background_thread_info_get(arena); in extent_hooks_set()
213 extent_hooks_t *ret = base_extent_hooks_set(arena->base, extent_hooks); in extent_hooks_set()
222 extent_hooks_assure_initialized(arena_t *arena, in extent_hooks_assure_initialized() argument
225 *r_extent_hooks = extent_hooks_get(arena); in extent_hooks_assure_initialized()
443 extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, in extents_first_fit_locked() argument
493 extents_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, in extents_fit_locked() argument
504 extents_first_fit_locked(tsdn, arena, extents, max_size); in extents_fit_locked()
520 extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, in extent_try_delayed_coalesce() argument
525 extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx, in extent_try_delayed_coalesce()
537 extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extents_alloc() argument
545 extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks, extents, in extents_alloc()
552 extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extents_dalloc() argument
563 extent_record(tsdn, arena, r_extent_hooks, extents, extent, false); in extents_dalloc()
567 extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extents_evict() argument
597 if (extent_try_delayed_coalesce(tsdn, arena, r_extent_hooks, in extents_evict()
635 extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extents_abandon_vm() argument
639 arena_stats_accum_zu(&arena->stats.abandoned_vm, sz); in extents_abandon_vm()
646 if (extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, in extents_abandon_vm()
648 extent_purge_forced_impl(tsdn, arena, r_extent_hooks, in extents_abandon_vm()
653 extent_dalloc(tsdn, arena, extent); in extents_abandon_vm()
672 extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, in extent_deactivate_locked() argument
674 assert(extent_arena_get(extent) == arena); in extent_deactivate_locked()
682 extent_deactivate(tsdn_t *tsdn, arena_t *arena, extents_t *extents, in extent_deactivate() argument
685 extent_deactivate_locked(tsdn, arena, extents, extent); in extent_deactivate()
690 extent_activate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, in extent_activate_locked() argument
692 assert(extent_arena_get(extent) == arena); in extent_activate_locked()
892 extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, in extent_recycle_extract() argument
918 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_recycle_extract()
930 if (extent_arena_get(extent) != arena || in extent_recycle_extract()
939 extent = extents_fit_locked(tsdn, arena, extents, esize, in extent_recycle_extract()
947 extent_activate_locked(tsdn, arena, extents, extent); in extent_recycle_extract()
980 extent_split_interior(tsdn_t *tsdn, arena_t *arena, in extent_split_interior() argument
1005 *extent = extent_split_impl(tsdn, arena, r_extent_hooks, in extent_split_interior()
1017 *trail = extent_split_impl(tsdn, arena, r_extent_hooks, *extent, in extent_split_interior()
1057 extent_recycle_split(tsdn_t *tsdn, arena_t *arena, in extent_recycle_split() argument
1067 tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail, in extent_recycle_split()
1078 extent_deactivate(tsdn, arena, extents, to_leak); in extent_recycle_split()
1084 extent_deactivate(tsdn, arena, extents, lead); in extent_recycle_split()
1087 extent_deactivate(tsdn, arena, extents, trail); in extent_recycle_split()
1102 extents_abandon_vm(tsdn, arena, r_extent_hooks, extents, in extent_recycle_split()
1113 extent_need_manual_zero(arena_t *arena) { in extent_need_manual_zero() argument
1119 return (!arena_has_default_hooks(arena) || in extent_need_manual_zero()
1128 extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extent_recycle() argument
1141 extent_t *extent = extent_recycle_extract(tsdn, arena, r_extent_hooks, in extent_recycle()
1148 extent = extent_recycle_split(tsdn, arena, r_extent_hooks, rtree_ctx, in extent_recycle()
1156 if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, in extent_recycle()
1158 extent_record(tsdn, arena, r_extent_hooks, extents, in extent_recycle()
1162 if (!extent_need_manual_zero(arena)) { in extent_recycle()
1187 if (extent_need_manual_zero(arena) || in extent_recycle()
1209 extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, in extent_alloc_core() argument
1218 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, in extent_alloc_core()
1229 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, in extent_alloc_core()
1239 extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr, in extent_alloc_default_impl() argument
1241 void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero, in extent_alloc_default_impl()
1242 commit, (dss_prec_t)atomic_load_u(&arena->dss_prec, in extent_alloc_default_impl()
1254 arena_t *arena; in extent_alloc_default() local
1257 arena = arena_get(tsdn, arena_ind, false); in extent_alloc_default()
1262 assert(arena != NULL); in extent_alloc_default()
1264 return extent_alloc_default_impl(tsdn, arena, new_addr, size, in extent_alloc_default()
1269 extent_hook_pre_reentrancy(tsdn_t *tsdn, arena_t *arena) { in extent_hook_pre_reentrancy() argument
1271 if (arena == arena_get(tsd_tsdn(tsd), 0, false)) { in extent_hook_pre_reentrancy()
1281 pre_reentrancy(tsd, arena); in extent_hook_pre_reentrancy()
1297 extent_grow_retained(tsdn_t *tsdn, arena_t *arena, in extent_grow_retained() argument
1300 malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx); in extent_grow_retained()
1315 size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); in extent_grow_retained()
1318 if (arena->extent_grow_next + egn_skip >= in extent_grow_retained()
1323 alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); in extent_grow_retained()
1326 extent_t *extent = extent_alloc(tsdn, arena); in extent_grow_retained()
1335 ptr = extent_alloc_default_impl(tsdn, arena, NULL, in extent_grow_retained()
1338 extent_hook_pre_reentrancy(tsdn, arena); in extent_grow_retained()
1341 arena_ind_get(arena)); in extent_grow_retained()
1345 extent_init(extent, arena, ptr, alloc_size, false, SC_NSIZES, in extent_grow_retained()
1346 arena_extent_sn_next(arena), extent_state_active, zeroed, in extent_grow_retained()
1349 extent_dalloc(tsdn, arena, extent); in extent_grow_retained()
1354 extent_dalloc(tsdn, arena, extent); in extent_grow_retained()
1373 tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail, in extent_grow_retained()
1379 extent_record(tsdn, arena, r_extent_hooks, in extent_grow_retained()
1380 &arena->extents_retained, lead, true); in extent_grow_retained()
1383 extent_record(tsdn, arena, r_extent_hooks, in extent_grow_retained()
1384 &arena->extents_retained, trail, true); in extent_grow_retained()
1396 extent_record(tsdn, arena, r_extent_hooks, in extent_grow_retained()
1397 &arena->extents_retained, to_salvage, true); in extent_grow_retained()
1401 extents_abandon_vm(tsdn, arena, r_extent_hooks, in extent_grow_retained()
1402 &arena->extents_retained, to_leak, true); in extent_grow_retained()
1408 if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, 0, in extent_grow_retained()
1410 extent_record(tsdn, arena, r_extent_hooks, in extent_grow_retained()
1411 &arena->extents_retained, extent, true); in extent_grow_retained()
1414 if (!extent_need_manual_zero(arena)) { in extent_grow_retained()
1423 if (arena->extent_grow_next + egn_skip + 1 <= in extent_grow_retained()
1424 arena->retain_grow_limit) { in extent_grow_retained()
1425 arena->extent_grow_next += egn_skip + 1; in extent_grow_retained()
1427 arena->extent_grow_next = arena->retain_grow_limit; in extent_grow_retained()
1430 malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); in extent_grow_retained()
1450 if (extent_need_manual_zero(arena) || in extent_grow_retained()
1458 malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); in extent_grow_retained()
1463 extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, in extent_alloc_retained() argument
1469 malloc_mutex_lock(tsdn, &arena->extent_grow_mtx); in extent_alloc_retained()
1471 extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks, in extent_alloc_retained()
1472 &arena->extents_retained, new_addr, size, pad, alignment, slab, in extent_alloc_retained()
1475 malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); in extent_alloc_retained()
1480 extent = extent_grow_retained(tsdn, arena, r_extent_hooks, size, in extent_alloc_retained()
1484 malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); in extent_alloc_retained()
1486 malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx); in extent_alloc_retained()
1492 extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, in extent_alloc_wrapper_hard() argument
1496 extent_t *extent = extent_alloc(tsdn, arena); in extent_alloc_wrapper_hard()
1504 addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize, in extent_alloc_wrapper_hard()
1507 extent_hook_pre_reentrancy(tsdn, arena); in extent_alloc_wrapper_hard()
1509 esize, palignment, zero, commit, arena_ind_get(arena)); in extent_alloc_wrapper_hard()
1513 extent_dalloc(tsdn, arena, extent); in extent_alloc_wrapper_hard()
1516 extent_init(extent, arena, addr, esize, slab, szind, in extent_alloc_wrapper_hard()
1517 arena_extent_sn_next(arena), extent_state_active, *zero, *commit, in extent_alloc_wrapper_hard()
1523 extent_dalloc(tsdn, arena, extent); in extent_alloc_wrapper_hard()
1531 extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_alloc_wrapper() argument
1537 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_alloc_wrapper()
1539 extent_t *extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, in extent_alloc_wrapper()
1551 extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks, in extent_alloc_wrapper()
1560 extent_can_coalesce(arena_t *arena, extents_t *extents, const extent_t *inner, in extent_can_coalesce() argument
1562 assert(extent_arena_get(inner) == arena); in extent_can_coalesce()
1563 if (extent_arena_get(outer) != arena) { in extent_can_coalesce()
1580 extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extent_coalesce() argument
1583 assert(extent_can_coalesce(arena, extents, inner, outer)); in extent_coalesce()
1585 extent_activate_locked(tsdn, arena, extents, outer); in extent_coalesce()
1588 bool err = extent_merge_impl(tsdn, arena, r_extent_hooks, in extent_coalesce()
1593 extent_deactivate_locked(tsdn, arena, extents, outer); in extent_coalesce()
1600 extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, in extent_try_coalesce_impl() argument
1626 bool can_coalesce = extent_can_coalesce(arena, extents, in extent_try_coalesce_impl()
1631 if (can_coalesce && !extent_coalesce(tsdn, arena, in extent_try_coalesce_impl()
1650 bool can_coalesce = extent_can_coalesce(arena, extents, in extent_try_coalesce_impl()
1654 if (can_coalesce && !extent_coalesce(tsdn, arena, in extent_try_coalesce_impl()
1675 extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, in extent_try_coalesce() argument
1678 return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, in extent_try_coalesce()
1683 extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, in extent_try_coalesce_large() argument
1686 return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, in extent_try_coalesce_large()
1695 extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extent_record() argument
1705 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_record()
1717 extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, in extent_record()
1720 assert(extents == &arena->extents_dirty); in extent_record()
1725 extent = extent_try_coalesce_large(tsdn, arena, in extent_record()
1732 arena_decay_extent(tsdn, arena, r_extent_hooks, extent); in extent_record()
1736 extent_deactivate_locked(tsdn, arena, extents, extent); in extent_record()
1742 extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { in extent_dalloc_gap() argument
1749 extent_dalloc(tsdn, arena, extent); in extent_dalloc_gap()
1752 extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); in extent_dalloc_gap()
1776 extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, in extent_dalloc_wrapper_try() argument
1787 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_dalloc_wrapper_try()
1794 extent_hook_pre_reentrancy(tsdn, arena); in extent_dalloc_wrapper_try()
1798 extent_committed_get(extent), arena_ind_get(arena))); in extent_dalloc_wrapper_try()
1803 extent_dalloc(tsdn, arena, extent); in extent_dalloc_wrapper_try()
1810 extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_dalloc_wrapper() argument
1823 if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, in extent_dalloc_wrapper()
1831 extent_hook_pre_reentrancy(tsdn, arena); in extent_dalloc_wrapper()
1837 } else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent, in extent_dalloc_wrapper()
1843 extent_size_get(extent), arena_ind_get(arena))) { in extent_dalloc_wrapper()
1849 extent_size_get(extent), arena_ind_get(arena)))) { in extent_dalloc_wrapper()
1863 extent_record(tsdn, arena, r_extent_hooks, &arena->extents_retained, in extent_dalloc_wrapper()
1881 extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_destroy_wrapper() argument
1893 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_destroy_wrapper()
1900 extent_hook_pre_reentrancy(tsdn, arena); in extent_destroy_wrapper()
1903 extent_committed_get(extent), arena_ind_get(arena)); in extent_destroy_wrapper()
1907 extent_dalloc(tsdn, arena, extent); in extent_destroy_wrapper()
1918 extent_commit_impl(tsdn_t *tsdn, arena_t *arena, in extent_commit_impl() argument
1924 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_commit_impl()
1926 extent_hook_pre_reentrancy(tsdn, arena); in extent_commit_impl()
1930 extent_size_get(extent), offset, length, arena_ind_get(arena))); in extent_commit_impl()
1939 extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_commit_wrapper() argument
1942 return extent_commit_impl(tsdn, arena, r_extent_hooks, extent, offset, in extent_commit_wrapper()
1954 extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_decommit_wrapper() argument
1960 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_decommit_wrapper()
1963 extent_hook_pre_reentrancy(tsdn, arena); in extent_decommit_wrapper()
1968 arena_ind_get(arena))); in extent_decommit_wrapper()
1991 extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, in extent_purge_lazy_impl() argument
1997 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_purge_lazy_impl()
2003 extent_hook_pre_reentrancy(tsdn, arena); in extent_purge_lazy_impl()
2007 arena_ind_get(arena)); in extent_purge_lazy_impl()
2016 extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_purge_lazy_wrapper() argument
2019 return extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, extent, in extent_purge_lazy_wrapper()
2038 extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, in extent_purge_forced_impl() argument
2044 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_purge_forced_impl()
2050 extent_hook_pre_reentrancy(tsdn, arena); in extent_purge_forced_impl()
2054 arena_ind_get(arena)); in extent_purge_forced_impl()
2062 extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_purge_forced_wrapper() argument
2065 return extent_purge_forced_impl(tsdn, arena, r_extent_hooks, extent, in extent_purge_forced_wrapper()
2092 extent_split_impl(tsdn_t *tsdn, arena_t *arena, in extent_split_impl() argument
2100 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_split_impl()
2106 extent_t *trail = extent_alloc(tsdn, arena); in extent_split_impl()
2111 extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) + in extent_split_impl()
2123 extent_init(&lead, arena, extent_addr_get(extent), size_a, in extent_split_impl()
2144 extent_hook_pre_reentrancy(tsdn, arena); in extent_split_impl()
2148 arena_ind_get(arena)); in extent_split_impl()
2170 extent_dalloc(tsdn, arena, trail); in extent_split_impl()
2176 extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_split_wrapper() argument
2179 return extent_split_impl(tsdn, arena, r_extent_hooks, extent, size_a, in extent_split_wrapper()
2242 extent_merge_impl(tsdn_t *tsdn, arena_t *arena, in extent_merge_impl() argument
2249 extent_hooks_assure_initialized(arena, r_extent_hooks); in extent_merge_impl()
2261 extent_hook_pre_reentrancy(tsdn, arena); in extent_merge_impl()
2265 arena_ind_get(arena)); in extent_merge_impl()
2316 extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_merge_wrapper() argument
2318 return extent_merge_impl(tsdn, arena, r_extent_hooks, a, b, false); in extent_merge_wrapper()
2390 const arena_t *arena = extent_arena_get(extent); in extent_util_stats_verbose_get() local
2391 assert(arena != NULL); in extent_util_stats_verbose_get()
2393 bin_t *bin = &arena->bins[szind].bin_shards[binshard]; in extent_util_stats_verbose_get()