34 static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena,
43 static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
50 static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
56 static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena,
63 static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena,
102 static void extent_deregister(tsdn_t *tsdn, extent_t *extent);
103 static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena,
107 static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
110 static void extent_record(tsdn_t *tsdn, arena_t *arena,
130 extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm,  in extent_rtree_leaf_elm_try_lock()  argument
132 	extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree,  in extent_rtree_leaf_elm_try_lock()
136 	if (extent1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn,  in extent_rtree_leaf_elm_try_lock()
145 	extent_lock(tsdn, extent1);  in extent_rtree_leaf_elm_try_lock()
146 	extent_t *extent2 = rtree_leaf_elm_extent_read(tsdn,  in extent_rtree_leaf_elm_try_lock()
153 		extent_unlock(tsdn, extent1);  in extent_rtree_leaf_elm_try_lock()
163 extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr,  in extent_lock_from_addr()  argument
166 	rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree,  in extent_lock_from_addr()
173 		lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret,  in extent_lock_from_addr()
180 extent_alloc(tsdn_t *tsdn, arena_t *arena) {  in extent_alloc()  argument
181 	malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);  in extent_alloc()
184 		malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);  in extent_alloc()
185 		return base_alloc_extent(tsdn, arena->base);  in extent_alloc()
189 	malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);  in extent_alloc()
194 extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {  in extent_dalloc()  argument
195 	malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);  in extent_dalloc()
198 	malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);  in extent_dalloc()
286 extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,  in extents_init()  argument
340 extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) {  in extents_insert_locked()  argument
341 	malloc_mutex_assert_owner(tsdn, &extents->mtx);  in extents_insert_locked()
371 extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) {  in extents_remove_locked()  argument
372 	malloc_mutex_assert_owner(tsdn, &extents->mtx);  in extents_remove_locked()
443 extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,  in extents_first_fit_locked()  argument
493 extents_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,  in extents_fit_locked()  argument
495 	malloc_mutex_assert_owner(tsdn, &extents->mtx);  in extents_fit_locked()
504 	    extents_first_fit_locked(tsdn, arena, extents, max_size);  in extents_fit_locked()
520 extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena,  in extent_try_delayed_coalesce()  argument
525 	extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx,  in extent_try_delayed_coalesce()
532 	extents_insert_locked(tsdn, extents, extent);  in extent_try_delayed_coalesce()
537 extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,  in extents_alloc()  argument
542 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extents_alloc()
545 	extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks, extents,  in extents_alloc()
552 extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,  in extents_dalloc()  argument
557 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extents_dalloc()
563 	extent_record(tsdn, arena, r_extent_hooks, extents, extent, false);  in extents_dalloc()
567 extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,  in extents_evict()  argument
570 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);  in extents_evict()
572 	malloc_mutex_lock(tsdn, &extents->mtx);  in extents_evict()
592 		extents_remove_locked(tsdn, extents, extent);  in extents_evict()
597 		if (extent_try_delayed_coalesce(tsdn, arena, r_extent_hooks,  in extents_evict()
619 		extent_deregister(tsdn, extent);  in extents_evict()
626 	malloc_mutex_unlock(tsdn, &extents->mtx);  in extents_evict()
635 extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,  in extents_abandon_vm()  argument
646 		if (extent_purge_lazy_impl(tsdn, arena, r_extent_hooks,  in extents_abandon_vm()
648 			extent_purge_forced_impl(tsdn, arena, r_extent_hooks,  in extents_abandon_vm()
653 	extent_dalloc(tsdn, arena, extent);  in extents_abandon_vm()
657 extents_prefork(tsdn_t *tsdn, extents_t *extents) {  in extents_prefork()  argument
658 	malloc_mutex_prefork(tsdn, &extents->mtx);  in extents_prefork()
662 extents_postfork_parent(tsdn_t *tsdn, extents_t *extents) {  in extents_postfork_parent()  argument
663 	malloc_mutex_postfork_parent(tsdn, &extents->mtx);  in extents_postfork_parent()
667 extents_postfork_child(tsdn_t *tsdn, extents_t *extents) {  in extents_postfork_child()  argument
668 	malloc_mutex_postfork_child(tsdn, &extents->mtx);  in extents_postfork_child()
672 extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,  in extent_deactivate_locked()  argument
678 	extents_insert_locked(tsdn, extents, extent);  in extent_deactivate_locked()
682 extent_deactivate(tsdn_t *tsdn, arena_t *arena, extents_t *extents,  in extent_deactivate()  argument
684 	malloc_mutex_lock(tsdn, &extents->mtx);  in extent_deactivate()
685 	extent_deactivate_locked(tsdn, arena, extents, extent);  in extent_deactivate()
686 	malloc_mutex_unlock(tsdn, &extents->mtx);  in extent_deactivate()
690 extent_activate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,  in extent_activate_locked()  argument
695 	extents_remove_locked(tsdn, extents, extent);  in extent_activate_locked()
700 extent_rtree_leaf_elms_lookup(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,  in extent_rtree_leaf_elms_lookup()  argument
703 	*r_elm_a = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,  in extent_rtree_leaf_elms_lookup()
710 	*r_elm_b = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,  in extent_rtree_leaf_elms_lookup()
721 extent_rtree_write_acquired(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a,  in extent_rtree_write_acquired()  argument
723 	rtree_leaf_elm_write(tsdn, &extents_rtree, elm_a, extent, szind, slab);  in extent_rtree_write_acquired()
725 		rtree_leaf_elm_write(tsdn, &extents_rtree, elm_b, extent, szind,  in extent_rtree_write_acquired()
731 extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, extent_t *extent,  in extent_interior_register()  argument
737 		rtree_write(tsdn, &extents_rtree, rtree_ctx,  in extent_interior_register()
744 extent_gdump_add(tsdn_t *tsdn, const extent_t *extent) {  in extent_gdump_add()  argument
747 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_gdump_add()
764 			prof_gdump(tsdn);  in extent_gdump_add()
770 extent_gdump_sub(tsdn_t *tsdn, const extent_t *extent) {  in extent_gdump_sub()  argument
781 extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool gdump_add) {  in extent_register_impl()  argument
783 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);  in extent_register_impl()
790 	extent_lock(tsdn, extent);  in extent_register_impl()
792 	if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false, true,  in extent_register_impl()
794 		extent_unlock(tsdn, extent);  in extent_register_impl()
800 	extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent, szind, slab);  in extent_register_impl()
802 		extent_interior_register(tsdn, rtree_ctx, extent, szind);  in extent_register_impl()
805 	extent_unlock(tsdn, extent);  in extent_register_impl()
808 		extent_gdump_add(tsdn, extent);  in extent_register_impl()
815 extent_register(tsdn_t *tsdn, extent_t *extent) {  in extent_register()  argument
816 	return extent_register_impl(tsdn, extent, true);  in extent_register()
820 extent_register_no_gdump_add(tsdn_t *tsdn, extent_t *extent) {  in extent_register_no_gdump_add()  argument
821 	return extent_register_impl(tsdn, extent, false);  in extent_register_no_gdump_add()
825 extent_reregister(tsdn_t *tsdn, extent_t *extent) {  in extent_reregister()  argument
826 	bool err = extent_register(tsdn, extent);  in extent_reregister()
838 extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,  in extent_interior_deregister()  argument
845 		rtree_clear(tsdn, &extents_rtree, rtree_ctx,  in extent_interior_deregister()
855 extent_deregister_impl(tsdn_t *tsdn, extent_t *extent, bool gdump) {  in extent_deregister_impl()  argument
857 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);  in extent_deregister_impl()
859 	extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, true, false,  in extent_deregister_impl()
862 	extent_lock(tsdn, extent);  in extent_deregister_impl()
864 	extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false);  in extent_deregister_impl()
866 		extent_interior_deregister(tsdn, rtree_ctx, extent);  in extent_deregister_impl()
870 	extent_unlock(tsdn, extent);  in extent_deregister_impl()
873 		extent_gdump_sub(tsdn, extent);  in extent_deregister_impl()
878 extent_deregister(tsdn_t *tsdn, extent_t *extent) {  in extent_deregister()  argument
879 	extent_deregister_impl(tsdn, extent, true);  in extent_deregister()
883 extent_deregister_no_gdump_sub(tsdn_t *tsdn, extent_t *extent) {  in extent_deregister_no_gdump_sub()  argument
884 	extent_deregister_impl(tsdn, extent, false);  in extent_deregister_no_gdump_sub()
892 extent_recycle_extract(tsdn_t *tsdn, arena_t *arena,  in extent_recycle_extract()  argument
896 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_recycle_extract()
917 	malloc_mutex_lock(tsdn, &extents->mtx);  in extent_recycle_extract()
921 		extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr,  in extent_recycle_extract()
936 			extent_unlock(tsdn, unlock_extent);  in extent_recycle_extract()
939 		extent = extents_fit_locked(tsdn, arena, extents, esize,  in extent_recycle_extract()
943 		malloc_mutex_unlock(tsdn, &extents->mtx);  in extent_recycle_extract()
947 	extent_activate_locked(tsdn, arena, extents, extent);  in extent_recycle_extract()
948 	malloc_mutex_unlock(tsdn, &extents->mtx);  in extent_recycle_extract()
980 extent_split_interior(tsdn_t *tsdn, arena_t *arena,  in extent_split_interior()  argument
1005 		*extent = extent_split_impl(tsdn, arena, r_extent_hooks,  in extent_split_interior()
1017 		*trail = extent_split_impl(tsdn, arena, r_extent_hooks, *extent,  in extent_split_interior()
1036 			rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx,  in extent_split_interior()
1039 				rtree_szind_slab_update(tsdn, &extents_rtree,  in extent_split_interior()
1057 extent_recycle_split(tsdn_t *tsdn, arena_t *arena,  in extent_recycle_split()  argument
1067 	    tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail,  in extent_recycle_split()
1078 		extent_deactivate(tsdn, arena, extents, to_leak);  in extent_recycle_split()
1084 			extent_deactivate(tsdn, arena, extents, lead);  in extent_recycle_split()
1087 			extent_deactivate(tsdn, arena, extents, trail);  in extent_recycle_split()
1097 			extent_deregister(tsdn, to_salvage);  in extent_recycle_split()
1101 			extent_deregister_no_gdump_sub(tsdn, to_leak);  in extent_recycle_split()
1102 			extents_abandon_vm(tsdn, arena, r_extent_hooks, extents,  in extent_recycle_split()
1104 			assert(extent_lock_from_addr(tsdn, rtree_ctx, leak,  in extent_recycle_split()
1128 extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,  in extent_recycle()  argument
1132 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_recycle()
1139 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);  in extent_recycle()
1141 	extent_t *extent = extent_recycle_extract(tsdn, arena, r_extent_hooks,  in extent_recycle()
1148 	extent = extent_recycle_split(tsdn, arena, r_extent_hooks, rtree_ctx,  in extent_recycle()
1156 		if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent,  in extent_recycle()
1158 			extent_record(tsdn, arena, r_extent_hooks, extents,  in extent_recycle()
1175 		extent_addr_randomize(tsdn, extent, alignment);  in extent_recycle()
1180 		extent_interior_register(tsdn, rtree_ctx, extent, szind);  in extent_recycle()
1209 extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,  in extent_alloc_core()  argument
1218 	    extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,  in extent_alloc_core()
1229 	    extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,  in extent_alloc_core()
1239 extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr,  in extent_alloc_default_impl()  argument
1241 	void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero,  in extent_alloc_default_impl()
1253 	tsdn_t *tsdn;  in extent_alloc_default()  local
1256 	tsdn = tsdn_fetch();  in extent_alloc_default()
1257 	arena = arena_get(tsdn, arena_ind, false);  in extent_alloc_default()
1264 	return extent_alloc_default_impl(tsdn, arena, new_addr, size,  in extent_alloc_default()
1269 extent_hook_pre_reentrancy(tsdn_t *tsdn, arena_t *arena) {  in extent_hook_pre_reentrancy()  argument
1270 	tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);  in extent_hook_pre_reentrancy()
1286 extent_hook_post_reentrancy(tsdn_t *tsdn) {  in extent_hook_post_reentrancy()  argument
1287 	tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);  in extent_hook_post_reentrancy()
1297 extent_grow_retained(tsdn_t *tsdn, arena_t *arena,  in extent_grow_retained()  argument
1300 	malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx);  in extent_grow_retained()
1326 	extent_t *extent = extent_alloc(tsdn, arena);  in extent_grow_retained()
1335 		ptr = extent_alloc_default_impl(tsdn, arena, NULL,  in extent_grow_retained()
1338 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_grow_retained()
1342 		extent_hook_post_reentrancy(tsdn);  in extent_grow_retained()
1349 		extent_dalloc(tsdn, arena, extent);  in extent_grow_retained()
1353 	if (extent_register_no_gdump_add(tsdn, extent)) {  in extent_grow_retained()
1354 		extent_dalloc(tsdn, arena, extent);  in extent_grow_retained()
1366 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);  in extent_grow_retained()
1373 	    tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail,  in extent_grow_retained()
1379 			extent_record(tsdn, arena, r_extent_hooks,  in extent_grow_retained()
1383 			extent_record(tsdn, arena, r_extent_hooks,  in extent_grow_retained()
1394 				extent_gdump_add(tsdn, to_salvage);  in extent_grow_retained()
1396 			extent_record(tsdn, arena, r_extent_hooks,  in extent_grow_retained()
1400 			extent_deregister_no_gdump_sub(tsdn, to_leak);  in extent_grow_retained()
1401 			extents_abandon_vm(tsdn, arena, r_extent_hooks,  in extent_grow_retained()
1408 		if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, 0,  in extent_grow_retained()
1410 			extent_record(tsdn, arena, r_extent_hooks,  in extent_grow_retained()
1430 	malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);  in extent_grow_retained()
1434 		extent_gdump_add(tsdn, extent);  in extent_grow_retained()
1437 		extent_addr_randomize(tsdn, extent, alignment);  in extent_grow_retained()
1441 		rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,  in extent_grow_retained()
1445 		extent_interior_register(tsdn, rtree_ctx, extent, szind);  in extent_grow_retained()
1458 	malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);  in extent_grow_retained()
1463 extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,  in extent_alloc_retained()  argument
1469 	malloc_mutex_lock(tsdn, &arena->extent_grow_mtx);  in extent_alloc_retained()
1471 	extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks,  in extent_alloc_retained()
1475 		malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);  in extent_alloc_retained()
1477 			extent_gdump_add(tsdn, extent);  in extent_alloc_retained()
1480 		extent = extent_grow_retained(tsdn, arena, r_extent_hooks, size,  in extent_alloc_retained()
1484 		malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);  in extent_alloc_retained()
1486 	malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx);  in extent_alloc_retained()
1492 extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,  in extent_alloc_wrapper_hard()  argument
1496 	extent_t *extent = extent_alloc(tsdn, arena);  in extent_alloc_wrapper_hard()
1504 		addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize,  in extent_alloc_wrapper_hard()
1507 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_alloc_wrapper_hard()
1510 		extent_hook_post_reentrancy(tsdn);  in extent_alloc_wrapper_hard()
1513 		extent_dalloc(tsdn, arena, extent);  in extent_alloc_wrapper_hard()
1520 		extent_addr_randomize(tsdn, extent, alignment);  in extent_alloc_wrapper_hard()
1522 	if (extent_register(tsdn, extent)) {  in extent_alloc_wrapper_hard()
1523 		extent_dalloc(tsdn, arena, extent);  in extent_alloc_wrapper_hard()
1531 extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,  in extent_alloc_wrapper()  argument
1534 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_alloc_wrapper()
1539 	extent_t *extent = extent_alloc_retained(tsdn, arena, r_extent_hooks,  in extent_alloc_wrapper()
1551 		extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks,  in extent_alloc_wrapper()
1580 extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,  in extent_coalesce()  argument
1585 	extent_activate_locked(tsdn, arena, extents, outer);  in extent_coalesce()
1587 	malloc_mutex_unlock(tsdn, &extents->mtx);  in extent_coalesce()
1588 	bool err = extent_merge_impl(tsdn, arena, r_extent_hooks,  in extent_coalesce()
1590 	malloc_mutex_lock(tsdn, &extents->mtx);  in extent_coalesce()
1593 		extent_deactivate_locked(tsdn, arena, extents, outer);  in extent_coalesce()
1600 extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena,  in extent_try_coalesce_impl()  argument
1618 		extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx,  in extent_try_coalesce_impl()
1629 			extent_unlock(tsdn, next);  in extent_try_coalesce_impl()
1631 			if (can_coalesce && !extent_coalesce(tsdn, arena,  in extent_try_coalesce_impl()
1646 			prev = extent_lock_from_addr(tsdn, rtree_ctx,  in extent_try_coalesce_impl()
1652 			extent_unlock(tsdn, prev);  in extent_try_coalesce_impl()
1654 			if (can_coalesce && !extent_coalesce(tsdn, arena,  in extent_try_coalesce_impl()
1675 extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,  in extent_try_coalesce()  argument
1678 	return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx,  in extent_try_coalesce()
1683 extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena,  in extent_try_coalesce_large()  argument
1686 	return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx,  in extent_try_coalesce_large()
1695 extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,  in extent_record()  argument
1698 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);  in extent_record()
1704 	malloc_mutex_lock(tsdn, &extents->mtx);  in extent_record()
1709 		extent_interior_deregister(tsdn, rtree_ctx, extent);  in extent_record()
1713 	assert(rtree_extent_read(tsdn, &extents_rtree, rtree_ctx,  in extent_record()
1717 		extent = extent_try_coalesce(tsdn, arena, r_extent_hooks,  in extent_record()
1725 			extent = extent_try_coalesce_large(tsdn, arena,  in extent_record()
1731 			malloc_mutex_unlock(tsdn, &extents->mtx);  in extent_record()
1732 			arena_decay_extent(tsdn, arena, r_extent_hooks, extent);  in extent_record()
1736 	extent_deactivate_locked(tsdn, arena, extents, extent);  in extent_record()
1738 	malloc_mutex_unlock(tsdn, &extents->mtx);  in extent_record()
1742 extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {  in extent_dalloc_gap()  argument
1745 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_dalloc_gap()
1748 	if (extent_register(tsdn, extent)) {  in extent_dalloc_gap()
1749 		extent_dalloc(tsdn, arena, extent);  in extent_dalloc_gap()
1752 	extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent);  in extent_dalloc_gap()
1776 extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,  in extent_dalloc_wrapper_try()  argument
1782 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_dalloc_wrapper_try()
1794 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_dalloc_wrapper_try()
1799 		extent_hook_post_reentrancy(tsdn);  in extent_dalloc_wrapper_try()
1803 		extent_dalloc(tsdn, arena, extent);  in extent_dalloc_wrapper_try()
1810 extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,  in extent_dalloc_wrapper()  argument
1813 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_dalloc_wrapper()
1822 		extent_deregister(tsdn, extent);  in extent_dalloc_wrapper()
1823 		if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks,  in extent_dalloc_wrapper()
1827 		extent_reregister(tsdn, extent);  in extent_dalloc_wrapper()
1831 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_dalloc_wrapper()
1837 	} else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,  in extent_dalloc_wrapper()
1855 		extent_hook_post_reentrancy(tsdn);  in extent_dalloc_wrapper()
1860 		extent_gdump_sub(tsdn, extent);  in extent_dalloc_wrapper()
1863 	extent_record(tsdn, arena, r_extent_hooks, &arena->extents_retained,  in extent_dalloc_wrapper()
1881 extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena,  in extent_destroy_wrapper()  argument
1885 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_destroy_wrapper()
1889 	extent_deregister(tsdn, extent);  in extent_destroy_wrapper()
1900 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_destroy_wrapper()
1904 		extent_hook_post_reentrancy(tsdn);  in extent_destroy_wrapper()
1907 	extent_dalloc(tsdn, arena, extent);  in extent_destroy_wrapper()
1918 extent_commit_impl(tsdn_t *tsdn, arena_t *arena,  in extent_commit_impl()  argument
1921 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_commit_impl()
1926 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_commit_impl()
1932 		extent_hook_post_reentrancy(tsdn);  in extent_commit_impl()
1939 extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,  in extent_commit_wrapper()  argument
1942 	return extent_commit_impl(tsdn, arena, r_extent_hooks, extent, offset,  in extent_commit_wrapper()
1954 extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,  in extent_decommit_wrapper()  argument
1957 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_decommit_wrapper()
1963 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_decommit_wrapper()
1970 		extent_hook_post_reentrancy(tsdn);  in extent_decommit_wrapper()
1991 extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,  in extent_purge_lazy_impl()  argument
1994 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_purge_lazy_impl()
2003 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_purge_lazy_impl()
2009 		extent_hook_post_reentrancy(tsdn);  in extent_purge_lazy_impl()
2016 extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,  in extent_purge_lazy_wrapper()  argument
2019 	return extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, extent,  in extent_purge_lazy_wrapper()
2038 extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,  in extent_purge_forced_impl()  argument
2041 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_purge_forced_impl()
2050 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_purge_forced_impl()
2056 		extent_hook_post_reentrancy(tsdn);  in extent_purge_forced_impl()
2062 extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,  in extent_purge_forced_wrapper()  argument
2065 	return extent_purge_forced_impl(tsdn, arena, r_extent_hooks, extent,  in extent_purge_forced_wrapper()
2092 extent_split_impl(tsdn_t *tsdn, arena_t *arena,  in extent_split_impl()  argument
2097 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_split_impl()
2106 	extent_t *trail = extent_alloc(tsdn, arena);  in extent_split_impl()
2118 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);  in extent_split_impl()
2129 		extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false,  in extent_split_impl()
2133 	extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, trail, false, true,  in extent_split_impl()
2141 	extent_lock2(tsdn, extent, trail);  in extent_split_impl()
2144 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_split_impl()
2150 		extent_hook_post_reentrancy(tsdn);  in extent_split_impl()
2159 	extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent,  in extent_split_impl()
2161 	extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail,  in extent_split_impl()
2164 	extent_unlock2(tsdn, extent, trail);  in extent_split_impl()
2168 	extent_unlock2(tsdn, extent, trail);  in extent_split_impl()
2170 	extent_dalloc(tsdn, arena, trail);  in extent_split_impl()
2176 extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,  in extent_split_wrapper()  argument
2179 	return extent_split_impl(tsdn, arena, r_extent_hooks, extent, size_a,  in extent_split_wrapper()
2231 		tsdn_t *tsdn = tsdn_fetch();  in extent_merge_default()  local
2232 		extent_t *a = iealloc(tsdn, addr_a);  in extent_merge_default()
2233 		extent_t *b = iealloc(tsdn, addr_b);  in extent_merge_default()
2242 extent_merge_impl(tsdn_t *tsdn, arena_t *arena,  in extent_merge_impl()  argument
2245 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),  in extent_merge_impl()
2261 		extent_hook_pre_reentrancy(tsdn, arena);  in extent_merge_impl()
2266 		extent_hook_post_reentrancy(tsdn);  in extent_merge_impl()
2279 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);  in extent_merge_impl()
2281 	extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, a, true, false, &a_elm_a,  in extent_merge_impl()
2283 	extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, b, true, false, &b_elm_a,  in extent_merge_impl()
2286 	extent_lock2(tsdn, a, b);  in extent_merge_impl()
2289 		rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL,  in extent_merge_impl()
2293 		rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL,  in extent_merge_impl()
2305 	extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES,  in extent_merge_impl()
2308 	extent_unlock2(tsdn, a, b);  in extent_merge_impl()
2310 	extent_dalloc(tsdn, extent_arena_get(b), b);  in extent_merge_impl()
2316 extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,  in extent_merge_wrapper()  argument
2318 	return extent_merge_impl(tsdn, arena, r_extent_hooks, a, b, false);  in extent_merge_wrapper()
2340 extent_util_stats_get(tsdn_t *tsdn, const void *ptr,  in extent_util_stats_get()  argument
2344 	const extent_t *extent = iealloc(tsdn, ptr);  in extent_util_stats_get()
2363 extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,  in extent_util_stats_verbose_get()  argument
2369 	const extent_t *extent = iealloc(tsdn, ptr);  in extent_util_stats_verbose_get()
2395 	malloc_mutex_lock(tsdn, &bin->lock);  in extent_util_stats_verbose_get()
2405 	malloc_mutex_unlock(tsdn, &bin->lock);  in extent_util_stats_verbose_get()