237 static void	prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
240 static void	prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
313 prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) {  in rb_gen()
325 		tdata = prof_tdata_get(tsd, true);  in rb_gen()
332 		malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);  in rb_gen()
334 		if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) {  in rb_gen()
335 			prof_tctx_destroy(tsd, tctx);  in rb_gen()
337 			malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);  in rb_gen()
365 prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) {  in prof_log_bt_index()  argument
367 	malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);  in prof_log_bt_index()
379 		    iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,  in prof_log_bt_index()
400 		ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL);  in prof_log_bt_index()
407 prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) {  in prof_log_thr_index()  argument
409 	malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);  in prof_log_thr_index()
420 		    iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,  in prof_log_thr_index()
436 		ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL);  in prof_log_thr_index()
444 prof_try_log(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx) {  in prof_try_log()  argument
445 	malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_try_log()
447 	prof_tdata_t *cons_tdata = prof_tdata_get(tsd, false);  in prof_try_log()
457 	malloc_mutex_lock(tsd_tsdn(tsd), &log_mtx);  in prof_try_log()
464 		bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS,  in prof_try_log()
466 		bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS,  in prof_try_log()
474 	nstime_t alloc_time = prof_alloc_time_get(tsd_tsdn(tsd), ptr,  in prof_try_log()
481 	    iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true,  in prof_try_log()
486 	const char *cons_thr_name = prof_thread_name_get(tsd);  in prof_try_log()
498 	new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid,  in prof_try_log()
500 	new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid,  in prof_try_log()
502 	new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt);  in prof_try_log()
503 	new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt);  in prof_try_log()
517 	malloc_mutex_unlock(tsd_tsdn(tsd), &log_mtx);  in prof_try_log()
521 prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize,  in prof_free_sampled_object()  argument
523 	malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_free_sampled_object()
530 	prof_try_log(tsd, ptr, usize, tctx);  in prof_free_sampled_object()
532 	if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) {  in prof_free_sampled_object()
533 		prof_tctx_destroy(tsd, tctx);  in prof_free_sampled_object()
535 		malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_free_sampled_object()
548 prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_enter()  argument
550 	assert(tdata == prof_tdata_get(tsd, false));  in prof_enter()
557 	malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);  in prof_enter()
561 prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_leave()  argument
563 	assert(tdata == prof_tdata_get(tsd, false));  in prof_leave()
565 	malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);  in prof_leave()
578 			prof_idump(tsd_tsdn(tsd));  in prof_leave()
581 			prof_gdump(tsd_tsdn(tsd));  in prof_leave()
847 prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,  in prof_gctx_try_destroy()  argument
858 	prof_enter(tsd, tdata_self);  in prof_gctx_try_destroy()
859 	malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);  in prof_gctx_try_destroy()
863 		if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) {  in prof_gctx_try_destroy()
866 		prof_leave(tsd, tdata_self);  in prof_gctx_try_destroy()
868 		malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);  in prof_gctx_try_destroy()
869 		idalloctm(tsd_tsdn(tsd), gctx, NULL, NULL, true, true);  in prof_gctx_try_destroy()
876 		malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);  in prof_gctx_try_destroy()
877 		prof_leave(tsd, tdata_self);  in prof_gctx_try_destroy()
912 prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) {  in prof_tctx_destroy()  argument
917 	malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_tctx_destroy()
925 	ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);  in prof_tctx_destroy()
926 	destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false);  in prof_tctx_destroy()
927 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_tctx_destroy()
929 	malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);  in prof_tctx_destroy()
970 	malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);  in prof_tctx_destroy()
972 		prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,  in prof_tctx_destroy()
976 	malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_tctx_destroy()
979 		prof_tdata_destroy(tsd, tdata, false);  in prof_tctx_destroy()
983 		idalloctm(tsd_tsdn(tsd), tctx, NULL, NULL, true, true);  in prof_tctx_destroy()
988 prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,  in prof_lookup_global()  argument
1000 	prof_enter(tsd, tdata);  in prof_lookup_global()
1003 		prof_leave(tsd, tdata);  in prof_lookup_global()
1004 		tgctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);  in prof_lookup_global()
1008 		prof_enter(tsd, tdata);  in prof_lookup_global()
1012 			if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {  in prof_lookup_global()
1014 				prof_leave(tsd, tdata);  in prof_lookup_global()
1015 				idalloctm(tsd_tsdn(tsd), gctx.v, NULL, NULL,  in prof_lookup_global()
1033 		malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);  in prof_lookup_global()
1035 		malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);  in prof_lookup_global()
1040 			idalloctm(tsd_tsdn(tsd), tgctx.v, NULL, NULL, true,  in prof_lookup_global()
1044 	prof_leave(tsd, tdata);  in prof_lookup_global()
1053 prof_lookup(tsd_t *tsd, prof_bt_t *bt) {  in prof_lookup()  argument
1063 	tdata = prof_tdata_get(tsd, false);  in prof_lookup()
1068 	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
1073 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
1083 		if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,  in prof_lookup()
1089 		ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),  in prof_lookup()
1091 		    arena_ichoose(tsd, NULL), true);  in prof_lookup()
1094 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);  in prof_lookup()
1106 		malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
1107 		error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);  in prof_lookup()
1108 		malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
1111 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);  in prof_lookup()
1113 			idalloctm(tsd_tsdn(tsd), ret.v, NULL, NULL, true, true);  in prof_lookup()
1116 		malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);  in prof_lookup()
1120 		malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);  in prof_lookup()
1210 	tsd_t *tsd;  in prof_bt_count()  local
1213 	tsd = tsd_fetch();  in prof_bt_count()
1214 	tdata = prof_tdata_get(tsd, false);  in prof_bt_count()
1219 	malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);  in prof_bt_count()
1221 	malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);  in prof_bt_count()
1490 prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) {  in prof_gctx_finish()  argument
1491 	prof_tdata_t *tdata = prof_tdata_get(tsd, false);  in prof_gctx_finish()
1502 		malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);  in prof_gctx_finish()
1511 				    (void *)tsd_tsdn(tsd));  in prof_gctx_finish()
1517 					idalloctm(tsd_tsdn(tsd), to_destroy,  in prof_gctx_finish()
1527 			malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);  in prof_gctx_finish()
1528 			prof_gctx_try_destroy(tsd, tdata, gctx, tdata);  in prof_gctx_finish()
1530 			malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);  in prof_gctx_finish()
1823 prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata,  in prof_dump_prep()  argument
1833 	prof_enter(tsd, tdata);  in prof_dump_prep()
1841 		prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, gctxs);  in prof_dump_prep()
1848 	prof_tdata_merge_iter_arg->tsdn = tsd_tsdn(tsd);  in prof_dump_prep()
1850 	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);  in prof_dump_prep()
1853 	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);  in prof_dump_prep()
1856 	prof_gctx_merge_iter_arg->tsdn = tsd_tsdn(tsd);  in prof_dump_prep()
1861 	prof_leave(tsd, tdata);  in prof_dump_prep()
1865 prof_dump_file(tsd_t *tsd, bool propagate_err, const char *filename,  in prof_dump_file()  argument
1877 	if (prof_dump_header(tsd_tsdn(tsd), propagate_err,  in prof_dump_file()
1883 	prof_gctx_dump_iter_arg->tsdn = tsd_tsdn(tsd);  in prof_dump_file()
1906 prof_dump(tsd_t *tsd, bool propagate_err, const char *filename,  in prof_dump()  argument
1909 	assert(tsd_reentrancy_level_get(tsd) == 0);  in prof_dump()
1911 	prof_tdata_t * tdata = prof_tdata_get(tsd, true);  in prof_dump()
1916 	pre_reentrancy(tsd, NULL);  in prof_dump()
1917 	malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);  in prof_dump()
1923 	prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,  in prof_dump()
1925 	bool err = prof_dump_file(tsd, propagate_err, filename, leakcheck, tdata,  in prof_dump()
1928 	prof_gctx_finish(tsd, &gctxs);  in prof_dump()
1930 	malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);  in prof_dump()
1931 	post_reentrancy(tsd);  in prof_dump()
1948 	tsd_t *tsd;  in prof_cnt_all()  local
1954 	tsd = tsd_fetch();  in prof_cnt_all()
1955 	tdata = prof_tdata_get(tsd, false);  in prof_cnt_all()
1972 	prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,  in prof_cnt_all()
1974 	prof_gctx_finish(tsd, &gctxs);  in prof_cnt_all()
2013 	tsd_t *tsd;  in prof_fdump()  local
2023 	tsd = tsd_fetch();  in prof_fdump()
2024 	assert(tsd_reentrancy_level_get(tsd) == 0);  in prof_fdump()
2026 	malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);  in prof_fdump()
2028 	malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);  in prof_fdump()
2029 	prof_dump(tsd, false, filename, opt_prof_leak);  in prof_fdump()
2050 	tsd_t *tsd;  in prof_idump()  local
2058 	tsd = tsdn_tsd(tsdn);  in prof_idump()
2059 	if (tsd_reentrancy_level_get(tsd) > 0) {  in prof_idump()
2063 	tdata = prof_tdata_get(tsd, false);  in prof_idump()
2074 		malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);  in prof_idump()
2077 		malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);  in prof_idump()
2078 		prof_dump(tsd, false, filename, false);  in prof_idump()
2083 prof_mdump(tsd_t *tsd, const char *filename) {  in prof_mdump()  argument
2085 	assert(tsd_reentrancy_level_get(tsd) == 0);  in prof_mdump()
2096 		malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);  in prof_mdump()
2099 		malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);  in prof_mdump()
2102 	return prof_dump(tsd, true, filename, false);  in prof_mdump()
2107 	tsd_t *tsd;  in prof_gdump()  local
2115 	tsd = tsdn_tsd(tsdn);  in prof_gdump()
2116 	if (tsd_reentrancy_level_get(tsd) > 0) {  in prof_gdump()
2120 	tdata = prof_tdata_get(tsd, false);  in prof_gdump()
2135 		prof_dump(tsd, false, filename, false);  in prof_gdump()
2201 prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,  in prof_tdata_init_impl()  argument
2208 	tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),  in prof_tdata_init_impl()
2223 	if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,  in prof_tdata_init_impl()
2225 		idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);  in prof_tdata_init_impl()
2239 	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);  in prof_tdata_init_impl()
2241 	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);  in prof_tdata_init_impl()
2247 prof_tdata_init(tsd_t *tsd) {  in prof_tdata_init()  argument
2248 	return prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0,  in prof_tdata_init()
2249 	    NULL, prof_thread_active_init_get(tsd_tsdn(tsd)));  in prof_tdata_init()
2272 prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,  in prof_tdata_destroy_locked()  argument
2274 	malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx);  in prof_tdata_destroy_locked()
2281 		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,  in prof_tdata_destroy_locked()
2284 	ckh_delete(tsd, &tdata->bt2tctx);  in prof_tdata_destroy_locked()
2285 	idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);  in prof_tdata_destroy_locked()
2289 prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {  in prof_tdata_destroy()  argument
2290 	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);  in prof_tdata_destroy()
2291 	prof_tdata_destroy_locked(tsd, tdata, even_if_attached);  in prof_tdata_destroy()
2292 	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);  in prof_tdata_destroy()
2296 prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_tdata_detach()  argument
2299 	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);  in prof_tdata_detach()
2301 		destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,  in prof_tdata_detach()
2310 		tsd_prof_tdata_set(tsd, NULL);  in prof_tdata_detach()
2314 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_tdata_detach()
2316 		prof_tdata_destroy(tsd, tdata, true);  in prof_tdata_detach()
2321 prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_tdata_reinit()  argument
2325 	    prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL;  in prof_tdata_reinit()
2328 	prof_tdata_detach(tsd, tdata);  in prof_tdata_reinit()
2329 	return prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,  in prof_tdata_reinit()
2359 prof_reset(tsd_t *tsd, size_t lg_sample) {  in prof_reset()  argument
2364 	malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);  in prof_reset()
2365 	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);  in prof_reset()
2372 		    prof_tdata_reset_iter, (void *)tsd);  in prof_reset()
2375 			prof_tdata_destroy_locked(tsd, to_destroy, false);  in prof_reset()
2381 	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);  in prof_reset()
2382 	malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);  in prof_reset()
2386 prof_tdata_cleanup(tsd_t *tsd) {  in prof_tdata_cleanup()  argument
2393 	tdata = tsd_prof_tdata_get(tsd);  in prof_tdata_cleanup()
2395 		prof_tdata_detach(tsd, tdata);  in prof_tdata_cleanup()
2554 	tsd_t *tsd = tsd_fetch();  in prof_log_stop_final()  local
2555 	prof_log_stop(tsd_tsdn(tsd));  in prof_log_stop_final()
2581 prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) {  in prof_log_emit_threads()  argument
2599 		idalloc(tsd, thr_old_node);  in prof_log_emit_threads()
2605 prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) {  in prof_log_emit_traces()  argument
2628 		idalloc(tsd, bt_old_node);  in prof_log_emit_traces()
2634 prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) {  in prof_log_emit_allocs()  argument
2666 		idalloc(tsd, alloc_old_node);  in prof_log_emit_allocs()
2701 	tsd_t *tsd = tsdn_tsd(tsdn);  in prof_log_stop()  local
2750 	prof_log_emit_threads(tsd, &emitter);  in prof_log_stop()
2751 	prof_log_emit_traces(tsd, &emitter);  in prof_log_stop()
2752 	prof_log_emit_allocs(tsd, &emitter);  in prof_log_stop()
2757 		ckh_delete(tsd, &log_bt_node_set);  in prof_log_stop()
2758 		ckh_delete(tsd, &log_thr_node_set);  in prof_log_stop()
2783 prof_thread_name_get(tsd_t *tsd) {  in prof_thread_name_get()  argument
2786 	tdata = prof_tdata_get(tsd, true);  in prof_thread_name_get()
2817 prof_thread_name_set(tsd_t *tsd, const char *thread_name) {  in prof_thread_name_set()  argument
2822 	tdata = prof_tdata_get(tsd, true);  in prof_thread_name_set()
2838 	s = prof_thread_name_alloc(tsd_tsdn(tsd), thread_name);  in prof_thread_name_set()
2844 		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,  in prof_thread_name_set()
2855 prof_thread_active_get(tsd_t *tsd) {  in prof_thread_active_get()  argument
2858 	tdata = prof_tdata_get(tsd, true);  in prof_thread_active_get()
2866 prof_thread_active_set(tsd_t *tsd, bool active) {  in prof_thread_active_set()  argument
2869 	tdata = prof_tdata_get(tsd, true);  in prof_thread_active_set()
2952 prof_boot2(tsd_t *tsd) {  in prof_boot2()  argument
2980 		if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,  in prof_boot2()
3019 			prof_log_start(tsd_tsdn(tsd), NULL);  in prof_boot2()
3035 		if (ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS,  in prof_boot2()
3040 		if (ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS,  in prof_boot2()
3047 		gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),  in prof_boot2()
3061 		tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),  in prof_boot2()