238 static bool	prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
240 static void	prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
314 	prof_tdata_t *tdata;  in rb_gen()  local
325 		tdata = prof_tdata_get(tsd, true);  in rb_gen()
326 		if (tdata != NULL) {  in rb_gen()
327 			prof_sample_threshold_update(tdata);  in rb_gen()
332 		malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);  in rb_gen()
337 			malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);  in rb_gen()
353 	malloc_mutex_lock(tsdn, tctx->tdata->lock);  in prof_malloc_sample_object()
361 	malloc_mutex_unlock(tsdn, tctx->tdata->lock);  in prof_malloc_sample_object()
445 	malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_try_log()
484 	const char *prod_thr_name = (tctx->tdata->thread_name == NULL)?  in prof_try_log()
485 				        "" : tctx->tdata->thread_name;  in prof_try_log()
498 	new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid,  in prof_try_log()
523 	malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_free_sampled_object()
535 		malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_free_sampled_object()
548 prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_enter()  argument
550 	assert(tdata == prof_tdata_get(tsd, false));  in prof_enter()
552 	if (tdata != NULL) {  in prof_enter()
553 		assert(!tdata->enq);  in prof_enter()
554 		tdata->enq = true;  in prof_enter()
561 prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_leave()  argument
563 	assert(tdata == prof_tdata_get(tsd, false));  in prof_leave()
567 	if (tdata != NULL) {  in prof_leave()
570 		assert(tdata->enq);  in prof_leave()
571 		tdata->enq = false;  in prof_leave()
572 		idump = tdata->enq_idump;  in prof_leave()
573 		tdata->enq_idump = false;  in prof_leave()
574 		gdump = tdata->enq_gdump;  in prof_leave()
575 		tdata->enq_gdump = false;  in prof_leave()
848     prof_tdata_t *tdata) {  in prof_gctx_try_destroy()
883 	malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);  in prof_tctx_should_destroy()
913 	prof_tdata_t *tdata = tctx->tdata;  in prof_tctx_destroy()  local
917 	malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_tctx_destroy()
925 	ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);  in prof_tctx_destroy()
926 	destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false);  in prof_tctx_destroy()
927 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_tctx_destroy()
973 		    tdata);  in prof_tctx_destroy()
976 	malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_tctx_destroy()
979 		prof_tdata_destroy(tsd, tdata, false);  in prof_tctx_destroy()
988 prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,  in prof_lookup_global()  argument
1000 	prof_enter(tsd, tdata);  in prof_lookup_global()
1003 		prof_leave(tsd, tdata);  in prof_lookup_global()
1008 		prof_enter(tsd, tdata);  in prof_lookup_global()
1014 				prof_leave(tsd, tdata);  in prof_lookup_global()
1044 	prof_leave(tsd, tdata);  in prof_lookup_global()
1058 	prof_tdata_t *tdata;  in prof_lookup()  local
1063 	tdata = prof_tdata_get(tsd, false);  in prof_lookup()
1064 	if (tdata == NULL) {  in prof_lookup()
1068 	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
1069 	not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);  in prof_lookup()
1073 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
1083 		if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,  in prof_lookup()
1094 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);  in prof_lookup()
1098 		ret.p->tdata = tdata;  in prof_lookup()
1099 		ret.p->thr_uid = tdata->thr_uid;  in prof_lookup()
1100 		ret.p->thr_discrim = tdata->thr_discrim;  in prof_lookup()
1103 		ret.p->tctx_uid = tdata->tctx_uid_next++;  in prof_lookup()
1106 		malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
1107 		error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);  in prof_lookup()
1108 		malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
1111 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);  in prof_lookup()
1140 prof_sample_threshold_update(prof_tdata_t *tdata) {  in prof_sample_threshold_update()  argument
1169 	uint64_t r = prng_lg_range_u64(&tdata->prng_state, 53);  in prof_sample_threshold_update()
1184 prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,  in prof_tdata_count_iter()  argument
1211 	prof_tdata_t *tdata;  in prof_bt_count()  local
1214 	tdata = prof_tdata_get(tsd, false);  in prof_bt_count()
1215 	if (tdata == NULL) {  in prof_bt_count()
1327 prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {  in prof_tctx_merge_tdata()  argument
1328 	malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);  in prof_tctx_merge_tdata()
1342 		tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;  in prof_tctx_merge_tdata()
1343 		tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;  in prof_tctx_merge_tdata()
1345 			tdata->cnt_summed.accumobjs +=  in prof_tctx_merge_tdata()
1347 			tdata->cnt_summed.accumbytes +=  in prof_tctx_merge_tdata()
1491 	prof_tdata_t *tdata = prof_tdata_get(tsd, false);  in prof_gctx_finish()  local
1528 			prof_gctx_try_destroy(tsd, tdata, gctx, tdata);  in prof_gctx_finish()
1541 prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,  in prof_tdata_merge_iter()  argument
1546 	malloc_mutex_lock(arg->tsdn, tdata->lock);  in prof_tdata_merge_iter()
1547 	if (!tdata->expired) {  in prof_tdata_merge_iter()
1554 		tdata->dumping = true;  in prof_tdata_merge_iter()
1555 		memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));  in prof_tdata_merge_iter()
1556 		for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,  in prof_tdata_merge_iter()
1558 			prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);  in prof_tdata_merge_iter()
1561 		arg->cnt_all.curobjs += tdata->cnt_summed.curobjs;  in prof_tdata_merge_iter()
1562 		arg->cnt_all.curbytes += tdata->cnt_summed.curbytes;  in prof_tdata_merge_iter()
1564 			arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs;  in prof_tdata_merge_iter()
1565 			arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes;  in prof_tdata_merge_iter()
1568 		tdata->dumping = false;  in prof_tdata_merge_iter()
1570 	malloc_mutex_unlock(arg->tsdn, tdata->lock);  in prof_tdata_merge_iter()
1576 prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,  in prof_tdata_dump_iter()  argument
1580 	if (!tdata->dumping) {  in prof_tdata_dump_iter()
1586 	    tdata->thr_uid, tdata->cnt_summed.curobjs,  in prof_tdata_dump_iter()
1587 	    tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,  in prof_tdata_dump_iter()
1588 	    tdata->cnt_summed.accumbytes,  in prof_tdata_dump_iter()
1589 	    (tdata->thread_name != NULL) ? " " : "",  in prof_tdata_dump_iter()
1590 	    (tdata->thread_name != NULL) ? tdata->thread_name : "")) {  in prof_tdata_dump_iter()
1591 		return tdata;  in prof_tdata_dump_iter()
1823 prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata,  in prof_dump_prep()  argument
1833 	prof_enter(tsd, tdata);  in prof_dump_prep()
1861 	prof_leave(tsd, tdata);  in prof_dump_prep()
1866     bool leakcheck, prof_tdata_t *tdata,  in prof_dump_file()  argument
1911 	prof_tdata_t * tdata = prof_tdata_get(tsd, true);  in prof_dump()  local
1912 	if (tdata == NULL) {  in prof_dump()
1923 	prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,  in prof_dump()
1925 	bool err = prof_dump_file(tsd, propagate_err, filename, leakcheck, tdata,  in prof_dump()
1949 	prof_tdata_t *tdata;  in prof_cnt_all()  local
1955 	tdata = prof_tdata_get(tsd, false);  in prof_cnt_all()
1956 	if (tdata == NULL) {  in prof_cnt_all()
1972 	prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,  in prof_cnt_all()
2051 	prof_tdata_t *tdata;  in prof_idump()  local
2063 	tdata = prof_tdata_get(tsd, false);  in prof_idump()
2064 	if (tdata == NULL) {  in prof_idump()
2067 	if (tdata->enq) {  in prof_idump()
2068 		tdata->enq_idump = true;  in prof_idump()
2108 	prof_tdata_t *tdata;  in prof_gdump()  local
2120 	tdata = prof_tdata_get(tsd, false);  in prof_gdump()
2121 	if (tdata == NULL) {  in prof_gdump()
2124 	if (tdata->enq) {  in prof_gdump()
2125 		tdata->enq_gdump = true;  in prof_gdump()
2203 	prof_tdata_t *tdata;  in prof_tdata_init_impl()  local
2208 	tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),  in prof_tdata_init_impl()
2211 	if (tdata == NULL) {  in prof_tdata_init_impl()
2215 	tdata->lock = prof_tdata_mutex_choose(thr_uid);  in prof_tdata_init_impl()
2216 	tdata->thr_uid = thr_uid;  in prof_tdata_init_impl()
2217 	tdata->thr_discrim = thr_discrim;  in prof_tdata_init_impl()
2218 	tdata->thread_name = thread_name;  in prof_tdata_init_impl()
2219 	tdata->attached = true;  in prof_tdata_init_impl()
2220 	tdata->expired = false;  in prof_tdata_init_impl()
2221 	tdata->tctx_uid_next = 0;  in prof_tdata_init_impl()
2223 	if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,  in prof_tdata_init_impl()
2225 		idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);  in prof_tdata_init_impl()
2229 	tdata->prng_state = (uint64_t)(uintptr_t)tdata;  in prof_tdata_init_impl()
2230 	prof_sample_threshold_update(tdata);  in prof_tdata_init_impl()
2232 	tdata->enq = false;  in prof_tdata_init_impl()
2233 	tdata->enq_idump = false;  in prof_tdata_init_impl()
2234 	tdata->enq_gdump = false;  in prof_tdata_init_impl()
2236 	tdata->dumping = false;  in prof_tdata_init_impl()
2237 	tdata->active = active;  in prof_tdata_init_impl()
2240 	tdata_tree_insert(&tdatas, tdata);  in prof_tdata_init_impl()
2243 	return tdata;  in prof_tdata_init_impl()
2253 prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) {  in prof_tdata_should_destroy_unlocked()  argument
2254 	if (tdata->attached && !even_if_attached) {  in prof_tdata_should_destroy_unlocked()
2257 	if (ckh_count(&tdata->bt2tctx) != 0) {  in prof_tdata_should_destroy_unlocked()
2264 prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,  in prof_tdata_should_destroy()  argument
2266 	malloc_mutex_assert_owner(tsdn, tdata->lock);  in prof_tdata_should_destroy()
2268 	return prof_tdata_should_destroy_unlocked(tdata, even_if_attached);  in prof_tdata_should_destroy()
2272 prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,  in prof_tdata_destroy_locked()  argument
2276 	tdata_tree_remove(&tdatas, tdata);  in prof_tdata_destroy_locked()
2278 	assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));  in prof_tdata_destroy_locked()
2280 	if (tdata->thread_name != NULL) {  in prof_tdata_destroy_locked()
2281 		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,  in prof_tdata_destroy_locked()
2284 	ckh_delete(tsd, &tdata->bt2tctx);  in prof_tdata_destroy_locked()
2285 	idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);  in prof_tdata_destroy_locked()
2289 prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {  in prof_tdata_destroy()  argument
2291 	prof_tdata_destroy_locked(tsd, tdata, even_if_attached);  in prof_tdata_destroy()
2296 prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_tdata_detach()  argument
2299 	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);  in prof_tdata_detach()
2300 	if (tdata->attached) {  in prof_tdata_detach()
2301 		destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,  in prof_tdata_detach()
2308 			tdata->attached = false;  in prof_tdata_detach()
2314 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_tdata_detach()
2316 		prof_tdata_destroy(tsd, tdata, true);  in prof_tdata_detach()
2321 prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_tdata_reinit()  argument
2322 	uint64_t thr_uid = tdata->thr_uid;  in prof_tdata_reinit()
2323 	uint64_t thr_discrim = tdata->thr_discrim + 1;  in prof_tdata_reinit()
2324 	char *thread_name = (tdata->thread_name != NULL) ?  in prof_tdata_reinit()
2325 	    prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL;  in prof_tdata_reinit()
2326 	bool active = tdata->active;  in prof_tdata_reinit()
2328 	prof_tdata_detach(tsd, tdata);  in prof_tdata_reinit()
2334 prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) {  in prof_tdata_expire()  argument
2337 	malloc_mutex_lock(tsdn, tdata->lock);  in prof_tdata_expire()
2338 	if (!tdata->expired) {  in prof_tdata_expire()
2339 		tdata->expired = true;  in prof_tdata_expire()
2340 		destroy_tdata = tdata->attached ? false :  in prof_tdata_expire()
2341 		    prof_tdata_should_destroy(tsdn, tdata, false);  in prof_tdata_expire()
2345 	malloc_mutex_unlock(tsdn, tdata->lock);  in prof_tdata_expire()
2351 prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,  in prof_tdata_reset_iter()  argument
2355 	return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);  in prof_tdata_reset_iter()
2387 	prof_tdata_t *tdata;  in prof_tdata_cleanup()  local
2393 	tdata = tsd_prof_tdata_get(tsd);  in prof_tdata_cleanup()
2394 	if (tdata != NULL) {  in prof_tdata_cleanup()
2395 		prof_tdata_detach(tsd, tdata);  in prof_tdata_cleanup()
2784 	prof_tdata_t *tdata;  in prof_thread_name_get()  local
2786 	tdata = prof_tdata_get(tsd, true);  in prof_thread_name_get()
2787 	if (tdata == NULL) {  in prof_thread_name_get()
2790 	return (tdata->thread_name != NULL ? tdata->thread_name : "");  in prof_thread_name_get()
2818 	prof_tdata_t *tdata;  in prof_thread_name_set()  local
2822 	tdata = prof_tdata_get(tsd, true);  in prof_thread_name_set()
2823 	if (tdata == NULL) {  in prof_thread_name_set()
2843 	if (tdata->thread_name != NULL) {  in prof_thread_name_set()
2844 		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,  in prof_thread_name_set()
2846 		tdata->thread_name = NULL;  in prof_thread_name_set()
2849 		tdata->thread_name = s;  in prof_thread_name_set()
2856 	prof_tdata_t *tdata;  in prof_thread_active_get()  local
2858 	tdata = prof_tdata_get(tsd, true);  in prof_thread_active_get()
2859 	if (tdata == NULL) {  in prof_thread_active_get()
2862 	return tdata->active;  in prof_thread_active_get()
2867 	prof_tdata_t *tdata;  in prof_thread_active_set()  local
2869 	tdata = prof_tdata_get(tsd, true);  in prof_thread_active_set()
2870 	if (tdata == NULL) {  in prof_thread_active_set()
2873 	tdata->active = active;  in prof_thread_active_set()