69 tsd_in_nominal_list(tsd_t *tsd) {  in tsd_in_nominal_list()  argument
78 		if (tsd == tsd_list) {  in tsd_in_nominal_list()
88 tsd_add_nominal(tsd_t *tsd) {  in tsd_add_nominal()  argument
89 	assert(!tsd_in_nominal_list(tsd));  in tsd_add_nominal()
90 	assert(tsd_state_get(tsd) <= tsd_state_nominal_max);  in tsd_add_nominal()
91 	ql_elm_new(tsd, TSD_MANGLE(tcache).tsd_link);  in tsd_add_nominal()
92 	malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);  in tsd_add_nominal()
93 	ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link);  in tsd_add_nominal()
94 	malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);  in tsd_add_nominal()
98 tsd_remove_nominal(tsd_t *tsd) {  in tsd_remove_nominal()  argument
99 	assert(tsd_in_nominal_list(tsd));  in tsd_remove_nominal()
100 	assert(tsd_state_get(tsd) <= tsd_state_nominal_max);  in tsd_remove_nominal()
101 	malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);  in tsd_remove_nominal()
102 	ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link);  in tsd_remove_nominal()
103 	malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);  in tsd_remove_nominal()
145 tsd_local_slow(tsd_t *tsd) {  in tsd_local_slow()  argument
146 	return !tsd_tcache_enabled_get(tsd)  in tsd_local_slow()
147 	    || tsd_reentrancy_level_get(tsd) > 0;  in tsd_local_slow()
158 tsd_state_compute(tsd_t *tsd) {  in tsd_state_compute()  argument
159 	if (!tsd_nominal(tsd)) {  in tsd_state_compute()
160 		return tsd_state_get(tsd);  in tsd_state_compute()
163 	if (malloc_slow || tsd_local_slow(tsd) || tsd_global_slow()) {  in tsd_state_compute()
171 tsd_slow_update(tsd_t *tsd) {  in tsd_slow_update()  argument
174 		uint8_t new_state = tsd_state_compute(tsd);  in tsd_slow_update()
175 		old_state = tsd_atomic_exchange(&tsd->state, new_state,  in tsd_slow_update()
181 tsd_state_set(tsd_t *tsd, uint8_t new_state) {  in tsd_state_set()  argument
184 	uint8_t old_state = tsd_atomic_load(&tsd->state, ATOMIC_RELAXED);  in tsd_state_set()
190 		assert(!tsd_in_nominal_list(tsd));  in tsd_state_set()
191 		tsd_atomic_store(&tsd->state, new_state, ATOMIC_RELAXED);  in tsd_state_set()
193 			tsd_add_nominal(tsd);  in tsd_state_set()
201 		assert(tsd_in_nominal_list(tsd));  in tsd_state_set()
203 			tsd_remove_nominal(tsd);  in tsd_state_set()
204 			tsd_atomic_store(&tsd->state, new_state,  in tsd_state_set()
213 			tsd_slow_update(tsd);  in tsd_state_set()
219 tsd_data_init(tsd_t *tsd) {  in tsd_data_init()  argument
224 	rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd));  in tsd_data_init()
233 	*tsd_offset_statep_get(tsd) = config_debug ? 0 :  in tsd_data_init()
234 	    (uint64_t)(uintptr_t)tsd;  in tsd_data_init()
236 	return tsd_tcache_enabled_data_init(tsd);  in tsd_data_init()
240 assert_tsd_data_cleanup_done(tsd_t *tsd) {  in assert_tsd_data_cleanup_done()  argument
241 	assert(!tsd_nominal(tsd));  in assert_tsd_data_cleanup_done()
242 	assert(!tsd_in_nominal_list(tsd));  in assert_tsd_data_cleanup_done()
243 	assert(*tsd_arenap_get_unsafe(tsd) == NULL);  in assert_tsd_data_cleanup_done()
244 	assert(*tsd_iarenap_get_unsafe(tsd) == NULL);  in assert_tsd_data_cleanup_done()
245 	assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true);  in assert_tsd_data_cleanup_done()
246 	assert(*tsd_arenas_tdatap_get_unsafe(tsd) == NULL);  in assert_tsd_data_cleanup_done()
247 	assert(*tsd_tcache_enabledp_get_unsafe(tsd) == false);  in assert_tsd_data_cleanup_done()
248 	assert(*tsd_prof_tdatap_get_unsafe(tsd) == NULL);  in assert_tsd_data_cleanup_done()
252 tsd_data_init_nocleanup(tsd_t *tsd) {  in tsd_data_init_nocleanup()  argument
253 	assert(tsd_state_get(tsd) == tsd_state_reincarnated ||  in tsd_data_init_nocleanup()
254 	    tsd_state_get(tsd) == tsd_state_minimal_initialized);  in tsd_data_init_nocleanup()
260 	rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd));  in tsd_data_init_nocleanup()
261 	*tsd_arenas_tdata_bypassp_get(tsd) = true;  in tsd_data_init_nocleanup()
262 	*tsd_tcache_enabledp_get_unsafe(tsd) = false;  in tsd_data_init_nocleanup()
263 	*tsd_reentrancy_levelp_get(tsd) = 1;  in tsd_data_init_nocleanup()
264 	assert_tsd_data_cleanup_done(tsd);  in tsd_data_init_nocleanup()
270 tsd_fetch_slow(tsd_t *tsd, bool minimal) {  in tsd_fetch_slow()  argument
271 	assert(!tsd_fast(tsd));  in tsd_fetch_slow()
273 	if (tsd_state_get(tsd) == tsd_state_nominal_slow) {  in tsd_fetch_slow()
280 	} else if (tsd_state_get(tsd) == tsd_state_nominal_recompute) {  in tsd_fetch_slow()
281 		tsd_slow_update(tsd);  in tsd_fetch_slow()
282 	} else if (tsd_state_get(tsd) == tsd_state_uninitialized) {  in tsd_fetch_slow()
285 				tsd_state_set(tsd, tsd_state_nominal);  in tsd_fetch_slow()
286 				tsd_slow_update(tsd);  in tsd_fetch_slow()
288 				tsd_set(tsd);  in tsd_fetch_slow()
289 				tsd_data_init(tsd);  in tsd_fetch_slow()
292 			tsd_state_set(tsd, tsd_state_minimal_initialized);  in tsd_fetch_slow()
293 			tsd_set(tsd);  in tsd_fetch_slow()
294 			tsd_data_init_nocleanup(tsd);  in tsd_fetch_slow()
296 	} else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) {  in tsd_fetch_slow()
299 			tsd_state_set(tsd, tsd_state_nominal);  in tsd_fetch_slow()
300 			assert(*tsd_reentrancy_levelp_get(tsd) >= 1);  in tsd_fetch_slow()
301 			(*tsd_reentrancy_levelp_get(tsd))--;  in tsd_fetch_slow()
302 			tsd_slow_update(tsd);  in tsd_fetch_slow()
303 			tsd_data_init(tsd);  in tsd_fetch_slow()
305 			assert_tsd_data_cleanup_done(tsd);  in tsd_fetch_slow()
307 	} else if (tsd_state_get(tsd) == tsd_state_purgatory) {  in tsd_fetch_slow()
308 		tsd_state_set(tsd, tsd_state_reincarnated);  in tsd_fetch_slow()
309 		tsd_set(tsd);  in tsd_fetch_slow()
310 		tsd_data_init_nocleanup(tsd);  in tsd_fetch_slow()
312 		assert(tsd_state_get(tsd) == tsd_state_reincarnated);  in tsd_fetch_slow()
315 	return tsd;  in tsd_fetch_slow()
363 tsd_do_data_cleanup(tsd_t *tsd) {  in tsd_do_data_cleanup()  argument
364 	prof_tdata_cleanup(tsd);  in tsd_do_data_cleanup()
365 	iarena_cleanup(tsd);  in tsd_do_data_cleanup()
366 	arena_cleanup(tsd);  in tsd_do_data_cleanup()
367 	arenas_tdata_cleanup(tsd);  in tsd_do_data_cleanup()
368 	tcache_cleanup(tsd);  in tsd_do_data_cleanup()
369 	witnesses_cleanup(tsd_witness_tsdp_get_unsafe(tsd));  in tsd_do_data_cleanup()
374 	tsd_t *tsd = (tsd_t *)arg;  in tsd_cleanup()  local
376 	switch (tsd_state_get(tsd)) {  in tsd_cleanup()
389 		assert_tsd_data_cleanup_done(tsd);  in tsd_cleanup()
393 		tsd_do_data_cleanup(tsd);  in tsd_cleanup()
394 		tsd_state_set(tsd, tsd_state_purgatory);  in tsd_cleanup()
395 		tsd_set(tsd);  in tsd_cleanup()
409 	test_callback_t test_callback = *tsd_test_callbackp_get_unsafe(tsd);  in tsd_cleanup()
410 	int *data = tsd_test_datap_get_unsafe(tsd);  in tsd_cleanup()
419 	tsd_t *tsd;  in malloc_tsd_boot0()  local
429 	tsd = tsd_fetch();  in malloc_tsd_boot0()
430 	*tsd_arenas_tdata_bypassp_get(tsd) = true;  in malloc_tsd_boot0()
431 	return tsd;  in malloc_tsd_boot0()
437 	tsd_t *tsd = tsd_fetch();  in malloc_tsd_boot1()  local
439 	tsd_slow_update(tsd);  in malloc_tsd_boot1()
440 	*tsd_arenas_tdata_bypassp_get(tsd) = false;  in malloc_tsd_boot1()
517 tsd_prefork(tsd_t *tsd) {  in tsd_prefork()  argument
518 	malloc_mutex_prefork(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);  in tsd_prefork()
522 tsd_postfork_parent(tsd_t *tsd) {  in tsd_postfork_parent()  argument
523 	malloc_mutex_postfork_parent(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);  in tsd_postfork_parent()
527 tsd_postfork_child(tsd_t *tsd) {  in tsd_postfork_child()  argument
528 	malloc_mutex_postfork_child(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);  in tsd_postfork_child()
531 	if (tsd_state_get(tsd) <= tsd_state_nominal_max) {  in tsd_postfork_child()
532 		tsd_add_nominal(tsd);  in tsd_postfork_child()