xref: /freebsd/contrib/jemalloc/src/prof.c (revision b2d48be1bc7df45ddd13b143a160d0acb5a383c5)
1 #define	JEMALLOC_PROF_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3 /******************************************************************************/
4 
5 #ifdef JEMALLOC_PROF_LIBUNWIND
6 #define	UNW_LOCAL_ONLY
7 #include <libunwind.h>
8 #endif
9 
10 #ifdef JEMALLOC_PROF_LIBGCC
11 #include <unwind.h>
12 #endif
13 
14 /******************************************************************************/
15 /* Data. */
16 
17 bool		opt_prof = false;
18 bool		opt_prof_active = true;
19 bool		opt_prof_thread_active_init = true;
20 size_t		opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
21 ssize_t		opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
22 bool		opt_prof_gdump = false;
23 bool		opt_prof_final = false;
24 bool		opt_prof_leak = false;
25 bool		opt_prof_accum = false;
26 char		opt_prof_prefix[
27     /* Minimize memory bloat for non-prof builds. */
28 #ifdef JEMALLOC_PROF
29     PATH_MAX +
30 #endif
31     1];
32 
33 /*
34  * Initialized as opt_prof_active, and accessed via
35  * prof_active_[gs]et{_unlocked,}().
36  */
37 bool			prof_active;
38 static malloc_mutex_t	prof_active_mtx;
39 
40 /*
41  * Initialized as opt_prof_thread_active_init, and accessed via
42  * prof_thread_active_init_[gs]et().
43  */
44 static bool		prof_thread_active_init;
45 static malloc_mutex_t	prof_thread_active_init_mtx;
46 
47 /*
48  * Initialized as opt_prof_gdump, and accessed via
49  * prof_gdump_[gs]et{_unlocked,}().
50  */
51 bool			prof_gdump_val;
52 static malloc_mutex_t	prof_gdump_mtx;
53 
54 uint64_t	prof_interval = 0;
55 
56 size_t		lg_prof_sample;
57 
58 /*
59  * Table of mutexes that are shared among gctx's.  These are leaf locks, so
60  * there is no problem with using them for more than one gctx at the same time.
61  * The primary motivation for this sharing though is that gctx's are ephemeral,
62  * and destroying mutexes causes complications for systems that allocate when
63  * creating/destroying mutexes.
64  */
65 static malloc_mutex_t	*gctx_locks;
66 static unsigned		cum_gctxs; /* Atomic counter. */
67 
68 /*
69  * Table of mutexes that are shared among tdata's.  No operations require
70  * holding multiple tdata locks, so there is no problem with using them for more
71  * than one tdata at the same time, even though a gctx lock may be acquired
72  * while holding a tdata lock.
73  */
74 static malloc_mutex_t	*tdata_locks;
75 
76 /*
77  * Global hash of (prof_bt_t *)-->(prof_gctx_t *).  This is the master data
78  * structure that knows about all backtraces currently captured.
79  */
80 static ckh_t		bt2gctx;
81 static malloc_mutex_t	bt2gctx_mtx;
82 
83 /*
84  * Tree of all extant prof_tdata_t structures, regardless of state,
85  * {attached,detached,expired}.
86  */
87 static prof_tdata_tree_t	tdatas;
88 static malloc_mutex_t	tdatas_mtx;
89 
90 static uint64_t		next_thr_uid;
91 static malloc_mutex_t	next_thr_uid_mtx;
92 
93 static malloc_mutex_t	prof_dump_seq_mtx;
94 static uint64_t		prof_dump_seq;
95 static uint64_t		prof_dump_iseq;
96 static uint64_t		prof_dump_mseq;
97 static uint64_t		prof_dump_useq;
98 
99 /*
100  * This buffer is rather large for stack allocation, so use a single buffer for
101  * all profile dumps.
102  */
103 static malloc_mutex_t	prof_dump_mtx;
104 static char		prof_dump_buf[
105     /* Minimize memory bloat for non-prof builds. */
106 #ifdef JEMALLOC_PROF
107     PROF_DUMP_BUFSIZE
108 #else
109     1
110 #endif
111 ];
112 static unsigned		prof_dump_buf_end;
113 static int		prof_dump_fd;
114 
115 /* Do not dump any profiles until bootstrapping is complete. */
116 static bool		prof_booted = false;
117 
118 /******************************************************************************/
119 /*
120  * Function prototypes for static functions that are referenced prior to
121  * definition.
122  */
123 
124 static bool	prof_tctx_should_destroy(prof_tctx_t *tctx);
125 static void	prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
126 static bool	prof_tdata_should_destroy(prof_tdata_t *tdata,
127     bool even_if_attached);
128 static void	prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
129     bool even_if_attached);
130 static char	*prof_thread_name_alloc(tsd_t *tsd, const char *thread_name);
131 
132 /******************************************************************************/
133 /* Red-black trees. */
134 
135 JEMALLOC_INLINE_C int
136 prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b)
137 {
138 	uint64_t a_thr_uid = a->thr_uid;
139 	uint64_t b_thr_uid = b->thr_uid;
140 	int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
141 	if (ret == 0) {
142 		uint64_t a_thr_discrim = a->thr_discrim;
143 		uint64_t b_thr_discrim = b->thr_discrim;
144 		ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
145 		    b_thr_discrim);
146 		if (ret == 0) {
147 			uint64_t a_tctx_uid = a->tctx_uid;
148 			uint64_t b_tctx_uid = b->tctx_uid;
149 			ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
150 			    b_tctx_uid);
151 		}
152 	}
153 	return (ret);
154 }
155 
156 rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
157     tctx_link, prof_tctx_comp)
158 
159 JEMALLOC_INLINE_C int
160 prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b)
161 {
162 	unsigned a_len = a->bt.len;
163 	unsigned b_len = b->bt.len;
164 	unsigned comp_len = (a_len < b_len) ? a_len : b_len;
165 	int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
166 	if (ret == 0)
167 		ret = (a_len > b_len) - (a_len < b_len);
168 	return (ret);
169 }
170 
171 rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
172     prof_gctx_comp)
173 
174 JEMALLOC_INLINE_C int
175 prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b)
176 {
177 	int ret;
178 	uint64_t a_uid = a->thr_uid;
179 	uint64_t b_uid = b->thr_uid;
180 
181 	ret = ((a_uid > b_uid) - (a_uid < b_uid));
182 	if (ret == 0) {
183 		uint64_t a_discrim = a->thr_discrim;
184 		uint64_t b_discrim = b->thr_discrim;
185 
186 		ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
187 	}
188 	return (ret);
189 }
190 
191 rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
192     prof_tdata_comp)
193 
194 /******************************************************************************/
195 
196 void
197 prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
198 {
199 	prof_tdata_t *tdata;
200 
201 	cassert(config_prof);
202 
203 	if (updated) {
204 		/*
205 		 * Compute a new sample threshold.  This isn't very important in
206 		 * practice, because this function is rarely executed, so the
207 		 * potential for sample bias is minimal except in contrived
208 		 * programs.
209 		 */
210 		tdata = prof_tdata_get(tsd, true);
211 		if (tdata != NULL)
212 			prof_sample_threshold_update(tdata);
213 	}
214 
215 	if ((uintptr_t)tctx > (uintptr_t)1U) {
216 		malloc_mutex_lock(tctx->tdata->lock);
217 		tctx->prepared = false;
218 		if (prof_tctx_should_destroy(tctx))
219 			prof_tctx_destroy(tsd, tctx);
220 		else
221 			malloc_mutex_unlock(tctx->tdata->lock);
222 	}
223 }
224 
225 void
226 prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx)
227 {
228 
229 	prof_tctx_set(ptr, usize, tctx);
230 
231 	malloc_mutex_lock(tctx->tdata->lock);
232 	tctx->cnts.curobjs++;
233 	tctx->cnts.curbytes += usize;
234 	if (opt_prof_accum) {
235 		tctx->cnts.accumobjs++;
236 		tctx->cnts.accumbytes += usize;
237 	}
238 	tctx->prepared = false;
239 	malloc_mutex_unlock(tctx->tdata->lock);
240 }
241 
242 void
243 prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
244 {
245 
246 	malloc_mutex_lock(tctx->tdata->lock);
247 	assert(tctx->cnts.curobjs > 0);
248 	assert(tctx->cnts.curbytes >= usize);
249 	tctx->cnts.curobjs--;
250 	tctx->cnts.curbytes -= usize;
251 
252 	if (prof_tctx_should_destroy(tctx))
253 		prof_tctx_destroy(tsd, tctx);
254 	else
255 		malloc_mutex_unlock(tctx->tdata->lock);
256 }
257 
258 void
259 bt_init(prof_bt_t *bt, void **vec)
260 {
261 
262 	cassert(config_prof);
263 
264 	bt->vec = vec;
265 	bt->len = 0;
266 }
267 
268 JEMALLOC_INLINE_C void
269 prof_enter(tsd_t *tsd, prof_tdata_t *tdata)
270 {
271 
272 	cassert(config_prof);
273 	assert(tdata == prof_tdata_get(tsd, false));
274 
275 	if (tdata != NULL) {
276 		assert(!tdata->enq);
277 		tdata->enq = true;
278 	}
279 
280 	malloc_mutex_lock(&bt2gctx_mtx);
281 }
282 
283 JEMALLOC_INLINE_C void
284 prof_leave(tsd_t *tsd, prof_tdata_t *tdata)
285 {
286 
287 	cassert(config_prof);
288 	assert(tdata == prof_tdata_get(tsd, false));
289 
290 	malloc_mutex_unlock(&bt2gctx_mtx);
291 
292 	if (tdata != NULL) {
293 		bool idump, gdump;
294 
295 		assert(tdata->enq);
296 		tdata->enq = false;
297 		idump = tdata->enq_idump;
298 		tdata->enq_idump = false;
299 		gdump = tdata->enq_gdump;
300 		tdata->enq_gdump = false;
301 
302 		if (idump)
303 			prof_idump();
304 		if (gdump)
305 			prof_gdump();
306 	}
307 }
308 
309 #ifdef JEMALLOC_PROF_LIBUNWIND
310 void
311 prof_backtrace(prof_bt_t *bt)
312 {
313 	int nframes;
314 
315 	cassert(config_prof);
316 	assert(bt->len == 0);
317 	assert(bt->vec != NULL);
318 
319 	nframes = unw_backtrace(bt->vec, PROF_BT_MAX);
320 	if (nframes <= 0)
321 		return;
322 	bt->len = nframes;
323 }
324 #elif (defined(JEMALLOC_PROF_LIBGCC))
325 static _Unwind_Reason_Code
326 prof_unwind_init_callback(struct _Unwind_Context *context, void *arg)
327 {
328 
329 	cassert(config_prof);
330 
331 	return (_URC_NO_REASON);
332 }
333 
334 static _Unwind_Reason_Code
335 prof_unwind_callback(struct _Unwind_Context *context, void *arg)
336 {
337 	prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
338 	void *ip;
339 
340 	cassert(config_prof);
341 
342 	ip = (void *)_Unwind_GetIP(context);
343 	if (ip == NULL)
344 		return (_URC_END_OF_STACK);
345 	data->bt->vec[data->bt->len] = ip;
346 	data->bt->len++;
347 	if (data->bt->len == data->max)
348 		return (_URC_END_OF_STACK);
349 
350 	return (_URC_NO_REASON);
351 }
352 
353 void
354 prof_backtrace(prof_bt_t *bt)
355 {
356 	prof_unwind_data_t data = {bt, PROF_BT_MAX};
357 
358 	cassert(config_prof);
359 
360 	_Unwind_Backtrace(prof_unwind_callback, &data);
361 }
362 #elif (defined(JEMALLOC_PROF_GCC))
363 void
364 prof_backtrace(prof_bt_t *bt)
365 {
366 #define	BT_FRAME(i)							\
367 	if ((i) < PROF_BT_MAX) {					\
368 		void *p;						\
369 		if (__builtin_frame_address(i) == 0)			\
370 			return;						\
371 		p = __builtin_return_address(i);			\
372 		if (p == NULL)						\
373 			return;						\
374 		bt->vec[(i)] = p;					\
375 		bt->len = (i) + 1;					\
376 	} else								\
377 		return;
378 
379 	cassert(config_prof);
380 
381 	BT_FRAME(0)
382 	BT_FRAME(1)
383 	BT_FRAME(2)
384 	BT_FRAME(3)
385 	BT_FRAME(4)
386 	BT_FRAME(5)
387 	BT_FRAME(6)
388 	BT_FRAME(7)
389 	BT_FRAME(8)
390 	BT_FRAME(9)
391 
392 	BT_FRAME(10)
393 	BT_FRAME(11)
394 	BT_FRAME(12)
395 	BT_FRAME(13)
396 	BT_FRAME(14)
397 	BT_FRAME(15)
398 	BT_FRAME(16)
399 	BT_FRAME(17)
400 	BT_FRAME(18)
401 	BT_FRAME(19)
402 
403 	BT_FRAME(20)
404 	BT_FRAME(21)
405 	BT_FRAME(22)
406 	BT_FRAME(23)
407 	BT_FRAME(24)
408 	BT_FRAME(25)
409 	BT_FRAME(26)
410 	BT_FRAME(27)
411 	BT_FRAME(28)
412 	BT_FRAME(29)
413 
414 	BT_FRAME(30)
415 	BT_FRAME(31)
416 	BT_FRAME(32)
417 	BT_FRAME(33)
418 	BT_FRAME(34)
419 	BT_FRAME(35)
420 	BT_FRAME(36)
421 	BT_FRAME(37)
422 	BT_FRAME(38)
423 	BT_FRAME(39)
424 
425 	BT_FRAME(40)
426 	BT_FRAME(41)
427 	BT_FRAME(42)
428 	BT_FRAME(43)
429 	BT_FRAME(44)
430 	BT_FRAME(45)
431 	BT_FRAME(46)
432 	BT_FRAME(47)
433 	BT_FRAME(48)
434 	BT_FRAME(49)
435 
436 	BT_FRAME(50)
437 	BT_FRAME(51)
438 	BT_FRAME(52)
439 	BT_FRAME(53)
440 	BT_FRAME(54)
441 	BT_FRAME(55)
442 	BT_FRAME(56)
443 	BT_FRAME(57)
444 	BT_FRAME(58)
445 	BT_FRAME(59)
446 
447 	BT_FRAME(60)
448 	BT_FRAME(61)
449 	BT_FRAME(62)
450 	BT_FRAME(63)
451 	BT_FRAME(64)
452 	BT_FRAME(65)
453 	BT_FRAME(66)
454 	BT_FRAME(67)
455 	BT_FRAME(68)
456 	BT_FRAME(69)
457 
458 	BT_FRAME(70)
459 	BT_FRAME(71)
460 	BT_FRAME(72)
461 	BT_FRAME(73)
462 	BT_FRAME(74)
463 	BT_FRAME(75)
464 	BT_FRAME(76)
465 	BT_FRAME(77)
466 	BT_FRAME(78)
467 	BT_FRAME(79)
468 
469 	BT_FRAME(80)
470 	BT_FRAME(81)
471 	BT_FRAME(82)
472 	BT_FRAME(83)
473 	BT_FRAME(84)
474 	BT_FRAME(85)
475 	BT_FRAME(86)
476 	BT_FRAME(87)
477 	BT_FRAME(88)
478 	BT_FRAME(89)
479 
480 	BT_FRAME(90)
481 	BT_FRAME(91)
482 	BT_FRAME(92)
483 	BT_FRAME(93)
484 	BT_FRAME(94)
485 	BT_FRAME(95)
486 	BT_FRAME(96)
487 	BT_FRAME(97)
488 	BT_FRAME(98)
489 	BT_FRAME(99)
490 
491 	BT_FRAME(100)
492 	BT_FRAME(101)
493 	BT_FRAME(102)
494 	BT_FRAME(103)
495 	BT_FRAME(104)
496 	BT_FRAME(105)
497 	BT_FRAME(106)
498 	BT_FRAME(107)
499 	BT_FRAME(108)
500 	BT_FRAME(109)
501 
502 	BT_FRAME(110)
503 	BT_FRAME(111)
504 	BT_FRAME(112)
505 	BT_FRAME(113)
506 	BT_FRAME(114)
507 	BT_FRAME(115)
508 	BT_FRAME(116)
509 	BT_FRAME(117)
510 	BT_FRAME(118)
511 	BT_FRAME(119)
512 
513 	BT_FRAME(120)
514 	BT_FRAME(121)
515 	BT_FRAME(122)
516 	BT_FRAME(123)
517 	BT_FRAME(124)
518 	BT_FRAME(125)
519 	BT_FRAME(126)
520 	BT_FRAME(127)
521 #undef BT_FRAME
522 }
523 #else
524 void
525 prof_backtrace(prof_bt_t *bt)
526 {
527 
528 	cassert(config_prof);
529 	not_reached();
530 }
531 #endif
532 
533 static malloc_mutex_t *
534 prof_gctx_mutex_choose(void)
535 {
536 	unsigned ngctxs = atomic_add_u(&cum_gctxs, 1);
537 
538 	return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]);
539 }
540 
541 static malloc_mutex_t *
542 prof_tdata_mutex_choose(uint64_t thr_uid)
543 {
544 
545 	return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]);
546 }
547 
548 static prof_gctx_t *
549 prof_gctx_create(tsd_t *tsd, prof_bt_t *bt)
550 {
551 	/*
552 	 * Create a single allocation that has space for vec of length bt->len.
553 	 */
554 	prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsd, offsetof(prof_gctx_t,
555 	    vec) + (bt->len * sizeof(void *)), false, tcache_get(tsd, true),
556 	    true, NULL);
557 	if (gctx == NULL)
558 		return (NULL);
559 	gctx->lock = prof_gctx_mutex_choose();
560 	/*
561 	 * Set nlimbo to 1, in order to avoid a race condition with
562 	 * prof_tctx_destroy()/prof_gctx_try_destroy().
563 	 */
564 	gctx->nlimbo = 1;
565 	tctx_tree_new(&gctx->tctxs);
566 	/* Duplicate bt. */
567 	memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
568 	gctx->bt.vec = gctx->vec;
569 	gctx->bt.len = bt->len;
570 	return (gctx);
571 }
572 
573 static void
574 prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
575     prof_tdata_t *tdata)
576 {
577 
578 	cassert(config_prof);
579 
580 	/*
581 	 * Check that gctx is still unused by any thread cache before destroying
582 	 * it.  prof_lookup() increments gctx->nlimbo in order to avoid a race
583 	 * condition with this function, as does prof_tctx_destroy() in order to
584 	 * avoid a race between the main body of prof_tctx_destroy() and entry
585 	 * into this function.
586 	 */
587 	prof_enter(tsd, tdata_self);
588 	malloc_mutex_lock(gctx->lock);
589 	assert(gctx->nlimbo != 0);
590 	if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
591 		/* Remove gctx from bt2gctx. */
592 		if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL))
593 			not_reached();
594 		prof_leave(tsd, tdata_self);
595 		/* Destroy gctx. */
596 		malloc_mutex_unlock(gctx->lock);
597 		idalloctm(tsd, gctx, tcache_get(tsd, false), true);
598 	} else {
599 		/*
600 		 * Compensate for increment in prof_tctx_destroy() or
601 		 * prof_lookup().
602 		 */
603 		gctx->nlimbo--;
604 		malloc_mutex_unlock(gctx->lock);
605 		prof_leave(tsd, tdata_self);
606 	}
607 }
608 
609 /* tctx->tdata->lock must be held. */
610 static bool
611 prof_tctx_should_destroy(prof_tctx_t *tctx)
612 {
613 
614 	if (opt_prof_accum)
615 		return (false);
616 	if (tctx->cnts.curobjs != 0)
617 		return (false);
618 	if (tctx->prepared)
619 		return (false);
620 	return (true);
621 }
622 
623 static bool
624 prof_gctx_should_destroy(prof_gctx_t *gctx)
625 {
626 
627 	if (opt_prof_accum)
628 		return (false);
629 	if (!tctx_tree_empty(&gctx->tctxs))
630 		return (false);
631 	if (gctx->nlimbo != 0)
632 		return (false);
633 	return (true);
634 }
635 
636 /* tctx->tdata->lock is held upon entry, and released before return. */
637 static void
638 prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
639 {
640 	prof_tdata_t *tdata = tctx->tdata;
641 	prof_gctx_t *gctx = tctx->gctx;
642 	bool destroy_tdata, destroy_tctx, destroy_gctx;
643 
644 	assert(tctx->cnts.curobjs == 0);
645 	assert(tctx->cnts.curbytes == 0);
646 	assert(!opt_prof_accum);
647 	assert(tctx->cnts.accumobjs == 0);
648 	assert(tctx->cnts.accumbytes == 0);
649 
650 	ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
651 	destroy_tdata = prof_tdata_should_destroy(tdata, false);
652 	malloc_mutex_unlock(tdata->lock);
653 
654 	malloc_mutex_lock(gctx->lock);
655 	switch (tctx->state) {
656 	case prof_tctx_state_nominal:
657 		tctx_tree_remove(&gctx->tctxs, tctx);
658 		destroy_tctx = true;
659 		if (prof_gctx_should_destroy(gctx)) {
660 			/*
661 			 * Increment gctx->nlimbo in order to keep another
662 			 * thread from winning the race to destroy gctx while
663 			 * this one has gctx->lock dropped.  Without this, it
664 			 * would be possible for another thread to:
665 			 *
666 			 * 1) Sample an allocation associated with gctx.
667 			 * 2) Deallocate the sampled object.
668 			 * 3) Successfully prof_gctx_try_destroy(gctx).
669 			 *
670 			 * The result would be that gctx no longer exists by the
671 			 * time this thread accesses it in
672 			 * prof_gctx_try_destroy().
673 			 */
674 			gctx->nlimbo++;
675 			destroy_gctx = true;
676 		} else
677 			destroy_gctx = false;
678 		break;
679 	case prof_tctx_state_dumping:
680 		/*
681 		 * A dumping thread needs tctx to remain valid until dumping
682 		 * has finished.  Change state such that the dumping thread will
683 		 * complete destruction during a late dump iteration phase.
684 		 */
685 		tctx->state = prof_tctx_state_purgatory;
686 		destroy_tctx = false;
687 		destroy_gctx = false;
688 		break;
689 	default:
690 		not_reached();
691 		destroy_tctx = false;
692 		destroy_gctx = false;
693 	}
694 	malloc_mutex_unlock(gctx->lock);
695 	if (destroy_gctx) {
696 		prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,
697 		    tdata);
698 	}
699 
700 	if (destroy_tdata)
701 		prof_tdata_destroy(tsd, tdata, false);
702 
703 	if (destroy_tctx)
704 		idalloctm(tsd, tctx, tcache_get(tsd, false), true);
705 }
706 
707 static bool
708 prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
709     void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx)
710 {
711 	union {
712 		prof_gctx_t	*p;
713 		void		*v;
714 	} gctx;
715 	union {
716 		prof_bt_t	*p;
717 		void		*v;
718 	} btkey;
719 	bool new_gctx;
720 
721 	prof_enter(tsd, tdata);
722 	if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
723 		/* bt has never been seen before.  Insert it. */
724 		gctx.p = prof_gctx_create(tsd, bt);
725 		if (gctx.v == NULL) {
726 			prof_leave(tsd, tdata);
727 			return (true);
728 		}
729 		btkey.p = &gctx.p->bt;
730 		if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
731 			/* OOM. */
732 			prof_leave(tsd, tdata);
733 			idalloctm(tsd, gctx.v, tcache_get(tsd, false), true);
734 			return (true);
735 		}
736 		new_gctx = true;
737 	} else {
738 		/*
739 		 * Increment nlimbo, in order to avoid a race condition with
740 		 * prof_tctx_destroy()/prof_gctx_try_destroy().
741 		 */
742 		malloc_mutex_lock(gctx.p->lock);
743 		gctx.p->nlimbo++;
744 		malloc_mutex_unlock(gctx.p->lock);
745 		new_gctx = false;
746 	}
747 	prof_leave(tsd, tdata);
748 
749 	*p_btkey = btkey.v;
750 	*p_gctx = gctx.p;
751 	*p_new_gctx = new_gctx;
752 	return (false);
753 }
754 
755 prof_tctx_t *
756 prof_lookup(tsd_t *tsd, prof_bt_t *bt)
757 {
758 	union {
759 		prof_tctx_t	*p;
760 		void		*v;
761 	} ret;
762 	prof_tdata_t *tdata;
763 	bool not_found;
764 
765 	cassert(config_prof);
766 
767 	tdata = prof_tdata_get(tsd, false);
768 	if (tdata == NULL)
769 		return (NULL);
770 
771 	malloc_mutex_lock(tdata->lock);
772 	not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
773 	if (!not_found) /* Note double negative! */
774 		ret.p->prepared = true;
775 	malloc_mutex_unlock(tdata->lock);
776 	if (not_found) {
777 		tcache_t *tcache;
778 		void *btkey;
779 		prof_gctx_t *gctx;
780 		bool new_gctx, error;
781 
782 		/*
783 		 * This thread's cache lacks bt.  Look for it in the global
784 		 * cache.
785 		 */
786 		if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
787 		    &new_gctx))
788 			return (NULL);
789 
790 		/* Link a prof_tctx_t into gctx for this thread. */
791 		tcache = tcache_get(tsd, true);
792 		ret.v = iallocztm(tsd, sizeof(prof_tctx_t), false, tcache, true,
793 		    NULL);
794 		if (ret.p == NULL) {
795 			if (new_gctx)
796 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
797 			return (NULL);
798 		}
799 		ret.p->tdata = tdata;
800 		ret.p->thr_uid = tdata->thr_uid;
801 		ret.p->thr_discrim = tdata->thr_discrim;
802 		memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
803 		ret.p->gctx = gctx;
804 		ret.p->tctx_uid = tdata->tctx_uid_next++;
805 		ret.p->prepared = true;
806 		ret.p->state = prof_tctx_state_initializing;
807 		malloc_mutex_lock(tdata->lock);
808 		error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
809 		malloc_mutex_unlock(tdata->lock);
810 		if (error) {
811 			if (new_gctx)
812 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
813 			idalloctm(tsd, ret.v, tcache, true);
814 			return (NULL);
815 		}
816 		malloc_mutex_lock(gctx->lock);
817 		ret.p->state = prof_tctx_state_nominal;
818 		tctx_tree_insert(&gctx->tctxs, ret.p);
819 		gctx->nlimbo--;
820 		malloc_mutex_unlock(gctx->lock);
821 	}
822 
823 	return (ret.p);
824 }
825 
826 void
827 prof_sample_threshold_update(prof_tdata_t *tdata)
828 {
829 	/*
830 	 * The body of this function is compiled out unless heap profiling is
831 	 * enabled, so that it is possible to compile jemalloc with floating
832 	 * point support completely disabled.  Avoiding floating point code is
833 	 * important on memory-constrained systems, but it also enables a
834 	 * workaround for versions of glibc that don't properly save/restore
835 	 * floating point registers during dynamic lazy symbol loading (which
836 	 * internally calls into whatever malloc implementation happens to be
837 	 * integrated into the application).  Note that some compilers (e.g.
838 	 * gcc 4.8) may use floating point registers for fast memory moves, so
839 	 * jemalloc must be compiled with such optimizations disabled (e.g.
840 	 * -mno-sse) in order for the workaround to be complete.
841 	 */
842 #ifdef JEMALLOC_PROF
843 	uint64_t r;
844 	double u;
845 
846 	if (!config_prof)
847 		return;
848 
849 	if (lg_prof_sample == 0) {
850 		tdata->bytes_until_sample = 0;
851 		return;
852 	}
853 
854 	/*
855 	 * Compute sample interval as a geometrically distributed random
856 	 * variable with mean (2^lg_prof_sample).
857 	 *
858 	 *                             __        __
859 	 *                             |  log(u)  |                     1
860 	 * tdata->bytes_until_sample = | -------- |, where p = ---------------
861 	 *                             | log(1-p) |             lg_prof_sample
862 	 *                                                     2
863 	 *
864 	 * For more information on the math, see:
865 	 *
866 	 *   Non-Uniform Random Variate Generation
867 	 *   Luc Devroye
868 	 *   Springer-Verlag, New York, 1986
869 	 *   pp 500
870 	 *   (http://luc.devroye.org/rnbookindex.html)
871 	 */
872 	prng64(r, 53, tdata->prng_state, UINT64_C(6364136223846793005),
873 	    UINT64_C(1442695040888963407));
874 	u = (double)r * (1.0/9007199254740992.0L);
875 	tdata->bytes_until_sample = (uint64_t)(log(u) /
876 	    log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
877 	    + (uint64_t)1U;
878 #endif
879 }
880 
881 #ifdef JEMALLOC_JET
882 static prof_tdata_t *
883 prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
884 {
885 	size_t *tdata_count = (size_t *)arg;
886 
887 	(*tdata_count)++;
888 
889 	return (NULL);
890 }
891 
892 size_t
893 prof_tdata_count(void)
894 {
895 	size_t tdata_count = 0;
896 
897 	malloc_mutex_lock(&tdatas_mtx);
898 	tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
899 	    (void *)&tdata_count);
900 	malloc_mutex_unlock(&tdatas_mtx);
901 
902 	return (tdata_count);
903 }
904 #endif
905 
906 #ifdef JEMALLOC_JET
907 size_t
908 prof_bt_count(void)
909 {
910 	size_t bt_count;
911 	tsd_t *tsd;
912 	prof_tdata_t *tdata;
913 
914 	tsd = tsd_fetch();
915 	tdata = prof_tdata_get(tsd, false);
916 	if (tdata == NULL)
917 		return (0);
918 
919 	malloc_mutex_lock(&bt2gctx_mtx);
920 	bt_count = ckh_count(&bt2gctx);
921 	malloc_mutex_unlock(&bt2gctx_mtx);
922 
923 	return (bt_count);
924 }
925 #endif
926 
927 #ifdef JEMALLOC_JET
928 #undef prof_dump_open
929 #define	prof_dump_open JEMALLOC_N(prof_dump_open_impl)
930 #endif
931 static int
932 prof_dump_open(bool propagate_err, const char *filename)
933 {
934 	int fd;
935 
936 	fd = creat(filename, 0644);
937 	if (fd == -1 && !propagate_err) {
938 		malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n",
939 		    filename);
940 		if (opt_abort)
941 			abort();
942 	}
943 
944 	return (fd);
945 }
946 #ifdef JEMALLOC_JET
947 #undef prof_dump_open
948 #define	prof_dump_open JEMALLOC_N(prof_dump_open)
949 prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl);
950 #endif
951 
952 static bool
953 prof_dump_flush(bool propagate_err)
954 {
955 	bool ret = false;
956 	ssize_t err;
957 
958 	cassert(config_prof);
959 
960 	err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
961 	if (err == -1) {
962 		if (!propagate_err) {
963 			malloc_write("<jemalloc>: write() failed during heap "
964 			    "profile flush\n");
965 			if (opt_abort)
966 				abort();
967 		}
968 		ret = true;
969 	}
970 	prof_dump_buf_end = 0;
971 
972 	return (ret);
973 }
974 
975 static bool
976 prof_dump_close(bool propagate_err)
977 {
978 	bool ret;
979 
980 	assert(prof_dump_fd != -1);
981 	ret = prof_dump_flush(propagate_err);
982 	close(prof_dump_fd);
983 	prof_dump_fd = -1;
984 
985 	return (ret);
986 }
987 
988 static bool
989 prof_dump_write(bool propagate_err, const char *s)
990 {
991 	unsigned i, slen, n;
992 
993 	cassert(config_prof);
994 
995 	i = 0;
996 	slen = strlen(s);
997 	while (i < slen) {
998 		/* Flush the buffer if it is full. */
999 		if (prof_dump_buf_end == PROF_DUMP_BUFSIZE)
1000 			if (prof_dump_flush(propagate_err) && propagate_err)
1001 				return (true);
1002 
1003 		if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
1004 			/* Finish writing. */
1005 			n = slen - i;
1006 		} else {
1007 			/* Write as much of s as will fit. */
1008 			n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
1009 		}
1010 		memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
1011 		prof_dump_buf_end += n;
1012 		i += n;
1013 	}
1014 
1015 	return (false);
1016 }
1017 
1018 JEMALLOC_FORMAT_PRINTF(2, 3)
1019 static bool
1020 prof_dump_printf(bool propagate_err, const char *format, ...)
1021 {
1022 	bool ret;
1023 	va_list ap;
1024 	char buf[PROF_PRINTF_BUFSIZE];
1025 
1026 	va_start(ap, format);
1027 	malloc_vsnprintf(buf, sizeof(buf), format, ap);
1028 	va_end(ap);
1029 	ret = prof_dump_write(propagate_err, buf);
1030 
1031 	return (ret);
1032 }
1033 
1034 /* tctx->tdata->lock is held. */
1035 static void
1036 prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata)
1037 {
1038 
1039 	malloc_mutex_lock(tctx->gctx->lock);
1040 
1041 	switch (tctx->state) {
1042 	case prof_tctx_state_initializing:
1043 		malloc_mutex_unlock(tctx->gctx->lock);
1044 		return;
1045 	case prof_tctx_state_nominal:
1046 		tctx->state = prof_tctx_state_dumping;
1047 		malloc_mutex_unlock(tctx->gctx->lock);
1048 
1049 		memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
1050 
1051 		tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1052 		tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1053 		if (opt_prof_accum) {
1054 			tdata->cnt_summed.accumobjs +=
1055 			    tctx->dump_cnts.accumobjs;
1056 			tdata->cnt_summed.accumbytes +=
1057 			    tctx->dump_cnts.accumbytes;
1058 		}
1059 		break;
1060 	case prof_tctx_state_dumping:
1061 	case prof_tctx_state_purgatory:
1062 		not_reached();
1063 	}
1064 }
1065 
1066 /* gctx->lock is held. */
1067 static void
1068 prof_tctx_merge_gctx(prof_tctx_t *tctx, prof_gctx_t *gctx)
1069 {
1070 
1071 	gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1072 	gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1073 	if (opt_prof_accum) {
1074 		gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
1075 		gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
1076 	}
1077 }
1078 
1079 /* tctx->gctx is held. */
1080 static prof_tctx_t *
1081 prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1082 {
1083 
1084 	switch (tctx->state) {
1085 	case prof_tctx_state_nominal:
1086 		/* New since dumping started; ignore. */
1087 		break;
1088 	case prof_tctx_state_dumping:
1089 	case prof_tctx_state_purgatory:
1090 		prof_tctx_merge_gctx(tctx, tctx->gctx);
1091 		break;
1092 	default:
1093 		not_reached();
1094 	}
1095 
1096 	return (NULL);
1097 }
1098 
1099 /* gctx->lock is held. */
1100 static prof_tctx_t *
1101 prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1102 {
1103 	bool propagate_err = *(bool *)arg;
1104 
1105 	if (prof_dump_printf(propagate_err,
1106 	    "  t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1107 	    tctx->thr_uid, tctx->dump_cnts.curobjs, tctx->dump_cnts.curbytes,
1108 	    tctx->dump_cnts.accumobjs, tctx->dump_cnts.accumbytes))
1109 		return (tctx);
1110 	return (NULL);
1111 }
1112 
1113 /* tctx->gctx is held. */
1114 static prof_tctx_t *
1115 prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1116 {
1117 	prof_tctx_t *ret;
1118 
1119 	switch (tctx->state) {
1120 	case prof_tctx_state_nominal:
1121 		/* New since dumping started; ignore. */
1122 		break;
1123 	case prof_tctx_state_dumping:
1124 		tctx->state = prof_tctx_state_nominal;
1125 		break;
1126 	case prof_tctx_state_purgatory:
1127 		ret = tctx;
1128 		goto label_return;
1129 	default:
1130 		not_reached();
1131 	}
1132 
1133 	ret = NULL;
1134 label_return:
1135 	return (ret);
1136 }
1137 
1138 static void
1139 prof_dump_gctx_prep(prof_gctx_t *gctx, prof_gctx_tree_t *gctxs)
1140 {
1141 
1142 	cassert(config_prof);
1143 
1144 	malloc_mutex_lock(gctx->lock);
1145 
1146 	/*
1147 	 * Increment nlimbo so that gctx won't go away before dump.
1148 	 * Additionally, link gctx into the dump list so that it is included in
1149 	 * prof_dump()'s second pass.
1150 	 */
1151 	gctx->nlimbo++;
1152 	gctx_tree_insert(gctxs, gctx);
1153 
1154 	memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
1155 
1156 	malloc_mutex_unlock(gctx->lock);
1157 }
1158 
1159 static prof_gctx_t *
1160 prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
1161 {
1162 	size_t *leak_ngctx = (size_t *)arg;
1163 
1164 	malloc_mutex_lock(gctx->lock);
1165 	tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, NULL);
1166 	if (gctx->cnt_summed.curobjs != 0)
1167 		(*leak_ngctx)++;
1168 	malloc_mutex_unlock(gctx->lock);
1169 
1170 	return (NULL);
1171 }
1172 
1173 static void
1174 prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
1175 {
1176 	prof_tdata_t *tdata = prof_tdata_get(tsd, false);
1177 	prof_gctx_t *gctx;
1178 
1179 	/*
1180 	 * Standard tree iteration won't work here, because as soon as we
1181 	 * decrement gctx->nlimbo and unlock gctx, another thread can
1182 	 * concurrently destroy it, which will corrupt the tree.  Therefore,
1183 	 * tear down the tree one node at a time during iteration.
1184 	 */
1185 	while ((gctx = gctx_tree_first(gctxs)) != NULL) {
1186 		gctx_tree_remove(gctxs, gctx);
1187 		malloc_mutex_lock(gctx->lock);
1188 		{
1189 			prof_tctx_t *next;
1190 
1191 			next = NULL;
1192 			do {
1193 				prof_tctx_t *to_destroy =
1194 				    tctx_tree_iter(&gctx->tctxs, next,
1195 				    prof_tctx_finish_iter, NULL);
1196 				if (to_destroy != NULL) {
1197 					next = tctx_tree_next(&gctx->tctxs,
1198 					    to_destroy);
1199 					tctx_tree_remove(&gctx->tctxs,
1200 					    to_destroy);
1201 					idalloctm(tsd, to_destroy,
1202 					    tcache_get(tsd, false), true);
1203 				} else
1204 					next = NULL;
1205 			} while (next != NULL);
1206 		}
1207 		gctx->nlimbo--;
1208 		if (prof_gctx_should_destroy(gctx)) {
1209 			gctx->nlimbo++;
1210 			malloc_mutex_unlock(gctx->lock);
1211 			prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
1212 		} else
1213 			malloc_mutex_unlock(gctx->lock);
1214 	}
1215 }
1216 
1217 static prof_tdata_t *
1218 prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1219 {
1220 	prof_cnt_t *cnt_all = (prof_cnt_t *)arg;
1221 
1222 	malloc_mutex_lock(tdata->lock);
1223 	if (!tdata->expired) {
1224 		size_t tabind;
1225 		union {
1226 			prof_tctx_t	*p;
1227 			void		*v;
1228 		} tctx;
1229 
1230 		tdata->dumping = true;
1231 		memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
1232 		for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
1233 		    &tctx.v);)
1234 			prof_tctx_merge_tdata(tctx.p, tdata);
1235 
1236 		cnt_all->curobjs += tdata->cnt_summed.curobjs;
1237 		cnt_all->curbytes += tdata->cnt_summed.curbytes;
1238 		if (opt_prof_accum) {
1239 			cnt_all->accumobjs += tdata->cnt_summed.accumobjs;
1240 			cnt_all->accumbytes += tdata->cnt_summed.accumbytes;
1241 		}
1242 	} else
1243 		tdata->dumping = false;
1244 	malloc_mutex_unlock(tdata->lock);
1245 
1246 	return (NULL);
1247 }
1248 
1249 static prof_tdata_t *
1250 prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1251 {
1252 	bool propagate_err = *(bool *)arg;
1253 
1254 	if (!tdata->dumping)
1255 		return (NULL);
1256 
1257 	if (prof_dump_printf(propagate_err,
1258 	    "  t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n",
1259 	    tdata->thr_uid, tdata->cnt_summed.curobjs,
1260 	    tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,
1261 	    tdata->cnt_summed.accumbytes,
1262 	    (tdata->thread_name != NULL) ? " " : "",
1263 	    (tdata->thread_name != NULL) ? tdata->thread_name : ""))
1264 		return (tdata);
1265 	return (NULL);
1266 }
1267 
1268 #ifdef JEMALLOC_JET
1269 #undef prof_dump_header
1270 #define	prof_dump_header JEMALLOC_N(prof_dump_header_impl)
1271 #endif
1272 static bool
1273 prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all)
1274 {
1275 	bool ret;
1276 
1277 	if (prof_dump_printf(propagate_err,
1278 	    "heap_v2/%"FMTu64"\n"
1279 	    "  t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1280 	    ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs,
1281 	    cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes))
1282 		return (true);
1283 
1284 	malloc_mutex_lock(&tdatas_mtx);
1285 	ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
1286 	    (void *)&propagate_err) != NULL);
1287 	malloc_mutex_unlock(&tdatas_mtx);
1288 	return (ret);
1289 }
1290 #ifdef JEMALLOC_JET
1291 #undef prof_dump_header
1292 #define	prof_dump_header JEMALLOC_N(prof_dump_header)
1293 prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl);
1294 #endif
1295 
1296 /* gctx->lock is held. */
1297 static bool
1298 prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt,
1299     prof_gctx_tree_t *gctxs)
1300 {
1301 	bool ret;
1302 	unsigned i;
1303 
1304 	cassert(config_prof);
1305 
1306 	/* Avoid dumping such gctx's that have no useful data. */
1307 	if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
1308 	    (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
1309 		assert(gctx->cnt_summed.curobjs == 0);
1310 		assert(gctx->cnt_summed.curbytes == 0);
1311 		assert(gctx->cnt_summed.accumobjs == 0);
1312 		assert(gctx->cnt_summed.accumbytes == 0);
1313 		ret = false;
1314 		goto label_return;
1315 	}
1316 
1317 	if (prof_dump_printf(propagate_err, "@")) {
1318 		ret = true;
1319 		goto label_return;
1320 	}
1321 	for (i = 0; i < bt->len; i++) {
1322 		if (prof_dump_printf(propagate_err, " %#"FMTxPTR,
1323 		    (uintptr_t)bt->vec[i])) {
1324 			ret = true;
1325 			goto label_return;
1326 		}
1327 	}
1328 
1329 	if (prof_dump_printf(propagate_err,
1330 	    "\n"
1331 	    "  t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1332 	    gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes,
1333 	    gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) {
1334 		ret = true;
1335 		goto label_return;
1336 	}
1337 
1338 	if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
1339 	    (void *)&propagate_err) != NULL) {
1340 		ret = true;
1341 		goto label_return;
1342 	}
1343 
1344 	ret = false;
1345 label_return:
1346 	return (ret);
1347 }
1348 
1349 JEMALLOC_FORMAT_PRINTF(1, 2)
1350 static int
1351 prof_open_maps(const char *format, ...)
1352 {
1353 	int mfd;
1354 	va_list ap;
1355 	char filename[PATH_MAX + 1];
1356 
1357 	va_start(ap, format);
1358 	malloc_vsnprintf(filename, sizeof(filename), format, ap);
1359 	va_end(ap);
1360 	mfd = open(filename, O_RDONLY);
1361 
1362 	return (mfd);
1363 }
1364 
1365 static bool
1366 prof_dump_maps(bool propagate_err)
1367 {
1368 	bool ret;
1369 	int mfd;
1370 
1371 	cassert(config_prof);
1372 #ifdef __FreeBSD__
1373 	mfd = prof_open_maps("/proc/curproc/map");
1374 #else
1375 	{
1376 		int pid = getpid();
1377 
1378 		mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid);
1379 		if (mfd == -1)
1380 			mfd = prof_open_maps("/proc/%d/maps", pid);
1381 	}
1382 #endif
1383 	if (mfd != -1) {
1384 		ssize_t nread;
1385 
1386 		if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") &&
1387 		    propagate_err) {
1388 			ret = true;
1389 			goto label_return;
1390 		}
1391 		nread = 0;
1392 		do {
1393 			prof_dump_buf_end += nread;
1394 			if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
1395 				/* Make space in prof_dump_buf before read(). */
1396 				if (prof_dump_flush(propagate_err) &&
1397 				    propagate_err) {
1398 					ret = true;
1399 					goto label_return;
1400 				}
1401 			}
1402 			nread = read(mfd, &prof_dump_buf[prof_dump_buf_end],
1403 			    PROF_DUMP_BUFSIZE - prof_dump_buf_end);
1404 		} while (nread > 0);
1405 	} else {
1406 		ret = true;
1407 		goto label_return;
1408 	}
1409 
1410 	ret = false;
1411 label_return:
1412 	if (mfd != -1)
1413 		close(mfd);
1414 	return (ret);
1415 }
1416 
1417 static void
1418 prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
1419     const char *filename)
1420 {
1421 
1422 	if (cnt_all->curbytes != 0) {
1423 		malloc_printf("<jemalloc>: Leak summary: %"FMTu64" byte%s, %"
1424 		    FMTu64" object%s, %zu context%s\n",
1425 		    cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "",
1426 		    cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "",
1427 		    leak_ngctx, (leak_ngctx != 1) ? "s" : "");
1428 		malloc_printf(
1429 		    "<jemalloc>: Run jeprof on \"%s\" for leak detail\n",
1430 		    filename);
1431 	}
1432 }
1433 
1434 static prof_gctx_t *
1435 prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
1436 {
1437 	prof_gctx_t *ret;
1438 	bool propagate_err = *(bool *)arg;
1439 
1440 	malloc_mutex_lock(gctx->lock);
1441 
1442 	if (prof_dump_gctx(propagate_err, gctx, &gctx->bt, gctxs)) {
1443 		ret = gctx;
1444 		goto label_return;
1445 	}
1446 
1447 	ret = NULL;
1448 label_return:
1449 	malloc_mutex_unlock(gctx->lock);
1450 	return (ret);
1451 }
1452 
1453 static bool
1454 prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
1455 {
1456 	prof_tdata_t *tdata;
1457 	prof_cnt_t cnt_all;
1458 	size_t tabind;
1459 	union {
1460 		prof_gctx_t	*p;
1461 		void		*v;
1462 	} gctx;
1463 	size_t leak_ngctx;
1464 	prof_gctx_tree_t gctxs;
1465 
1466 	cassert(config_prof);
1467 
1468 	tdata = prof_tdata_get(tsd, true);
1469 	if (tdata == NULL)
1470 		return (true);
1471 
1472 	malloc_mutex_lock(&prof_dump_mtx);
1473 	prof_enter(tsd, tdata);
1474 
1475 	/*
1476 	 * Put gctx's in limbo and clear their counters in preparation for
1477 	 * summing.
1478 	 */
1479 	gctx_tree_new(&gctxs);
1480 	for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);)
1481 		prof_dump_gctx_prep(gctx.p, &gctxs);
1482 
1483 	/*
1484 	 * Iterate over tdatas, and for the non-expired ones snapshot their tctx
1485 	 * stats and merge them into the associated gctx's.
1486 	 */
1487 	memset(&cnt_all, 0, sizeof(prof_cnt_t));
1488 	malloc_mutex_lock(&tdatas_mtx);
1489 	tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, (void *)&cnt_all);
1490 	malloc_mutex_unlock(&tdatas_mtx);
1491 
1492 	/* Merge tctx stats into gctx's. */
1493 	leak_ngctx = 0;
1494 	gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, (void *)&leak_ngctx);
1495 
1496 	prof_leave(tsd, tdata);
1497 
1498 	/* Create dump file. */
1499 	if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1)
1500 		goto label_open_close_error;
1501 
1502 	/* Dump profile header. */
1503 	if (prof_dump_header(propagate_err, &cnt_all))
1504 		goto label_write_error;
1505 
1506 	/* Dump per gctx profile stats. */
1507 	if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter,
1508 	    (void *)&propagate_err) != NULL)
1509 		goto label_write_error;
1510 
1511 	/* Dump /proc/<pid>/maps if possible. */
1512 	if (prof_dump_maps(propagate_err))
1513 		goto label_write_error;
1514 
1515 	if (prof_dump_close(propagate_err))
1516 		goto label_open_close_error;
1517 
1518 	prof_gctx_finish(tsd, &gctxs);
1519 	malloc_mutex_unlock(&prof_dump_mtx);
1520 
1521 	if (leakcheck)
1522 		prof_leakcheck(&cnt_all, leak_ngctx, filename);
1523 
1524 	return (false);
1525 label_write_error:
1526 	prof_dump_close(propagate_err);
1527 label_open_close_error:
1528 	prof_gctx_finish(tsd, &gctxs);
1529 	malloc_mutex_unlock(&prof_dump_mtx);
1530 	return (true);
1531 }
1532 
1533 #define	DUMP_FILENAME_BUFSIZE	(PATH_MAX + 1)
1534 #define	VSEQ_INVALID		UINT64_C(0xffffffffffffffff)
1535 static void
1536 prof_dump_filename(char *filename, char v, uint64_t vseq)
1537 {
1538 
1539 	cassert(config_prof);
1540 
1541 	if (vseq != VSEQ_INVALID) {
1542 	        /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */
1543 		malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1544 		    "%s.%d.%"FMTu64".%c%"FMTu64".heap",
1545 		    opt_prof_prefix, (int)getpid(), prof_dump_seq, v, vseq);
1546 	} else {
1547 	        /* "<prefix>.<pid>.<seq>.<v>.heap" */
1548 		malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1549 		    "%s.%d.%"FMTu64".%c.heap",
1550 		    opt_prof_prefix, (int)getpid(), prof_dump_seq, v);
1551 	}
1552 	prof_dump_seq++;
1553 }
1554 
1555 static void
1556 prof_fdump(void)
1557 {
1558 	tsd_t *tsd;
1559 	char filename[DUMP_FILENAME_BUFSIZE];
1560 
1561 	cassert(config_prof);
1562 	assert(opt_prof_final);
1563 	assert(opt_prof_prefix[0] != '\0');
1564 
1565 	if (!prof_booted)
1566 		return;
1567 	tsd = tsd_fetch();
1568 
1569 	malloc_mutex_lock(&prof_dump_seq_mtx);
1570 	prof_dump_filename(filename, 'f', VSEQ_INVALID);
1571 	malloc_mutex_unlock(&prof_dump_seq_mtx);
1572 	prof_dump(tsd, false, filename, opt_prof_leak);
1573 }
1574 
1575 void
1576 prof_idump(void)
1577 {
1578 	tsd_t *tsd;
1579 	prof_tdata_t *tdata;
1580 
1581 	cassert(config_prof);
1582 
1583 	if (!prof_booted)
1584 		return;
1585 	tsd = tsd_fetch();
1586 	tdata = prof_tdata_get(tsd, false);
1587 	if (tdata == NULL)
1588 		return;
1589 	if (tdata->enq) {
1590 		tdata->enq_idump = true;
1591 		return;
1592 	}
1593 
1594 	if (opt_prof_prefix[0] != '\0') {
1595 		char filename[PATH_MAX + 1];
1596 		malloc_mutex_lock(&prof_dump_seq_mtx);
1597 		prof_dump_filename(filename, 'i', prof_dump_iseq);
1598 		prof_dump_iseq++;
1599 		malloc_mutex_unlock(&prof_dump_seq_mtx);
1600 		prof_dump(tsd, false, filename, false);
1601 	}
1602 }
1603 
1604 bool
1605 prof_mdump(const char *filename)
1606 {
1607 	tsd_t *tsd;
1608 	char filename_buf[DUMP_FILENAME_BUFSIZE];
1609 
1610 	cassert(config_prof);
1611 
1612 	if (!opt_prof || !prof_booted)
1613 		return (true);
1614 	tsd = tsd_fetch();
1615 
1616 	if (filename == NULL) {
1617 		/* No filename specified, so automatically generate one. */
1618 		if (opt_prof_prefix[0] == '\0')
1619 			return (true);
1620 		malloc_mutex_lock(&prof_dump_seq_mtx);
1621 		prof_dump_filename(filename_buf, 'm', prof_dump_mseq);
1622 		prof_dump_mseq++;
1623 		malloc_mutex_unlock(&prof_dump_seq_mtx);
1624 		filename = filename_buf;
1625 	}
1626 	return (prof_dump(tsd, true, filename, false));
1627 }
1628 
1629 void
1630 prof_gdump(void)
1631 {
1632 	tsd_t *tsd;
1633 	prof_tdata_t *tdata;
1634 
1635 	cassert(config_prof);
1636 
1637 	if (!prof_booted)
1638 		return;
1639 	tsd = tsd_fetch();
1640 	tdata = prof_tdata_get(tsd, false);
1641 	if (tdata == NULL)
1642 		return;
1643 	if (tdata->enq) {
1644 		tdata->enq_gdump = true;
1645 		return;
1646 	}
1647 
1648 	if (opt_prof_prefix[0] != '\0') {
1649 		char filename[DUMP_FILENAME_BUFSIZE];
1650 		malloc_mutex_lock(&prof_dump_seq_mtx);
1651 		prof_dump_filename(filename, 'u', prof_dump_useq);
1652 		prof_dump_useq++;
1653 		malloc_mutex_unlock(&prof_dump_seq_mtx);
1654 		prof_dump(tsd, false, filename, false);
1655 	}
1656 }
1657 
1658 static void
1659 prof_bt_hash(const void *key, size_t r_hash[2])
1660 {
1661 	prof_bt_t *bt = (prof_bt_t *)key;
1662 
1663 	cassert(config_prof);
1664 
1665 	hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
1666 }
1667 
1668 static bool
1669 prof_bt_keycomp(const void *k1, const void *k2)
1670 {
1671 	const prof_bt_t *bt1 = (prof_bt_t *)k1;
1672 	const prof_bt_t *bt2 = (prof_bt_t *)k2;
1673 
1674 	cassert(config_prof);
1675 
1676 	if (bt1->len != bt2->len)
1677 		return (false);
1678 	return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
1679 }
1680 
1681 JEMALLOC_INLINE_C uint64_t
1682 prof_thr_uid_alloc(void)
1683 {
1684 	uint64_t thr_uid;
1685 
1686 	malloc_mutex_lock(&next_thr_uid_mtx);
1687 	thr_uid = next_thr_uid;
1688 	next_thr_uid++;
1689 	malloc_mutex_unlock(&next_thr_uid_mtx);
1690 
1691 	return (thr_uid);
1692 }
1693 
1694 static prof_tdata_t *
1695 prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
1696     char *thread_name, bool active)
1697 {
1698 	prof_tdata_t *tdata;
1699 	tcache_t *tcache;
1700 
1701 	cassert(config_prof);
1702 
1703 	/* Initialize an empty cache for this thread. */
1704 	tcache = tcache_get(tsd, true);
1705 	tdata = (prof_tdata_t *)iallocztm(tsd, sizeof(prof_tdata_t), false,
1706 	    tcache, true, NULL);
1707 	if (tdata == NULL)
1708 		return (NULL);
1709 
1710 	tdata->lock = prof_tdata_mutex_choose(thr_uid);
1711 	tdata->thr_uid = thr_uid;
1712 	tdata->thr_discrim = thr_discrim;
1713 	tdata->thread_name = thread_name;
1714 	tdata->attached = true;
1715 	tdata->expired = false;
1716 	tdata->tctx_uid_next = 0;
1717 
1718 	if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS,
1719 	    prof_bt_hash, prof_bt_keycomp)) {
1720 		idalloctm(tsd, tdata, tcache, true);
1721 		return (NULL);
1722 	}
1723 
1724 	tdata->prng_state = (uint64_t)(uintptr_t)tdata;
1725 	prof_sample_threshold_update(tdata);
1726 
1727 	tdata->enq = false;
1728 	tdata->enq_idump = false;
1729 	tdata->enq_gdump = false;
1730 
1731 	tdata->dumping = false;
1732 	tdata->active = active;
1733 
1734 	malloc_mutex_lock(&tdatas_mtx);
1735 	tdata_tree_insert(&tdatas, tdata);
1736 	malloc_mutex_unlock(&tdatas_mtx);
1737 
1738 	return (tdata);
1739 }
1740 
1741 prof_tdata_t *
1742 prof_tdata_init(tsd_t *tsd)
1743 {
1744 
1745 	return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0, NULL,
1746 	    prof_thread_active_init_get()));
1747 }
1748 
1749 /* tdata->lock must be held. */
1750 static bool
1751 prof_tdata_should_destroy(prof_tdata_t *tdata, bool even_if_attached)
1752 {
1753 
1754 	if (tdata->attached && !even_if_attached)
1755 		return (false);
1756 	if (ckh_count(&tdata->bt2tctx) != 0)
1757 		return (false);
1758 	return (true);
1759 }
1760 
1761 /* tdatas_mtx must be held. */
1762 static void
1763 prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
1764     bool even_if_attached)
1765 {
1766 	tcache_t *tcache;
1767 
1768 	assert(prof_tdata_should_destroy(tdata, even_if_attached));
1769 	assert(tsd_prof_tdata_get(tsd) != tdata);
1770 
1771 	tdata_tree_remove(&tdatas, tdata);
1772 
1773 	tcache = tcache_get(tsd, false);
1774 	if (tdata->thread_name != NULL)
1775 		idalloctm(tsd, tdata->thread_name, tcache, true);
1776 	ckh_delete(tsd, &tdata->bt2tctx);
1777 	idalloctm(tsd, tdata, tcache, true);
1778 }
1779 
1780 static void
1781 prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached)
1782 {
1783 
1784 	malloc_mutex_lock(&tdatas_mtx);
1785 	prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
1786 	malloc_mutex_unlock(&tdatas_mtx);
1787 }
1788 
1789 static void
1790 prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata)
1791 {
1792 	bool destroy_tdata;
1793 
1794 	malloc_mutex_lock(tdata->lock);
1795 	if (tdata->attached) {
1796 		destroy_tdata = prof_tdata_should_destroy(tdata, true);
1797 		/*
1798 		 * Only detach if !destroy_tdata, because detaching would allow
1799 		 * another thread to win the race to destroy tdata.
1800 		 */
1801 		if (!destroy_tdata)
1802 			tdata->attached = false;
1803 		tsd_prof_tdata_set(tsd, NULL);
1804 	} else
1805 		destroy_tdata = false;
1806 	malloc_mutex_unlock(tdata->lock);
1807 	if (destroy_tdata)
1808 		prof_tdata_destroy(tsd, tdata, true);
1809 }
1810 
1811 prof_tdata_t *
1812 prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata)
1813 {
1814 	uint64_t thr_uid = tdata->thr_uid;
1815 	uint64_t thr_discrim = tdata->thr_discrim + 1;
1816 	char *thread_name = (tdata->thread_name != NULL) ?
1817 	    prof_thread_name_alloc(tsd, tdata->thread_name) : NULL;
1818 	bool active = tdata->active;
1819 
1820 	prof_tdata_detach(tsd, tdata);
1821 	return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,
1822 	    active));
1823 }
1824 
1825 static bool
1826 prof_tdata_expire(prof_tdata_t *tdata)
1827 {
1828 	bool destroy_tdata;
1829 
1830 	malloc_mutex_lock(tdata->lock);
1831 	if (!tdata->expired) {
1832 		tdata->expired = true;
1833 		destroy_tdata = tdata->attached ? false :
1834 		    prof_tdata_should_destroy(tdata, false);
1835 	} else
1836 		destroy_tdata = false;
1837 	malloc_mutex_unlock(tdata->lock);
1838 
1839 	return (destroy_tdata);
1840 }
1841 
1842 static prof_tdata_t *
1843 prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1844 {
1845 
1846 	return (prof_tdata_expire(tdata) ? tdata : NULL);
1847 }
1848 
1849 void
1850 prof_reset(tsd_t *tsd, size_t lg_sample)
1851 {
1852 	prof_tdata_t *next;
1853 
1854 	assert(lg_sample < (sizeof(uint64_t) << 3));
1855 
1856 	malloc_mutex_lock(&prof_dump_mtx);
1857 	malloc_mutex_lock(&tdatas_mtx);
1858 
1859 	lg_prof_sample = lg_sample;
1860 
1861 	next = NULL;
1862 	do {
1863 		prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
1864 		    prof_tdata_reset_iter, NULL);
1865 		if (to_destroy != NULL) {
1866 			next = tdata_tree_next(&tdatas, to_destroy);
1867 			prof_tdata_destroy_locked(tsd, to_destroy, false);
1868 		} else
1869 			next = NULL;
1870 	} while (next != NULL);
1871 
1872 	malloc_mutex_unlock(&tdatas_mtx);
1873 	malloc_mutex_unlock(&prof_dump_mtx);
1874 }
1875 
1876 void
1877 prof_tdata_cleanup(tsd_t *tsd)
1878 {
1879 	prof_tdata_t *tdata;
1880 
1881 	if (!config_prof)
1882 		return;
1883 
1884 	tdata = tsd_prof_tdata_get(tsd);
1885 	if (tdata != NULL)
1886 		prof_tdata_detach(tsd, tdata);
1887 }
1888 
1889 bool
1890 prof_active_get(void)
1891 {
1892 	bool prof_active_current;
1893 
1894 	malloc_mutex_lock(&prof_active_mtx);
1895 	prof_active_current = prof_active;
1896 	malloc_mutex_unlock(&prof_active_mtx);
1897 	return (prof_active_current);
1898 }
1899 
1900 bool
1901 prof_active_set(bool active)
1902 {
1903 	bool prof_active_old;
1904 
1905 	malloc_mutex_lock(&prof_active_mtx);
1906 	prof_active_old = prof_active;
1907 	prof_active = active;
1908 	malloc_mutex_unlock(&prof_active_mtx);
1909 	return (prof_active_old);
1910 }
1911 
1912 const char *
1913 prof_thread_name_get(void)
1914 {
1915 	tsd_t *tsd;
1916 	prof_tdata_t *tdata;
1917 
1918 	tsd = tsd_fetch();
1919 	tdata = prof_tdata_get(tsd, true);
1920 	if (tdata == NULL)
1921 		return ("");
1922 	return (tdata->thread_name != NULL ? tdata->thread_name : "");
1923 }
1924 
1925 static char *
1926 prof_thread_name_alloc(tsd_t *tsd, const char *thread_name)
1927 {
1928 	char *ret;
1929 	size_t size;
1930 
1931 	if (thread_name == NULL)
1932 		return (NULL);
1933 
1934 	size = strlen(thread_name) + 1;
1935 	if (size == 1)
1936 		return ("");
1937 
1938 	ret = iallocztm(tsd, size, false, tcache_get(tsd, true), true, NULL);
1939 	if (ret == NULL)
1940 		return (NULL);
1941 	memcpy(ret, thread_name, size);
1942 	return (ret);
1943 }
1944 
1945 int
1946 prof_thread_name_set(tsd_t *tsd, const char *thread_name)
1947 {
1948 	prof_tdata_t *tdata;
1949 	unsigned i;
1950 	char *s;
1951 
1952 	tdata = prof_tdata_get(tsd, true);
1953 	if (tdata == NULL)
1954 		return (EAGAIN);
1955 
1956 	/* Validate input. */
1957 	if (thread_name == NULL)
1958 		return (EFAULT);
1959 	for (i = 0; thread_name[i] != '\0'; i++) {
1960 		char c = thread_name[i];
1961 		if (!isgraph(c) && !isblank(c))
1962 			return (EFAULT);
1963 	}
1964 
1965 	s = prof_thread_name_alloc(tsd, thread_name);
1966 	if (s == NULL)
1967 		return (EAGAIN);
1968 
1969 	if (tdata->thread_name != NULL) {
1970 		idalloctm(tsd, tdata->thread_name, tcache_get(tsd, false),
1971 		    true);
1972 		tdata->thread_name = NULL;
1973 	}
1974 	if (strlen(s) > 0)
1975 		tdata->thread_name = s;
1976 	return (0);
1977 }
1978 
1979 bool
1980 prof_thread_active_get(void)
1981 {
1982 	tsd_t *tsd;
1983 	prof_tdata_t *tdata;
1984 
1985 	tsd = tsd_fetch();
1986 	tdata = prof_tdata_get(tsd, true);
1987 	if (tdata == NULL)
1988 		return (false);
1989 	return (tdata->active);
1990 }
1991 
1992 bool
1993 prof_thread_active_set(bool active)
1994 {
1995 	tsd_t *tsd;
1996 	prof_tdata_t *tdata;
1997 
1998 	tsd = tsd_fetch();
1999 	tdata = prof_tdata_get(tsd, true);
2000 	if (tdata == NULL)
2001 		return (true);
2002 	tdata->active = active;
2003 	return (false);
2004 }
2005 
2006 bool
2007 prof_thread_active_init_get(void)
2008 {
2009 	bool active_init;
2010 
2011 	malloc_mutex_lock(&prof_thread_active_init_mtx);
2012 	active_init = prof_thread_active_init;
2013 	malloc_mutex_unlock(&prof_thread_active_init_mtx);
2014 	return (active_init);
2015 }
2016 
2017 bool
2018 prof_thread_active_init_set(bool active_init)
2019 {
2020 	bool active_init_old;
2021 
2022 	malloc_mutex_lock(&prof_thread_active_init_mtx);
2023 	active_init_old = prof_thread_active_init;
2024 	prof_thread_active_init = active_init;
2025 	malloc_mutex_unlock(&prof_thread_active_init_mtx);
2026 	return (active_init_old);
2027 }
2028 
2029 bool
2030 prof_gdump_get(void)
2031 {
2032 	bool prof_gdump_current;
2033 
2034 	malloc_mutex_lock(&prof_gdump_mtx);
2035 	prof_gdump_current = prof_gdump_val;
2036 	malloc_mutex_unlock(&prof_gdump_mtx);
2037 	return (prof_gdump_current);
2038 }
2039 
2040 bool
2041 prof_gdump_set(bool gdump)
2042 {
2043 	bool prof_gdump_old;
2044 
2045 	malloc_mutex_lock(&prof_gdump_mtx);
2046 	prof_gdump_old = prof_gdump_val;
2047 	prof_gdump_val = gdump;
2048 	malloc_mutex_unlock(&prof_gdump_mtx);
2049 	return (prof_gdump_old);
2050 }
2051 
2052 void
2053 prof_boot0(void)
2054 {
2055 
2056 	cassert(config_prof);
2057 
2058 	memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT,
2059 	    sizeof(PROF_PREFIX_DEFAULT));
2060 }
2061 
2062 void
2063 prof_boot1(void)
2064 {
2065 
2066 	cassert(config_prof);
2067 
2068 	/*
2069 	 * opt_prof must be in its final state before any arenas are
2070 	 * initialized, so this function must be executed early.
2071 	 */
2072 
2073 	if (opt_prof_leak && !opt_prof) {
2074 		/*
2075 		 * Enable opt_prof, but in such a way that profiles are never
2076 		 * automatically dumped.
2077 		 */
2078 		opt_prof = true;
2079 		opt_prof_gdump = false;
2080 	} else if (opt_prof) {
2081 		if (opt_lg_prof_interval >= 0) {
2082 			prof_interval = (((uint64_t)1U) <<
2083 			    opt_lg_prof_interval);
2084 		}
2085 	}
2086 }
2087 
2088 bool
2089 prof_boot2(void)
2090 {
2091 
2092 	cassert(config_prof);
2093 
2094 	if (opt_prof) {
2095 		tsd_t *tsd;
2096 		unsigned i;
2097 
2098 		lg_prof_sample = opt_lg_prof_sample;
2099 
2100 		prof_active = opt_prof_active;
2101 		if (malloc_mutex_init(&prof_active_mtx))
2102 			return (true);
2103 
2104 		prof_gdump_val = opt_prof_gdump;
2105 		if (malloc_mutex_init(&prof_gdump_mtx))
2106 			return (true);
2107 
2108 		prof_thread_active_init = opt_prof_thread_active_init;
2109 		if (malloc_mutex_init(&prof_thread_active_init_mtx))
2110 			return (true);
2111 
2112 		tsd = tsd_fetch();
2113 		if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
2114 		    prof_bt_keycomp))
2115 			return (true);
2116 		if (malloc_mutex_init(&bt2gctx_mtx))
2117 			return (true);
2118 
2119 		tdata_tree_new(&tdatas);
2120 		if (malloc_mutex_init(&tdatas_mtx))
2121 			return (true);
2122 
2123 		next_thr_uid = 0;
2124 		if (malloc_mutex_init(&next_thr_uid_mtx))
2125 			return (true);
2126 
2127 		if (malloc_mutex_init(&prof_dump_seq_mtx))
2128 			return (true);
2129 		if (malloc_mutex_init(&prof_dump_mtx))
2130 			return (true);
2131 
2132 		if (opt_prof_final && opt_prof_prefix[0] != '\0' &&
2133 		    atexit(prof_fdump) != 0) {
2134 			malloc_write("<jemalloc>: Error in atexit()\n");
2135 			if (opt_abort)
2136 				abort();
2137 		}
2138 
2139 		gctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS *
2140 		    sizeof(malloc_mutex_t));
2141 		if (gctx_locks == NULL)
2142 			return (true);
2143 		for (i = 0; i < PROF_NCTX_LOCKS; i++) {
2144 			if (malloc_mutex_init(&gctx_locks[i]))
2145 				return (true);
2146 		}
2147 
2148 		tdata_locks = (malloc_mutex_t *)base_alloc(PROF_NTDATA_LOCKS *
2149 		    sizeof(malloc_mutex_t));
2150 		if (tdata_locks == NULL)
2151 			return (true);
2152 		for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
2153 			if (malloc_mutex_init(&tdata_locks[i]))
2154 				return (true);
2155 		}
2156 	}
2157 
2158 #ifdef JEMALLOC_PROF_LIBGCC
2159 	/*
2160 	 * Cause the backtracing machinery to allocate its internal state
2161 	 * before enabling profiling.
2162 	 */
2163 	_Unwind_Backtrace(prof_unwind_init_callback, NULL);
2164 #endif
2165 
2166 	prof_booted = true;
2167 
2168 	return (false);
2169 }
2170 
2171 void
2172 prof_prefork(void)
2173 {
2174 
2175 	if (opt_prof) {
2176 		unsigned i;
2177 
2178 		malloc_mutex_prefork(&tdatas_mtx);
2179 		malloc_mutex_prefork(&bt2gctx_mtx);
2180 		malloc_mutex_prefork(&next_thr_uid_mtx);
2181 		malloc_mutex_prefork(&prof_dump_seq_mtx);
2182 		for (i = 0; i < PROF_NCTX_LOCKS; i++)
2183 			malloc_mutex_prefork(&gctx_locks[i]);
2184 		for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2185 			malloc_mutex_prefork(&tdata_locks[i]);
2186 	}
2187 }
2188 
2189 void
2190 prof_postfork_parent(void)
2191 {
2192 
2193 	if (opt_prof) {
2194 		unsigned i;
2195 
2196 		for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2197 			malloc_mutex_postfork_parent(&tdata_locks[i]);
2198 		for (i = 0; i < PROF_NCTX_LOCKS; i++)
2199 			malloc_mutex_postfork_parent(&gctx_locks[i]);
2200 		malloc_mutex_postfork_parent(&prof_dump_seq_mtx);
2201 		malloc_mutex_postfork_parent(&next_thr_uid_mtx);
2202 		malloc_mutex_postfork_parent(&bt2gctx_mtx);
2203 		malloc_mutex_postfork_parent(&tdatas_mtx);
2204 	}
2205 }
2206 
2207 void
2208 prof_postfork_child(void)
2209 {
2210 
2211 	if (opt_prof) {
2212 		unsigned i;
2213 
2214 		for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2215 			malloc_mutex_postfork_child(&tdata_locks[i]);
2216 		for (i = 0; i < PROF_NCTX_LOCKS; i++)
2217 			malloc_mutex_postfork_child(&gctx_locks[i]);
2218 		malloc_mutex_postfork_child(&prof_dump_seq_mtx);
2219 		malloc_mutex_postfork_child(&next_thr_uid_mtx);
2220 		malloc_mutex_postfork_child(&bt2gctx_mtx);
2221 		malloc_mutex_postfork_child(&tdatas_mtx);
2222 	}
2223 }
2224 
2225 /******************************************************************************/
2226