xref: /freebsd/contrib/jemalloc/src/jemalloc.c (revision 907b59d76938e654f0d040a888e8dfca3de1e222)
1 #define	JEMALLOC_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3 
4 /******************************************************************************/
5 /* Data. */
6 
7 /* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
8 const char	*__malloc_options_1_0 = NULL;
9 __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
10 
11 /* Runtime configuration options. */
12 const char	*je_malloc_conf JEMALLOC_ATTR(weak);
13 bool	opt_abort =
14 #ifdef JEMALLOC_DEBUG
15     true
16 #else
17     false
18 #endif
19     ;
20 const char	*opt_junk =
21 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
22     "true"
23 #else
24     "false"
25 #endif
26     ;
27 bool	opt_junk_alloc =
28 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
29     true
30 #else
31     false
32 #endif
33     ;
34 bool	opt_junk_free =
35 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
36     true
37 #else
38     false
39 #endif
40     ;
41 
42 size_t	opt_quarantine = ZU(0);
43 bool	opt_redzone = false;
44 bool	opt_utrace = false;
45 bool	opt_xmalloc = false;
46 bool	opt_zero = false;
47 unsigned	opt_narenas = 0;
48 
49 /* Initialized to true if the process is running inside Valgrind. */
50 bool	in_valgrind;
51 
52 unsigned	ncpus;
53 
54 /* Protects arenas initialization. */
55 static malloc_mutex_t	arenas_lock;
56 /*
57  * Arenas that are used to service external requests.  Not all elements of the
58  * arenas array are necessarily used; arenas are created lazily as needed.
59  *
60  * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
61  * arenas.  arenas[narenas_auto..narenas_total) are only used if the application
62  * takes some action to create them and allocate from them.
63  */
64 arena_t			**arenas;
65 static unsigned		narenas_total; /* Use narenas_total_*(). */
66 static arena_t		*a0; /* arenas[0]; read-only after initialization. */
67 unsigned		narenas_auto; /* Read-only after initialization. */
68 
69 typedef enum {
70 	malloc_init_uninitialized	= 3,
71 	malloc_init_a0_initialized	= 2,
72 	malloc_init_recursible		= 1,
73 	malloc_init_initialized		= 0 /* Common case --> jnz. */
74 } malloc_init_t;
75 static malloc_init_t	malloc_init_state = malloc_init_uninitialized;
76 
77 /* False should be the common case.  Set to true to trigger initialization. */
78 static bool	malloc_slow = true;
79 
80 /* When malloc_slow is true, set the corresponding bits for sanity check. */
81 enum {
82 	flag_opt_junk_alloc	= (1U),
83 	flag_opt_junk_free	= (1U << 1),
84 	flag_opt_quarantine	= (1U << 2),
85 	flag_opt_zero		= (1U << 3),
86 	flag_opt_utrace		= (1U << 4),
87 	flag_in_valgrind	= (1U << 5),
88 	flag_opt_xmalloc	= (1U << 6)
89 };
90 static uint8_t	malloc_slow_flags;
91 
92 /* Last entry for overflow detection only.  */
93 JEMALLOC_ALIGNED(CACHELINE)
94 const size_t	index2size_tab[NSIZES+1] = {
95 #define	SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
96 	((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)),
97 	SIZE_CLASSES
98 #undef SC
99 	ZU(0)
100 };
101 
102 JEMALLOC_ALIGNED(CACHELINE)
103 const uint8_t	size2index_tab[] = {
104 #if LG_TINY_MIN == 0
105 #warning "Dangerous LG_TINY_MIN"
106 #define	S2B_0(i)	i,
107 #elif LG_TINY_MIN == 1
108 #warning "Dangerous LG_TINY_MIN"
109 #define	S2B_1(i)	i,
110 #elif LG_TINY_MIN == 2
111 #warning "Dangerous LG_TINY_MIN"
112 #define	S2B_2(i)	i,
113 #elif LG_TINY_MIN == 3
114 #define	S2B_3(i)	i,
115 #elif LG_TINY_MIN == 4
116 #define	S2B_4(i)	i,
117 #elif LG_TINY_MIN == 5
118 #define	S2B_5(i)	i,
119 #elif LG_TINY_MIN == 6
120 #define	S2B_6(i)	i,
121 #elif LG_TINY_MIN == 7
122 #define	S2B_7(i)	i,
123 #elif LG_TINY_MIN == 8
124 #define	S2B_8(i)	i,
125 #elif LG_TINY_MIN == 9
126 #define	S2B_9(i)	i,
127 #elif LG_TINY_MIN == 10
128 #define	S2B_10(i)	i,
129 #elif LG_TINY_MIN == 11
130 #define	S2B_11(i)	i,
131 #else
132 #error "Unsupported LG_TINY_MIN"
133 #endif
134 #if LG_TINY_MIN < 1
135 #define	S2B_1(i)	S2B_0(i) S2B_0(i)
136 #endif
137 #if LG_TINY_MIN < 2
138 #define	S2B_2(i)	S2B_1(i) S2B_1(i)
139 #endif
140 #if LG_TINY_MIN < 3
141 #define	S2B_3(i)	S2B_2(i) S2B_2(i)
142 #endif
143 #if LG_TINY_MIN < 4
144 #define	S2B_4(i)	S2B_3(i) S2B_3(i)
145 #endif
146 #if LG_TINY_MIN < 5
147 #define	S2B_5(i)	S2B_4(i) S2B_4(i)
148 #endif
149 #if LG_TINY_MIN < 6
150 #define	S2B_6(i)	S2B_5(i) S2B_5(i)
151 #endif
152 #if LG_TINY_MIN < 7
153 #define	S2B_7(i)	S2B_6(i) S2B_6(i)
154 #endif
155 #if LG_TINY_MIN < 8
156 #define	S2B_8(i)	S2B_7(i) S2B_7(i)
157 #endif
158 #if LG_TINY_MIN < 9
159 #define	S2B_9(i)	S2B_8(i) S2B_8(i)
160 #endif
161 #if LG_TINY_MIN < 10
162 #define	S2B_10(i)	S2B_9(i) S2B_9(i)
163 #endif
164 #if LG_TINY_MIN < 11
165 #define	S2B_11(i)	S2B_10(i) S2B_10(i)
166 #endif
167 #define	S2B_no(i)
168 #define	SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
169 	S2B_##lg_delta_lookup(index)
170 	SIZE_CLASSES
171 #undef S2B_3
172 #undef S2B_4
173 #undef S2B_5
174 #undef S2B_6
175 #undef S2B_7
176 #undef S2B_8
177 #undef S2B_9
178 #undef S2B_10
179 #undef S2B_11
180 #undef S2B_no
181 #undef SC
182 };
183 
184 #ifdef JEMALLOC_THREADED_INIT
185 /* Used to let the initializing thread recursively allocate. */
186 #  define NO_INITIALIZER	((unsigned long)0)
187 #  define INITIALIZER		pthread_self()
188 #  define IS_INITIALIZER	(malloc_initializer == pthread_self())
189 static pthread_t		malloc_initializer = NO_INITIALIZER;
190 #else
191 #  define NO_INITIALIZER	false
192 #  define INITIALIZER		true
193 #  define IS_INITIALIZER	malloc_initializer
194 static bool			malloc_initializer = NO_INITIALIZER;
195 #endif
196 
197 /* Used to avoid initialization races. */
198 #ifdef _WIN32
199 #if _WIN32_WINNT >= 0x0600
200 static malloc_mutex_t	init_lock = SRWLOCK_INIT;
201 #else
202 static malloc_mutex_t	init_lock;
203 static bool init_lock_initialized = false;
204 
205 JEMALLOC_ATTR(constructor)
206 static void WINAPI
207 _init_init_lock(void)
208 {
209 
210 	/* If another constructor in the same binary is using mallctl to
211 	 * e.g. setup chunk hooks, it may end up running before this one,
212 	 * and malloc_init_hard will crash trying to lock the uninitialized
213 	 * lock. So we force an initialization of the lock in
214 	 * malloc_init_hard as well. We don't try to care about atomicity
215 	 * of the accessed to the init_lock_initialized boolean, since it
216 	 * really only matters early in the process creation, before any
217 	 * separate thread normally starts doing anything. */
218 	if (!init_lock_initialized)
219 		malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT);
220 	init_lock_initialized = true;
221 }
222 
223 #ifdef _MSC_VER
224 #  pragma section(".CRT$XCU", read)
225 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
226 static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
227 #endif
228 #endif
229 #else
230 static malloc_mutex_t	init_lock = MALLOC_MUTEX_INITIALIZER;
231 #endif
232 
233 typedef struct {
234 	void	*p;	/* Input pointer (as in realloc(p, s)). */
235 	size_t	s;	/* Request size. */
236 	void	*r;	/* Result pointer. */
237 } malloc_utrace_t;
238 
239 #ifdef JEMALLOC_UTRACE
240 #  define UTRACE(a, b, c) do {						\
241 	if (unlikely(opt_utrace)) {					\
242 		int utrace_serrno = errno;				\
243 		malloc_utrace_t ut;					\
244 		ut.p = (a);						\
245 		ut.s = (b);						\
246 		ut.r = (c);						\
247 		utrace(&ut, sizeof(ut));				\
248 		errno = utrace_serrno;					\
249 	}								\
250 } while (0)
251 #else
252 #  define UTRACE(a, b, c)
253 #endif
254 
255 /******************************************************************************/
256 /*
257  * Function prototypes for static functions that are referenced prior to
258  * definition.
259  */
260 
261 static bool	malloc_init_hard_a0(void);
262 static bool	malloc_init_hard(void);
263 
264 /******************************************************************************/
265 /*
266  * Begin miscellaneous support functions.
267  */
268 
269 JEMALLOC_ALWAYS_INLINE_C bool
270 malloc_initialized(void)
271 {
272 
273 	return (malloc_init_state == malloc_init_initialized);
274 }
275 
276 JEMALLOC_ALWAYS_INLINE_C void
277 malloc_thread_init(void)
278 {
279 
280 	/*
281 	 * TSD initialization can't be safely done as a side effect of
282 	 * deallocation, because it is possible for a thread to do nothing but
283 	 * deallocate its TLS data via free(), in which case writing to TLS
284 	 * would cause write-after-free memory corruption.  The quarantine
285 	 * facility *only* gets used as a side effect of deallocation, so make
286 	 * a best effort attempt at initializing its TSD by hooking all
287 	 * allocation events.
288 	 */
289 	if (config_fill && unlikely(opt_quarantine))
290 		quarantine_alloc_hook();
291 }
292 
293 JEMALLOC_ALWAYS_INLINE_C bool
294 malloc_init_a0(void)
295 {
296 
297 	if (unlikely(malloc_init_state == malloc_init_uninitialized))
298 		return (malloc_init_hard_a0());
299 	return (false);
300 }
301 
302 JEMALLOC_ALWAYS_INLINE_C bool
303 malloc_init(void)
304 {
305 
306 	if (unlikely(!malloc_initialized()) && malloc_init_hard())
307 		return (true);
308 	malloc_thread_init();
309 
310 	return (false);
311 }
312 
313 /*
314  * The a0*() functions are used instead of i{d,}alloc() in situations that
315  * cannot tolerate TLS variable access.
316  */
317 
318 static void *
319 a0ialloc(size_t size, bool zero, bool is_metadata)
320 {
321 
322 	if (unlikely(malloc_init_a0()))
323 		return (NULL);
324 
325 	return (iallocztm(TSDN_NULL, size, size2index(size), zero, NULL,
326 	    is_metadata, arena_get(TSDN_NULL, 0, true), true));
327 }
328 
329 static void
330 a0idalloc(void *ptr, bool is_metadata)
331 {
332 
333 	idalloctm(TSDN_NULL, ptr, false, is_metadata, true);
334 }
335 
336 void *
337 a0malloc(size_t size)
338 {
339 
340 	return (a0ialloc(size, false, true));
341 }
342 
343 void
344 a0dalloc(void *ptr)
345 {
346 
347 	a0idalloc(ptr, true);
348 }
349 
350 /*
351  * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive
352  * situations that cannot tolerate TLS variable access (TLS allocation and very
353  * early internal data structure initialization).
354  */
355 
356 void *
357 bootstrap_malloc(size_t size)
358 {
359 
360 	if (unlikely(size == 0))
361 		size = 1;
362 
363 	return (a0ialloc(size, false, false));
364 }
365 
366 void *
367 bootstrap_calloc(size_t num, size_t size)
368 {
369 	size_t num_size;
370 
371 	num_size = num * size;
372 	if (unlikely(num_size == 0)) {
373 		assert(num == 0 || size == 0);
374 		num_size = 1;
375 	}
376 
377 	return (a0ialloc(num_size, true, false));
378 }
379 
380 void
381 bootstrap_free(void *ptr)
382 {
383 
384 	if (unlikely(ptr == NULL))
385 		return;
386 
387 	a0idalloc(ptr, false);
388 }
389 
390 static void
391 arena_set(unsigned ind, arena_t *arena)
392 {
393 
394 	atomic_write_p((void **)&arenas[ind], arena);
395 }
396 
397 static void
398 narenas_total_set(unsigned narenas)
399 {
400 
401 	atomic_write_u(&narenas_total, narenas);
402 }
403 
404 static void
405 narenas_total_inc(void)
406 {
407 
408 	atomic_add_u(&narenas_total, 1);
409 }
410 
411 unsigned
412 narenas_total_get(void)
413 {
414 
415 	return (atomic_read_u(&narenas_total));
416 }
417 
418 /* Create a new arena and insert it into the arenas array at index ind. */
419 static arena_t *
420 arena_init_locked(tsdn_t *tsdn, unsigned ind)
421 {
422 	arena_t *arena;
423 
424 	assert(ind <= narenas_total_get());
425 	if (ind > MALLOCX_ARENA_MAX)
426 		return (NULL);
427 	if (ind == narenas_total_get())
428 		narenas_total_inc();
429 
430 	/*
431 	 * Another thread may have already initialized arenas[ind] if it's an
432 	 * auto arena.
433 	 */
434 	arena = arena_get(tsdn, ind, false);
435 	if (arena != NULL) {
436 		assert(ind < narenas_auto);
437 		return (arena);
438 	}
439 
440 	/* Actually initialize the arena. */
441 	arena = arena_new(tsdn, ind);
442 	arena_set(ind, arena);
443 	return (arena);
444 }
445 
446 arena_t *
447 arena_init(tsdn_t *tsdn, unsigned ind)
448 {
449 	arena_t *arena;
450 
451 	malloc_mutex_lock(tsdn, &arenas_lock);
452 	arena = arena_init_locked(tsdn, ind);
453 	malloc_mutex_unlock(tsdn, &arenas_lock);
454 	return (arena);
455 }
456 
457 static void
458 arena_bind(tsd_t *tsd, unsigned ind, bool internal)
459 {
460 	arena_t *arena;
461 
462 	arena = arena_get(tsd_tsdn(tsd), ind, false);
463 	arena_nthreads_inc(arena, internal);
464 
465 	if (tsd_nominal(tsd)) {
466 		if (internal)
467 			tsd_iarena_set(tsd, arena);
468 		else
469 			tsd_arena_set(tsd, arena);
470 	}
471 }
472 
473 void
474 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind)
475 {
476 	arena_t *oldarena, *newarena;
477 
478 	oldarena = arena_get(tsd_tsdn(tsd), oldind, false);
479 	newarena = arena_get(tsd_tsdn(tsd), newind, false);
480 	arena_nthreads_dec(oldarena, false);
481 	arena_nthreads_inc(newarena, false);
482 	tsd_arena_set(tsd, newarena);
483 }
484 
485 static void
486 arena_unbind(tsd_t *tsd, unsigned ind, bool internal)
487 {
488 	arena_t *arena;
489 
490 	arena = arena_get(tsd_tsdn(tsd), ind, false);
491 	arena_nthreads_dec(arena, internal);
492 	if (internal)
493 		tsd_iarena_set(tsd, NULL);
494 	else
495 		tsd_arena_set(tsd, NULL);
496 }
497 
498 arena_tdata_t *
499 arena_tdata_get_hard(tsd_t *tsd, unsigned ind)
500 {
501 	arena_tdata_t *tdata, *arenas_tdata_old;
502 	arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
503 	unsigned narenas_tdata_old, i;
504 	unsigned narenas_tdata = tsd_narenas_tdata_get(tsd);
505 	unsigned narenas_actual = narenas_total_get();
506 
507 	/*
508 	 * Dissociate old tdata array (and set up for deallocation upon return)
509 	 * if it's too small.
510 	 */
511 	if (arenas_tdata != NULL && narenas_tdata < narenas_actual) {
512 		arenas_tdata_old = arenas_tdata;
513 		narenas_tdata_old = narenas_tdata;
514 		arenas_tdata = NULL;
515 		narenas_tdata = 0;
516 		tsd_arenas_tdata_set(tsd, arenas_tdata);
517 		tsd_narenas_tdata_set(tsd, narenas_tdata);
518 	} else {
519 		arenas_tdata_old = NULL;
520 		narenas_tdata_old = 0;
521 	}
522 
523 	/* Allocate tdata array if it's missing. */
524 	if (arenas_tdata == NULL) {
525 		bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd);
526 		narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1;
527 
528 		if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) {
529 			*arenas_tdata_bypassp = true;
530 			arenas_tdata = (arena_tdata_t *)a0malloc(
531 			    sizeof(arena_tdata_t) * narenas_tdata);
532 			*arenas_tdata_bypassp = false;
533 		}
534 		if (arenas_tdata == NULL) {
535 			tdata = NULL;
536 			goto label_return;
537 		}
538 		assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp);
539 		tsd_arenas_tdata_set(tsd, arenas_tdata);
540 		tsd_narenas_tdata_set(tsd, narenas_tdata);
541 	}
542 
543 	/*
544 	 * Copy to tdata array.  It's possible that the actual number of arenas
545 	 * has increased since narenas_total_get() was called above, but that
546 	 * causes no correctness issues unless two threads concurrently execute
547 	 * the arenas.extend mallctl, which we trust mallctl synchronization to
548 	 * prevent.
549 	 */
550 
551 	/* Copy/initialize tickers. */
552 	for (i = 0; i < narenas_actual; i++) {
553 		if (i < narenas_tdata_old) {
554 			ticker_copy(&arenas_tdata[i].decay_ticker,
555 			    &arenas_tdata_old[i].decay_ticker);
556 		} else {
557 			ticker_init(&arenas_tdata[i].decay_ticker,
558 			    DECAY_NTICKS_PER_UPDATE);
559 		}
560 	}
561 	if (narenas_tdata > narenas_actual) {
562 		memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t)
563 		    * (narenas_tdata - narenas_actual));
564 	}
565 
566 	/* Read the refreshed tdata array. */
567 	tdata = &arenas_tdata[ind];
568 label_return:
569 	if (arenas_tdata_old != NULL)
570 		a0dalloc(arenas_tdata_old);
571 	return (tdata);
572 }
573 
574 /* Slow path, called only by arena_choose(). */
575 arena_t *
576 arena_choose_hard(tsd_t *tsd, bool internal)
577 {
578 	arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
579 
580 	if (narenas_auto > 1) {
581 		unsigned i, j, choose[2], first_null;
582 
583 		/*
584 		 * Determine binding for both non-internal and internal
585 		 * allocation.
586 		 *
587 		 *   choose[0]: For application allocation.
588 		 *   choose[1]: For internal metadata allocation.
589 		 */
590 
591 		for (j = 0; j < 2; j++)
592 			choose[j] = 0;
593 
594 		first_null = narenas_auto;
595 		malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
596 		assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
597 		for (i = 1; i < narenas_auto; i++) {
598 			if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
599 				/*
600 				 * Choose the first arena that has the lowest
601 				 * number of threads assigned to it.
602 				 */
603 				for (j = 0; j < 2; j++) {
604 					if (arena_nthreads_get(arena_get(
605 					    tsd_tsdn(tsd), i, false), !!j) <
606 					    arena_nthreads_get(arena_get(
607 					    tsd_tsdn(tsd), choose[j], false),
608 					    !!j))
609 						choose[j] = i;
610 				}
611 			} else if (first_null == narenas_auto) {
612 				/*
613 				 * Record the index of the first uninitialized
614 				 * arena, in case all extant arenas are in use.
615 				 *
616 				 * NB: It is possible for there to be
617 				 * discontinuities in terms of initialized
618 				 * versus uninitialized arenas, due to the
619 				 * "thread.arena" mallctl.
620 				 */
621 				first_null = i;
622 			}
623 		}
624 
625 		for (j = 0; j < 2; j++) {
626 			if (arena_nthreads_get(arena_get(tsd_tsdn(tsd),
627 			    choose[j], false), !!j) == 0 || first_null ==
628 			    narenas_auto) {
629 				/*
630 				 * Use an unloaded arena, or the least loaded
631 				 * arena if all arenas are already initialized.
632 				 */
633 				if (!!j == internal) {
634 					ret = arena_get(tsd_tsdn(tsd),
635 					    choose[j], false);
636 				}
637 			} else {
638 				arena_t *arena;
639 
640 				/* Initialize a new arena. */
641 				choose[j] = first_null;
642 				arena = arena_init_locked(tsd_tsdn(tsd),
643 				    choose[j]);
644 				if (arena == NULL) {
645 					malloc_mutex_unlock(tsd_tsdn(tsd),
646 					    &arenas_lock);
647 					return (NULL);
648 				}
649 				if (!!j == internal)
650 					ret = arena;
651 			}
652 			arena_bind(tsd, choose[j], !!j);
653 		}
654 		malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
655 	} else {
656 		ret = arena_get(tsd_tsdn(tsd), 0, false);
657 		arena_bind(tsd, 0, false);
658 		arena_bind(tsd, 0, true);
659 	}
660 
661 	return (ret);
662 }
663 
664 void
665 thread_allocated_cleanup(tsd_t *tsd)
666 {
667 
668 	/* Do nothing. */
669 }
670 
671 void
672 thread_deallocated_cleanup(tsd_t *tsd)
673 {
674 
675 	/* Do nothing. */
676 }
677 
678 void
679 iarena_cleanup(tsd_t *tsd)
680 {
681 	arena_t *iarena;
682 
683 	iarena = tsd_iarena_get(tsd);
684 	if (iarena != NULL)
685 		arena_unbind(tsd, iarena->ind, true);
686 }
687 
688 void
689 arena_cleanup(tsd_t *tsd)
690 {
691 	arena_t *arena;
692 
693 	arena = tsd_arena_get(tsd);
694 	if (arena != NULL)
695 		arena_unbind(tsd, arena->ind, false);
696 }
697 
698 void
699 arenas_tdata_cleanup(tsd_t *tsd)
700 {
701 	arena_tdata_t *arenas_tdata;
702 
703 	/* Prevent tsd->arenas_tdata from being (re)created. */
704 	*tsd_arenas_tdata_bypassp_get(tsd) = true;
705 
706 	arenas_tdata = tsd_arenas_tdata_get(tsd);
707 	if (arenas_tdata != NULL) {
708 		tsd_arenas_tdata_set(tsd, NULL);
709 		a0dalloc(arenas_tdata);
710 	}
711 }
712 
713 void
714 narenas_tdata_cleanup(tsd_t *tsd)
715 {
716 
717 	/* Do nothing. */
718 }
719 
720 void
721 arenas_tdata_bypass_cleanup(tsd_t *tsd)
722 {
723 
724 	/* Do nothing. */
725 }
726 
727 static void
728 stats_print_atexit(void)
729 {
730 
731 	if (config_tcache && config_stats) {
732 		tsdn_t *tsdn;
733 		unsigned narenas, i;
734 
735 		tsdn = tsdn_fetch();
736 
737 		/*
738 		 * Merge stats from extant threads.  This is racy, since
739 		 * individual threads do not lock when recording tcache stats
740 		 * events.  As a consequence, the final stats may be slightly
741 		 * out of date by the time they are reported, if other threads
742 		 * continue to allocate.
743 		 */
744 		for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
745 			arena_t *arena = arena_get(tsdn, i, false);
746 			if (arena != NULL) {
747 				tcache_t *tcache;
748 
749 				/*
750 				 * tcache_stats_merge() locks bins, so if any
751 				 * code is introduced that acquires both arena
752 				 * and bin locks in the opposite order,
753 				 * deadlocks may result.
754 				 */
755 				malloc_mutex_lock(tsdn, &arena->lock);
756 				ql_foreach(tcache, &arena->tcache_ql, link) {
757 					tcache_stats_merge(tsdn, tcache, arena);
758 				}
759 				malloc_mutex_unlock(tsdn, &arena->lock);
760 			}
761 		}
762 	}
763 	je_malloc_stats_print(NULL, NULL, NULL);
764 }
765 
766 /*
767  * End miscellaneous support functions.
768  */
769 /******************************************************************************/
770 /*
771  * Begin initialization functions.
772  */
773 
774 #ifndef JEMALLOC_HAVE_SECURE_GETENV
775 static char *
776 secure_getenv(const char *name)
777 {
778 
779 #  ifdef JEMALLOC_HAVE_ISSETUGID
780 	if (issetugid() != 0)
781 		return (NULL);
782 #  endif
783 	return (getenv(name));
784 }
785 #endif
786 
787 static unsigned
788 malloc_ncpus(void)
789 {
790 	long result;
791 
792 #ifdef _WIN32
793 	SYSTEM_INFO si;
794 	GetSystemInfo(&si);
795 	result = si.dwNumberOfProcessors;
796 #else
797 	result = sysconf(_SC_NPROCESSORS_ONLN);
798 #endif
799 	return ((result == -1) ? 1 : (unsigned)result);
800 }
801 
802 static bool
803 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
804     char const **v_p, size_t *vlen_p)
805 {
806 	bool accept;
807 	const char *opts = *opts_p;
808 
809 	*k_p = opts;
810 
811 	for (accept = false; !accept;) {
812 		switch (*opts) {
813 		case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
814 		case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
815 		case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
816 		case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
817 		case 'Y': case 'Z':
818 		case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
819 		case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
820 		case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
821 		case 's': case 't': case 'u': case 'v': case 'w': case 'x':
822 		case 'y': case 'z':
823 		case '0': case '1': case '2': case '3': case '4': case '5':
824 		case '6': case '7': case '8': case '9':
825 		case '_':
826 			opts++;
827 			break;
828 		case ':':
829 			opts++;
830 			*klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
831 			*v_p = opts;
832 			accept = true;
833 			break;
834 		case '\0':
835 			if (opts != *opts_p) {
836 				malloc_write("<jemalloc>: Conf string ends "
837 				    "with key\n");
838 			}
839 			return (true);
840 		default:
841 			malloc_write("<jemalloc>: Malformed conf string\n");
842 			return (true);
843 		}
844 	}
845 
846 	for (accept = false; !accept;) {
847 		switch (*opts) {
848 		case ',':
849 			opts++;
850 			/*
851 			 * Look ahead one character here, because the next time
852 			 * this function is called, it will assume that end of
853 			 * input has been cleanly reached if no input remains,
854 			 * but we have optimistically already consumed the
855 			 * comma if one exists.
856 			 */
857 			if (*opts == '\0') {
858 				malloc_write("<jemalloc>: Conf string ends "
859 				    "with comma\n");
860 			}
861 			*vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
862 			accept = true;
863 			break;
864 		case '\0':
865 			*vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
866 			accept = true;
867 			break;
868 		default:
869 			opts++;
870 			break;
871 		}
872 	}
873 
874 	*opts_p = opts;
875 	return (false);
876 }
877 
878 static void
879 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
880     size_t vlen)
881 {
882 
883 	malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
884 	    (int)vlen, v);
885 }
886 
887 static void
888 malloc_slow_flag_init(void)
889 {
890 	/*
891 	 * Combine the runtime options into malloc_slow for fast path.  Called
892 	 * after processing all the options.
893 	 */
894 	malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
895 	    | (opt_junk_free ? flag_opt_junk_free : 0)
896 	    | (opt_quarantine ? flag_opt_quarantine : 0)
897 	    | (opt_zero ? flag_opt_zero : 0)
898 	    | (opt_utrace ? flag_opt_utrace : 0)
899 	    | (opt_xmalloc ? flag_opt_xmalloc : 0);
900 
901 	if (config_valgrind)
902 		malloc_slow_flags |= (in_valgrind ? flag_in_valgrind : 0);
903 
904 	malloc_slow = (malloc_slow_flags != 0);
905 }
906 
907 static void
908 malloc_conf_init(void)
909 {
910 	unsigned i;
911 	char buf[PATH_MAX + 1];
912 	const char *opts, *k, *v;
913 	size_t klen, vlen;
914 
915 	/*
916 	 * Automatically configure valgrind before processing options.  The
917 	 * valgrind option remains in jemalloc 3.x for compatibility reasons.
918 	 */
919 	if (config_valgrind) {
920 		in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false;
921 		if (config_fill && unlikely(in_valgrind)) {
922 			opt_junk = "false";
923 			opt_junk_alloc = false;
924 			opt_junk_free = false;
925 			assert(!opt_zero);
926 			opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
927 			opt_redzone = true;
928 		}
929 		if (config_tcache && unlikely(in_valgrind))
930 			opt_tcache = false;
931 	}
932 
933 	for (i = 0; i < 4; i++) {
934 		/* Get runtime configuration. */
935 		switch (i) {
936 		case 0:
937 			opts = config_malloc_conf;
938 			break;
939 		case 1:
940 			if (je_malloc_conf != NULL) {
941 				/*
942 				 * Use options that were compiled into the
943 				 * program.
944 				 */
945 				opts = je_malloc_conf;
946 			} else {
947 				/* No configuration specified. */
948 				buf[0] = '\0';
949 				opts = buf;
950 			}
951 			break;
952 		case 2: {
953 			ssize_t linklen = 0;
954 #ifndef _WIN32
955 			int saved_errno = errno;
956 			const char *linkname =
957 #  ifdef JEMALLOC_PREFIX
958 			    "/etc/"JEMALLOC_PREFIX"malloc.conf"
959 #  else
960 			    "/etc/malloc.conf"
961 #  endif
962 			    ;
963 
964 			/*
965 			 * Try to use the contents of the "/etc/malloc.conf"
966 			 * symbolic link's name.
967 			 */
968 			linklen = readlink(linkname, buf, sizeof(buf) - 1);
969 			if (linklen == -1) {
970 				/* No configuration specified. */
971 				linklen = 0;
972 				/* Restore errno. */
973 				set_errno(saved_errno);
974 			}
975 #endif
976 			buf[linklen] = '\0';
977 			opts = buf;
978 			break;
979 		} case 3: {
980 			const char *envname =
981 #ifdef JEMALLOC_PREFIX
982 			    JEMALLOC_CPREFIX"MALLOC_CONF"
983 #else
984 			    "MALLOC_CONF"
985 #endif
986 			    ;
987 
988 			if ((opts = secure_getenv(envname)) != NULL) {
989 				/*
990 				 * Do nothing; opts is already initialized to
991 				 * the value of the MALLOC_CONF environment
992 				 * variable.
993 				 */
994 			} else {
995 				/* No configuration specified. */
996 				buf[0] = '\0';
997 				opts = buf;
998 			}
999 			break;
1000 		} default:
1001 			not_reached();
1002 			buf[0] = '\0';
1003 			opts = buf;
1004 		}
1005 
1006 		while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v,
1007 		    &vlen)) {
1008 #define	CONF_MATCH(n)							\
1009 	(sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
1010 #define	CONF_MATCH_VALUE(n)						\
1011 	(sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0)
1012 #define	CONF_HANDLE_BOOL(o, n, cont)					\
1013 			if (CONF_MATCH(n)) {				\
1014 				if (CONF_MATCH_VALUE("true"))		\
1015 					o = true;			\
1016 				else if (CONF_MATCH_VALUE("false"))	\
1017 					o = false;			\
1018 				else {					\
1019 					malloc_conf_error(		\
1020 					    "Invalid conf value",	\
1021 					    k, klen, v, vlen);		\
1022 				}					\
1023 				if (cont)				\
1024 					continue;			\
1025 			}
1026 #define	CONF_HANDLE_T_U(t, o, n, min, max, clip)			\
1027 			if (CONF_MATCH(n)) {				\
1028 				uintmax_t um;				\
1029 				char *end;				\
1030 									\
1031 				set_errno(0);				\
1032 				um = malloc_strtoumax(v, &end, 0);	\
1033 				if (get_errno() != 0 || (uintptr_t)end -\
1034 				    (uintptr_t)v != vlen) {		\
1035 					malloc_conf_error(		\
1036 					    "Invalid conf value",	\
1037 					    k, klen, v, vlen);		\
1038 				} else if (clip) {			\
1039 					if ((min) != 0 && um < (min))	\
1040 						o = (t)(min);		\
1041 					else if (um > (max))		\
1042 						o = (t)(max);		\
1043 					else				\
1044 						o = (t)um;		\
1045 				} else {				\
1046 					if (((min) != 0 && um < (min))	\
1047 					    || um > (max)) {		\
1048 						malloc_conf_error(	\
1049 						    "Out-of-range "	\
1050 						    "conf value",	\
1051 						    k, klen, v, vlen);	\
1052 					} else				\
1053 						o = (t)um;		\
1054 				}					\
1055 				continue;				\
1056 			}
1057 #define	CONF_HANDLE_UNSIGNED(o, n, min, max, clip)			\
1058 			CONF_HANDLE_T_U(unsigned, o, n, min, max, clip)
1059 #define	CONF_HANDLE_SIZE_T(o, n, min, max, clip)			\
1060 			CONF_HANDLE_T_U(size_t, o, n, min, max, clip)
1061 #define	CONF_HANDLE_SSIZE_T(o, n, min, max)				\
1062 			if (CONF_MATCH(n)) {				\
1063 				long l;					\
1064 				char *end;				\
1065 									\
1066 				set_errno(0);				\
1067 				l = strtol(v, &end, 0);			\
1068 				if (get_errno() != 0 || (uintptr_t)end -\
1069 				    (uintptr_t)v != vlen) {		\
1070 					malloc_conf_error(		\
1071 					    "Invalid conf value",	\
1072 					    k, klen, v, vlen);		\
1073 				} else if (l < (ssize_t)(min) || l >	\
1074 				    (ssize_t)(max)) {			\
1075 					malloc_conf_error(		\
1076 					    "Out-of-range conf value",	\
1077 					    k, klen, v, vlen);		\
1078 				} else					\
1079 					o = l;				\
1080 				continue;				\
1081 			}
1082 #define	CONF_HANDLE_CHAR_P(o, n, d)					\
1083 			if (CONF_MATCH(n)) {				\
1084 				size_t cpylen = (vlen <=		\
1085 				    sizeof(o)-1) ? vlen :		\
1086 				    sizeof(o)-1;			\
1087 				strncpy(o, v, cpylen);			\
1088 				o[cpylen] = '\0';			\
1089 				continue;				\
1090 			}
1091 
1092 			CONF_HANDLE_BOOL(opt_abort, "abort", true)
1093 			/*
1094 			 * Chunks always require at least one header page,
1095 			 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and
1096 			 * possibly an additional page in the presence of
1097 			 * redzones.  In order to simplify options processing,
1098 			 * use a conservative bound that accommodates all these
1099 			 * constraints.
1100 			 */
1101 			CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
1102 			    LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1),
1103 			    (sizeof(size_t) << 3) - 1, true)
1104 			if (strncmp("dss", k, klen) == 0) {
1105 				int i;
1106 				bool match = false;
1107 				for (i = 0; i < dss_prec_limit; i++) {
1108 					if (strncmp(dss_prec_names[i], v, vlen)
1109 					    == 0) {
1110 						if (chunk_dss_prec_set(NULL,
1111 						   i)) {
1112 							malloc_conf_error(
1113 							    "Error setting dss",
1114 							    k, klen, v, vlen);
1115 						} else {
1116 							opt_dss =
1117 							    dss_prec_names[i];
1118 							match = true;
1119 							break;
1120 						}
1121 					}
1122 				}
1123 				if (!match) {
1124 					malloc_conf_error("Invalid conf value",
1125 					    k, klen, v, vlen);
1126 				}
1127 				continue;
1128 			}
1129 			CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
1130 			    UINT_MAX, false)
1131 			if (strncmp("purge", k, klen) == 0) {
1132 				int i;
1133 				bool match = false;
1134 				for (i = 0; i < purge_mode_limit; i++) {
1135 					if (strncmp(purge_mode_names[i], v,
1136 					    vlen) == 0) {
1137 						opt_purge = (purge_mode_t)i;
1138 						match = true;
1139 						break;
1140 					}
1141 				}
1142 				if (!match) {
1143 					malloc_conf_error("Invalid conf value",
1144 					    k, klen, v, vlen);
1145 				}
1146 				continue;
1147 			}
1148 			CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
1149 			    -1, (sizeof(size_t) << 3) - 1)
1150 			CONF_HANDLE_SSIZE_T(opt_decay_time, "decay_time", -1,
1151 			    NSTIME_SEC_MAX);
1152 			CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true)
1153 			if (config_fill) {
1154 				if (CONF_MATCH("junk")) {
1155 					if (CONF_MATCH_VALUE("true")) {
1156 						opt_junk = "true";
1157 						opt_junk_alloc = opt_junk_free =
1158 						    true;
1159 					} else if (CONF_MATCH_VALUE("false")) {
1160 						opt_junk = "false";
1161 						opt_junk_alloc = opt_junk_free =
1162 						    false;
1163 					} else if (CONF_MATCH_VALUE("alloc")) {
1164 						opt_junk = "alloc";
1165 						opt_junk_alloc = true;
1166 						opt_junk_free = false;
1167 					} else if (CONF_MATCH_VALUE("free")) {
1168 						opt_junk = "free";
1169 						opt_junk_alloc = false;
1170 						opt_junk_free = true;
1171 					} else {
1172 						malloc_conf_error(
1173 						    "Invalid conf value", k,
1174 						    klen, v, vlen);
1175 					}
1176 					continue;
1177 				}
1178 				CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
1179 				    0, SIZE_T_MAX, false)
1180 				CONF_HANDLE_BOOL(opt_redzone, "redzone", true)
1181 				CONF_HANDLE_BOOL(opt_zero, "zero", true)
1182 			}
1183 			if (config_utrace) {
1184 				CONF_HANDLE_BOOL(opt_utrace, "utrace", true)
1185 			}
1186 			if (config_xmalloc) {
1187 				CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true)
1188 			}
1189 			if (config_tcache) {
1190 				CONF_HANDLE_BOOL(opt_tcache, "tcache",
1191 				    !config_valgrind || !in_valgrind)
1192 				if (CONF_MATCH("tcache")) {
1193 					assert(config_valgrind && in_valgrind);
1194 					if (opt_tcache) {
1195 						opt_tcache = false;
1196 						malloc_conf_error(
1197 						"tcache cannot be enabled "
1198 						"while running inside Valgrind",
1199 						k, klen, v, vlen);
1200 					}
1201 					continue;
1202 				}
1203 				CONF_HANDLE_SSIZE_T(opt_lg_tcache_max,
1204 				    "lg_tcache_max", -1,
1205 				    (sizeof(size_t) << 3) - 1)
1206 			}
1207 			if (config_prof) {
1208 				CONF_HANDLE_BOOL(opt_prof, "prof", true)
1209 				CONF_HANDLE_CHAR_P(opt_prof_prefix,
1210 				    "prof_prefix", "jeprof")
1211 				CONF_HANDLE_BOOL(opt_prof_active, "prof_active",
1212 				    true)
1213 				CONF_HANDLE_BOOL(opt_prof_thread_active_init,
1214 				    "prof_thread_active_init", true)
1215 				CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
1216 				    "lg_prof_sample", 0,
1217 				    (sizeof(uint64_t) << 3) - 1, true)
1218 				CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum",
1219 				    true)
1220 				CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
1221 				    "lg_prof_interval", -1,
1222 				    (sizeof(uint64_t) << 3) - 1)
1223 				CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump",
1224 				    true)
1225 				CONF_HANDLE_BOOL(opt_prof_final, "prof_final",
1226 				    true)
1227 				CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak",
1228 				    true)
1229 			}
1230 			malloc_conf_error("Invalid conf pair", k, klen, v,
1231 			    vlen);
1232 #undef CONF_MATCH
1233 #undef CONF_HANDLE_BOOL
1234 #undef CONF_HANDLE_SIZE_T
1235 #undef CONF_HANDLE_SSIZE_T
1236 #undef CONF_HANDLE_CHAR_P
1237 		}
1238 	}
1239 }
1240 
1241 static bool
1242 malloc_init_hard_needed(void)
1243 {
1244 
1245 	if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state ==
1246 	    malloc_init_recursible)) {
1247 		/*
1248 		 * Another thread initialized the allocator before this one
1249 		 * acquired init_lock, or this thread is the initializing
1250 		 * thread, and it is recursively allocating.
1251 		 */
1252 		return (false);
1253 	}
1254 #ifdef JEMALLOC_THREADED_INIT
1255 	if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
1256 		/* Busy-wait until the initializing thread completes. */
1257 		do {
1258 			malloc_mutex_unlock(NULL, &init_lock);
1259 			CPU_SPINWAIT;
1260 			malloc_mutex_lock(NULL, &init_lock);
1261 		} while (!malloc_initialized());
1262 		return (false);
1263 	}
1264 #endif
1265 	return (true);
1266 }
1267 
1268 static bool
1269 malloc_init_hard_a0_locked()
1270 {
1271 
1272 	malloc_initializer = INITIALIZER;
1273 
1274 	if (config_prof)
1275 		prof_boot0();
1276 	malloc_conf_init();
1277 	if (opt_stats_print) {
1278 		/* Print statistics at exit. */
1279 		if (atexit(stats_print_atexit) != 0) {
1280 			malloc_write("<jemalloc>: Error in atexit()\n");
1281 			if (opt_abort)
1282 				abort();
1283 		}
1284 	}
1285 	pages_boot();
1286 	if (base_boot())
1287 		return (true);
1288 	if (chunk_boot())
1289 		return (true);
1290 	if (ctl_boot())
1291 		return (true);
1292 	if (config_prof)
1293 		prof_boot1();
1294 	if (arena_boot())
1295 		return (true);
1296 	if (config_tcache && tcache_boot(TSDN_NULL))
1297 		return (true);
1298 	if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS))
1299 		return (true);
1300 	/*
1301 	 * Create enough scaffolding to allow recursive allocation in
1302 	 * malloc_ncpus().
1303 	 */
1304 	narenas_auto = 1;
1305 	narenas_total_set(narenas_auto);
1306 	arenas = &a0;
1307 	memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
1308 	/*
1309 	 * Initialize one arena here.  The rest are lazily created in
1310 	 * arena_choose_hard().
1311 	 */
1312 	if (arena_init(TSDN_NULL, 0) == NULL)
1313 		return (true);
1314 
1315 	malloc_init_state = malloc_init_a0_initialized;
1316 
1317 	return (false);
1318 }
1319 
1320 static bool
1321 malloc_init_hard_a0(void)
1322 {
1323 	bool ret;
1324 
1325 	malloc_mutex_lock(TSDN_NULL, &init_lock);
1326 	ret = malloc_init_hard_a0_locked();
1327 	malloc_mutex_unlock(TSDN_NULL, &init_lock);
1328 	return (ret);
1329 }
1330 
1331 /* Initialize data structures which may trigger recursive allocation. */
1332 static bool
1333 malloc_init_hard_recursible(void)
1334 {
1335 
1336 	malloc_init_state = malloc_init_recursible;
1337 
1338 	ncpus = malloc_ncpus();
1339 
1340 #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \
1341     && !defined(_WIN32) && !defined(__native_client__))
1342 	/* LinuxThreads' pthread_atfork() allocates. */
1343 	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
1344 	    jemalloc_postfork_child) != 0) {
1345 		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
1346 		if (opt_abort)
1347 			abort();
1348 		return (true);
1349 	}
1350 #endif
1351 
1352 	return (false);
1353 }
1354 
1355 static bool
1356 malloc_init_hard_finish(tsdn_t *tsdn)
1357 {
1358 
1359 	if (malloc_mutex_boot())
1360 		return (true);
1361 
1362 	if (opt_narenas == 0) {
1363 		/*
1364 		 * For SMP systems, create more than one arena per CPU by
1365 		 * default.
1366 		 */
1367 		if (ncpus > 1)
1368 			opt_narenas = ncpus << 2;
1369 		else
1370 			opt_narenas = 1;
1371 	}
1372 	narenas_auto = opt_narenas;
1373 	/*
1374 	 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
1375 	 */
1376 	if (narenas_auto > MALLOCX_ARENA_MAX) {
1377 		narenas_auto = MALLOCX_ARENA_MAX;
1378 		malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
1379 		    narenas_auto);
1380 	}
1381 	narenas_total_set(narenas_auto);
1382 
1383 	/* Allocate and initialize arenas. */
1384 	arenas = (arena_t **)base_alloc(tsdn, sizeof(arena_t *) *
1385 	    (MALLOCX_ARENA_MAX+1));
1386 	if (arenas == NULL)
1387 		return (true);
1388 	/* Copy the pointer to the one arena that was already initialized. */
1389 	arena_set(0, a0);
1390 
1391 	malloc_init_state = malloc_init_initialized;
1392 	malloc_slow_flag_init();
1393 
1394 	return (false);
1395 }
1396 
1397 static bool
1398 malloc_init_hard(void)
1399 {
1400 	tsd_t *tsd;
1401 
1402 #if defined(_WIN32) && _WIN32_WINNT < 0x0600
1403 	_init_init_lock();
1404 #endif
1405 	malloc_mutex_lock(TSDN_NULL, &init_lock);
1406 	if (!malloc_init_hard_needed()) {
1407 		malloc_mutex_unlock(TSDN_NULL, &init_lock);
1408 		return (false);
1409 	}
1410 
1411 	if (malloc_init_state != malloc_init_a0_initialized &&
1412 	    malloc_init_hard_a0_locked()) {
1413 		malloc_mutex_unlock(TSDN_NULL, &init_lock);
1414 		return (true);
1415 	}
1416 
1417 	malloc_mutex_unlock(TSDN_NULL, &init_lock);
1418 	/* Recursive allocation relies on functional tsd. */
1419 	tsd = malloc_tsd_boot0();
1420 	if (tsd == NULL)
1421 		return (true);
1422 	if (malloc_init_hard_recursible())
1423 		return (true);
1424 	malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
1425 
1426 	if (config_prof && prof_boot2(tsd_tsdn(tsd))) {
1427 		malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1428 		return (true);
1429 	}
1430 
1431 	if (malloc_init_hard_finish(tsd_tsdn(tsd))) {
1432 		malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1433 		return (true);
1434 	}
1435 
1436 	malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1437 	malloc_tsd_boot1();
1438 	return (false);
1439 }
1440 
1441 /*
1442  * End initialization functions.
1443  */
1444 /******************************************************************************/
1445 /*
1446  * Begin malloc(3)-compatible functions.
1447  */
1448 
1449 static void *
1450 ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero,
1451     prof_tctx_t *tctx, bool slow_path)
1452 {
1453 	void *p;
1454 
1455 	if (tctx == NULL)
1456 		return (NULL);
1457 	if (usize <= SMALL_MAXCLASS) {
1458 		szind_t ind_large = size2index(LARGE_MINCLASS);
1459 		p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path);
1460 		if (p == NULL)
1461 			return (NULL);
1462 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1463 	} else
1464 		p = ialloc(tsd, usize, ind, zero, slow_path);
1465 
1466 	return (p);
1467 }
1468 
1469 JEMALLOC_ALWAYS_INLINE_C void *
1470 ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path)
1471 {
1472 	void *p;
1473 	prof_tctx_t *tctx;
1474 
1475 	tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
1476 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
1477 		p = ialloc_prof_sample(tsd, usize, ind, zero, tctx, slow_path);
1478 	else
1479 		p = ialloc(tsd, usize, ind, zero, slow_path);
1480 	if (unlikely(p == NULL)) {
1481 		prof_alloc_rollback(tsd, tctx, true);
1482 		return (NULL);
1483 	}
1484 	prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
1485 
1486 	return (p);
1487 }
1488 
1489 /*
1490  * ialloc_body() is inlined so that fast and slow paths are generated separately
1491  * with statically known slow_path.
1492  *
1493  * This function guarantees that *tsdn is non-NULL on success.
1494  */
1495 JEMALLOC_ALWAYS_INLINE_C void *
1496 ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize,
1497     bool slow_path)
1498 {
1499 	tsd_t *tsd;
1500 	szind_t ind;
1501 
1502 	if (slow_path && unlikely(malloc_init())) {
1503 		*tsdn = NULL;
1504 		return (NULL);
1505 	}
1506 
1507 	tsd = tsd_fetch();
1508 	*tsdn = tsd_tsdn(tsd);
1509 	witness_assert_lockless(tsd_tsdn(tsd));
1510 
1511 	ind = size2index(size);
1512 	if (unlikely(ind >= NSIZES))
1513 		return (NULL);
1514 
1515 	if (config_stats || (config_prof && opt_prof) || (slow_path &&
1516 	    config_valgrind && unlikely(in_valgrind))) {
1517 		*usize = index2size(ind);
1518 		assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
1519 	}
1520 
1521 	if (config_prof && opt_prof)
1522 		return (ialloc_prof(tsd, *usize, ind, zero, slow_path));
1523 
1524 	return (ialloc(tsd, size, ind, zero, slow_path));
1525 }
1526 
1527 JEMALLOC_ALWAYS_INLINE_C void
1528 ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func,
1529     bool update_errno, bool slow_path)
1530 {
1531 
1532 	assert(!tsdn_null(tsdn) || ret == NULL);
1533 
1534 	if (unlikely(ret == NULL)) {
1535 		if (slow_path && config_xmalloc && unlikely(opt_xmalloc)) {
1536 			malloc_printf("<jemalloc>: Error in %s(): out of "
1537 			    "memory\n", func);
1538 			abort();
1539 		}
1540 		if (update_errno)
1541 			set_errno(ENOMEM);
1542 	}
1543 	if (config_stats && likely(ret != NULL)) {
1544 		assert(usize == isalloc(tsdn, ret, config_prof));
1545 		*tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize;
1546 	}
1547 	witness_assert_lockless(tsdn);
1548 }
1549 
1550 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1551 void JEMALLOC_NOTHROW *
1552 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
1553 je_malloc(size_t size)
1554 {
1555 	void *ret;
1556 	tsdn_t *tsdn;
1557 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1558 
1559 	if (size == 0)
1560 		size = 1;
1561 
1562 	if (likely(!malloc_slow)) {
1563 		ret = ialloc_body(size, false, &tsdn, &usize, false);
1564 		ialloc_post_check(ret, tsdn, usize, "malloc", true, false);
1565 	} else {
1566 		ret = ialloc_body(size, false, &tsdn, &usize, true);
1567 		ialloc_post_check(ret, tsdn, usize, "malloc", true, true);
1568 		UTRACE(0, size, ret);
1569 		JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, false);
1570 	}
1571 
1572 	return (ret);
1573 }
1574 
1575 static void *
1576 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
1577     prof_tctx_t *tctx)
1578 {
1579 	void *p;
1580 
1581 	if (tctx == NULL)
1582 		return (NULL);
1583 	if (usize <= SMALL_MAXCLASS) {
1584 		assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS);
1585 		p = ipalloc(tsd, LARGE_MINCLASS, alignment, false);
1586 		if (p == NULL)
1587 			return (NULL);
1588 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1589 	} else
1590 		p = ipalloc(tsd, usize, alignment, false);
1591 
1592 	return (p);
1593 }
1594 
1595 JEMALLOC_ALWAYS_INLINE_C void *
1596 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
1597 {
1598 	void *p;
1599 	prof_tctx_t *tctx;
1600 
1601 	tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
1602 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
1603 		p = imemalign_prof_sample(tsd, alignment, usize, tctx);
1604 	else
1605 		p = ipalloc(tsd, usize, alignment, false);
1606 	if (unlikely(p == NULL)) {
1607 		prof_alloc_rollback(tsd, tctx, true);
1608 		return (NULL);
1609 	}
1610 	prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
1611 
1612 	return (p);
1613 }
1614 
1615 JEMALLOC_ATTR(nonnull(1))
1616 static int
1617 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
1618 {
1619 	int ret;
1620 	tsd_t *tsd;
1621 	size_t usize;
1622 	void *result;
1623 
1624 	assert(min_alignment != 0);
1625 
1626 	if (unlikely(malloc_init())) {
1627 		tsd = NULL;
1628 		result = NULL;
1629 		goto label_oom;
1630 	}
1631 	tsd = tsd_fetch();
1632 	witness_assert_lockless(tsd_tsdn(tsd));
1633 	if (size == 0)
1634 		size = 1;
1635 
1636 	/* Make sure that alignment is a large enough power of 2. */
1637 	if (unlikely(((alignment - 1) & alignment) != 0
1638 	    || (alignment < min_alignment))) {
1639 		if (config_xmalloc && unlikely(opt_xmalloc)) {
1640 			malloc_write("<jemalloc>: Error allocating "
1641 			    "aligned memory: invalid alignment\n");
1642 			abort();
1643 		}
1644 		result = NULL;
1645 		ret = EINVAL;
1646 		goto label_return;
1647 	}
1648 
1649 	usize = sa2u(size, alignment);
1650 	if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) {
1651 		result = NULL;
1652 		goto label_oom;
1653 	}
1654 
1655 	if (config_prof && opt_prof)
1656 		result = imemalign_prof(tsd, alignment, usize);
1657 	else
1658 		result = ipalloc(tsd, usize, alignment, false);
1659 	if (unlikely(result == NULL))
1660 		goto label_oom;
1661 	assert(((uintptr_t)result & (alignment - 1)) == ZU(0));
1662 
1663 	*memptr = result;
1664 	ret = 0;
1665 label_return:
1666 	if (config_stats && likely(result != NULL)) {
1667 		assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof));
1668 		*tsd_thread_allocatedp_get(tsd) += usize;
1669 	}
1670 	UTRACE(0, size, result);
1671 	JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize,
1672 	    false);
1673 	witness_assert_lockless(tsd_tsdn(tsd));
1674 	return (ret);
1675 label_oom:
1676 	assert(result == NULL);
1677 	if (config_xmalloc && unlikely(opt_xmalloc)) {
1678 		malloc_write("<jemalloc>: Error allocating aligned memory: "
1679 		    "out of memory\n");
1680 		abort();
1681 	}
1682 	ret = ENOMEM;
1683 	witness_assert_lockless(tsd_tsdn(tsd));
1684 	goto label_return;
1685 }
1686 
1687 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
1688 JEMALLOC_ATTR(nonnull(1))
1689 je_posix_memalign(void **memptr, size_t alignment, size_t size)
1690 {
1691 	int ret;
1692 
1693 	ret = imemalign(memptr, alignment, size, sizeof(void *));
1694 
1695 	return (ret);
1696 }
1697 
1698 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1699 void JEMALLOC_NOTHROW *
1700 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2)
1701 je_aligned_alloc(size_t alignment, size_t size)
1702 {
1703 	void *ret;
1704 	int err;
1705 
1706 	if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) {
1707 		ret = NULL;
1708 		set_errno(err);
1709 	}
1710 
1711 	return (ret);
1712 }
1713 
1714 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1715 void JEMALLOC_NOTHROW *
1716 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2)
1717 je_calloc(size_t num, size_t size)
1718 {
1719 	void *ret;
1720 	tsdn_t *tsdn;
1721 	size_t num_size;
1722 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1723 
1724 	num_size = num * size;
1725 	if (unlikely(num_size == 0)) {
1726 		if (num == 0 || size == 0)
1727 			num_size = 1;
1728 		else
1729 			num_size = HUGE_MAXCLASS + 1; /* Trigger OOM. */
1730 	/*
1731 	 * Try to avoid division here.  We know that it isn't possible to
1732 	 * overflow during multiplication if neither operand uses any of the
1733 	 * most significant half of the bits in a size_t.
1734 	 */
1735 	} else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) <<
1736 	    2))) && (num_size / size != num)))
1737 		num_size = HUGE_MAXCLASS + 1; /* size_t overflow. */
1738 
1739 	if (likely(!malloc_slow)) {
1740 		ret = ialloc_body(num_size, true, &tsdn, &usize, false);
1741 		ialloc_post_check(ret, tsdn, usize, "calloc", true, false);
1742 	} else {
1743 		ret = ialloc_body(num_size, true, &tsdn, &usize, true);
1744 		ialloc_post_check(ret, tsdn, usize, "calloc", true, true);
1745 		UTRACE(0, num_size, ret);
1746 		JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, true);
1747 	}
1748 
1749 	return (ret);
1750 }
1751 
1752 static void *
1753 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
1754     prof_tctx_t *tctx)
1755 {
1756 	void *p;
1757 
1758 	if (tctx == NULL)
1759 		return (NULL);
1760 	if (usize <= SMALL_MAXCLASS) {
1761 		p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false);
1762 		if (p == NULL)
1763 			return (NULL);
1764 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1765 	} else
1766 		p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
1767 
1768 	return (p);
1769 }
1770 
1771 JEMALLOC_ALWAYS_INLINE_C void *
1772 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
1773 {
1774 	void *p;
1775 	bool prof_active;
1776 	prof_tctx_t *old_tctx, *tctx;
1777 
1778 	prof_active = prof_active_get_unlocked();
1779 	old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
1780 	tctx = prof_alloc_prep(tsd, usize, prof_active, true);
1781 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
1782 		p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx);
1783 	else
1784 		p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
1785 	if (unlikely(p == NULL)) {
1786 		prof_alloc_rollback(tsd, tctx, true);
1787 		return (NULL);
1788 	}
1789 	prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
1790 	    old_tctx);
1791 
1792 	return (p);
1793 }
1794 
1795 JEMALLOC_INLINE_C void
1796 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
1797 {
1798 	size_t usize;
1799 	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1800 
1801 	witness_assert_lockless(tsd_tsdn(tsd));
1802 
1803 	assert(ptr != NULL);
1804 	assert(malloc_initialized() || IS_INITIALIZER);
1805 
1806 	if (config_prof && opt_prof) {
1807 		usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1808 		prof_free(tsd, ptr, usize);
1809 	} else if (config_stats || config_valgrind)
1810 		usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1811 	if (config_stats)
1812 		*tsd_thread_deallocatedp_get(tsd) += usize;
1813 
1814 	if (likely(!slow_path))
1815 		iqalloc(tsd, ptr, tcache, false);
1816 	else {
1817 		if (config_valgrind && unlikely(in_valgrind))
1818 			rzsize = p2rz(tsd_tsdn(tsd), ptr);
1819 		iqalloc(tsd, ptr, tcache, true);
1820 		JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1821 	}
1822 }
1823 
1824 JEMALLOC_INLINE_C void
1825 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
1826 {
1827 	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1828 
1829 	witness_assert_lockless(tsd_tsdn(tsd));
1830 
1831 	assert(ptr != NULL);
1832 	assert(malloc_initialized() || IS_INITIALIZER);
1833 
1834 	if (config_prof && opt_prof)
1835 		prof_free(tsd, ptr, usize);
1836 	if (config_stats)
1837 		*tsd_thread_deallocatedp_get(tsd) += usize;
1838 	if (config_valgrind && unlikely(in_valgrind))
1839 		rzsize = p2rz(tsd_tsdn(tsd), ptr);
1840 	isqalloc(tsd, ptr, usize, tcache, slow_path);
1841 	JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1842 }
1843 
1844 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1845 void JEMALLOC_NOTHROW *
1846 JEMALLOC_ALLOC_SIZE(2)
1847 je_realloc(void *ptr, size_t size)
1848 {
1849 	void *ret;
1850 	tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL);
1851 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1852 	size_t old_usize = 0;
1853 	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1854 
1855 	if (unlikely(size == 0)) {
1856 		if (ptr != NULL) {
1857 			tsd_t *tsd;
1858 
1859 			/* realloc(ptr, 0) is equivalent to free(ptr). */
1860 			UTRACE(ptr, 0, 0);
1861 			tsd = tsd_fetch();
1862 			ifree(tsd, ptr, tcache_get(tsd, false), true);
1863 			return (NULL);
1864 		}
1865 		size = 1;
1866 	}
1867 
1868 	if (likely(ptr != NULL)) {
1869 		tsd_t *tsd;
1870 
1871 		assert(malloc_initialized() || IS_INITIALIZER);
1872 		malloc_thread_init();
1873 		tsd = tsd_fetch();
1874 
1875 		witness_assert_lockless(tsd_tsdn(tsd));
1876 
1877 		old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1878 		if (config_valgrind && unlikely(in_valgrind)) {
1879 			old_rzsize = config_prof ? p2rz(tsd_tsdn(tsd), ptr) :
1880 			    u2rz(old_usize);
1881 		}
1882 
1883 		if (config_prof && opt_prof) {
1884 			usize = s2u(size);
1885 			ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ?
1886 			    NULL : irealloc_prof(tsd, ptr, old_usize, usize);
1887 		} else {
1888 			if (config_stats || (config_valgrind &&
1889 			    unlikely(in_valgrind)))
1890 				usize = s2u(size);
1891 			ret = iralloc(tsd, ptr, old_usize, size, 0, false);
1892 		}
1893 		tsdn = tsd_tsdn(tsd);
1894 	} else {
1895 		/* realloc(NULL, size) is equivalent to malloc(size). */
1896 		if (likely(!malloc_slow))
1897 			ret = ialloc_body(size, false, &tsdn, &usize, false);
1898 		else
1899 			ret = ialloc_body(size, false, &tsdn, &usize, true);
1900 		assert(!tsdn_null(tsdn) || ret == NULL);
1901 	}
1902 
1903 	if (unlikely(ret == NULL)) {
1904 		if (config_xmalloc && unlikely(opt_xmalloc)) {
1905 			malloc_write("<jemalloc>: Error in realloc(): "
1906 			    "out of memory\n");
1907 			abort();
1908 		}
1909 		set_errno(ENOMEM);
1910 	}
1911 	if (config_stats && likely(ret != NULL)) {
1912 		tsd_t *tsd;
1913 
1914 		assert(usize == isalloc(tsdn, ret, config_prof));
1915 		tsd = tsdn_tsd(tsdn);
1916 		*tsd_thread_allocatedp_get(tsd) += usize;
1917 		*tsd_thread_deallocatedp_get(tsd) += old_usize;
1918 	}
1919 	UTRACE(ptr, size, ret);
1920 	JEMALLOC_VALGRIND_REALLOC(true, tsdn, ret, usize, true, ptr, old_usize,
1921 	    old_rzsize, true, false);
1922 	witness_assert_lockless(tsdn);
1923 	return (ret);
1924 }
1925 
1926 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
1927 je_free(void *ptr)
1928 {
1929 
1930 	UTRACE(ptr, 0, 0);
1931 	if (likely(ptr != NULL)) {
1932 		tsd_t *tsd = tsd_fetch();
1933 		witness_assert_lockless(tsd_tsdn(tsd));
1934 		if (likely(!malloc_slow))
1935 			ifree(tsd, ptr, tcache_get(tsd, false), false);
1936 		else
1937 			ifree(tsd, ptr, tcache_get(tsd, false), true);
1938 		witness_assert_lockless(tsd_tsdn(tsd));
1939 	}
1940 }
1941 
1942 /*
1943  * End malloc(3)-compatible functions.
1944  */
1945 /******************************************************************************/
1946 /*
1947  * Begin non-standard override functions.
1948  */
1949 
1950 #ifdef JEMALLOC_OVERRIDE_MEMALIGN
1951 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1952 void JEMALLOC_NOTHROW *
1953 JEMALLOC_ATTR(malloc)
1954 je_memalign(size_t alignment, size_t size)
1955 {
1956 	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1957 	if (unlikely(imemalign(&ret, alignment, size, 1) != 0))
1958 		ret = NULL;
1959 	return (ret);
1960 }
1961 #endif
1962 
1963 #ifdef JEMALLOC_OVERRIDE_VALLOC
1964 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1965 void JEMALLOC_NOTHROW *
1966 JEMALLOC_ATTR(malloc)
1967 je_valloc(size_t size)
1968 {
1969 	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1970 	if (unlikely(imemalign(&ret, PAGE, size, 1) != 0))
1971 		ret = NULL;
1972 	return (ret);
1973 }
1974 #endif
1975 
1976 /*
1977  * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has
1978  * #define je_malloc malloc
1979  */
1980 #define	malloc_is_malloc 1
1981 #define	is_malloc_(a) malloc_is_ ## a
1982 #define	is_malloc(a) is_malloc_(a)
1983 
1984 #if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK))
1985 /*
1986  * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
1987  * to inconsistently reference libc's malloc(3)-compatible functions
1988  * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
1989  *
1990  * These definitions interpose hooks in glibc.  The functions are actually
1991  * passed an extra argument for the caller return address, which will be
1992  * ignored.
1993  */
1994 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
1995 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
1996 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
1997 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK
1998 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
1999     je_memalign;
2000 # endif
2001 #endif
2002 
2003 /*
2004  * End non-standard override functions.
2005  */
2006 /******************************************************************************/
2007 /*
2008  * Begin non-standard functions.
2009  */
2010 
2011 JEMALLOC_ALWAYS_INLINE_C bool
2012 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize,
2013     size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena)
2014 {
2015 
2016 	if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) {
2017 		*alignment = 0;
2018 		*usize = s2u(size);
2019 	} else {
2020 		*alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
2021 		*usize = sa2u(size, *alignment);
2022 	}
2023 	if (unlikely(*usize == 0 || *usize > HUGE_MAXCLASS))
2024 		return (true);
2025 	*zero = MALLOCX_ZERO_GET(flags);
2026 	if ((flags & MALLOCX_TCACHE_MASK) != 0) {
2027 		if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
2028 			*tcache = NULL;
2029 		else
2030 			*tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2031 	} else
2032 		*tcache = tcache_get(tsd, true);
2033 	if ((flags & MALLOCX_ARENA_MASK) != 0) {
2034 		unsigned arena_ind = MALLOCX_ARENA_GET(flags);
2035 		*arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
2036 		if (unlikely(*arena == NULL))
2037 			return (true);
2038 	} else
2039 		*arena = NULL;
2040 	return (false);
2041 }
2042 
2043 JEMALLOC_ALWAYS_INLINE_C void *
2044 imallocx_flags(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
2045     tcache_t *tcache, arena_t *arena, bool slow_path)
2046 {
2047 	szind_t ind;
2048 
2049 	if (unlikely(alignment != 0))
2050 		return (ipalloct(tsdn, usize, alignment, zero, tcache, arena));
2051 	ind = size2index(usize);
2052 	assert(ind < NSIZES);
2053 	return (iallocztm(tsdn, usize, ind, zero, tcache, false, arena,
2054 	    slow_path));
2055 }
2056 
2057 static void *
2058 imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
2059     tcache_t *tcache, arena_t *arena, bool slow_path)
2060 {
2061 	void *p;
2062 
2063 	if (usize <= SMALL_MAXCLASS) {
2064 		assert(((alignment == 0) ? s2u(LARGE_MINCLASS) :
2065 		    sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS);
2066 		p = imallocx_flags(tsdn, LARGE_MINCLASS, alignment, zero,
2067 		    tcache, arena, slow_path);
2068 		if (p == NULL)
2069 			return (NULL);
2070 		arena_prof_promoted(tsdn, p, usize);
2071 	} else {
2072 		p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena,
2073 		    slow_path);
2074 	}
2075 
2076 	return (p);
2077 }
2078 
2079 JEMALLOC_ALWAYS_INLINE_C void *
2080 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path)
2081 {
2082 	void *p;
2083 	size_t alignment;
2084 	bool zero;
2085 	tcache_t *tcache;
2086 	arena_t *arena;
2087 	prof_tctx_t *tctx;
2088 
2089 	if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
2090 	    &zero, &tcache, &arena)))
2091 		return (NULL);
2092 	tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true);
2093 	if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
2094 		p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero,
2095 		    tcache, arena, slow_path);
2096 	} else if ((uintptr_t)tctx > (uintptr_t)1U) {
2097 		p = imallocx_prof_sample(tsd_tsdn(tsd), *usize, alignment, zero,
2098 		    tcache, arena, slow_path);
2099 	} else
2100 		p = NULL;
2101 	if (unlikely(p == NULL)) {
2102 		prof_alloc_rollback(tsd, tctx, true);
2103 		return (NULL);
2104 	}
2105 	prof_malloc(tsd_tsdn(tsd), p, *usize, tctx);
2106 
2107 	assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
2108 	return (p);
2109 }
2110 
2111 JEMALLOC_ALWAYS_INLINE_C void *
2112 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize,
2113     bool slow_path)
2114 {
2115 	void *p;
2116 	size_t alignment;
2117 	bool zero;
2118 	tcache_t *tcache;
2119 	arena_t *arena;
2120 
2121 	if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
2122 	    &zero, &tcache, &arena)))
2123 		return (NULL);
2124 	p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, tcache,
2125 	    arena, slow_path);
2126 	assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
2127 	return (p);
2128 }
2129 
2130 /* This function guarantees that *tsdn is non-NULL on success. */
2131 JEMALLOC_ALWAYS_INLINE_C void *
2132 imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize,
2133     bool slow_path)
2134 {
2135 	tsd_t *tsd;
2136 
2137 	if (slow_path && unlikely(malloc_init())) {
2138 		*tsdn = NULL;
2139 		return (NULL);
2140 	}
2141 
2142 	tsd = tsd_fetch();
2143 	*tsdn = tsd_tsdn(tsd);
2144 	witness_assert_lockless(tsd_tsdn(tsd));
2145 
2146 	if (likely(flags == 0)) {
2147 		szind_t ind = size2index(size);
2148 		if (unlikely(ind >= NSIZES))
2149 			return (NULL);
2150 		if (config_stats || (config_prof && opt_prof) || (slow_path &&
2151 		    config_valgrind && unlikely(in_valgrind))) {
2152 			*usize = index2size(ind);
2153 			assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
2154 		}
2155 
2156 		if (config_prof && opt_prof) {
2157 			return (ialloc_prof(tsd, *usize, ind, false,
2158 			    slow_path));
2159 		}
2160 
2161 		return (ialloc(tsd, size, ind, false, slow_path));
2162 	}
2163 
2164 	if (config_prof && opt_prof)
2165 		return (imallocx_prof(tsd, size, flags, usize, slow_path));
2166 
2167 	return (imallocx_no_prof(tsd, size, flags, usize, slow_path));
2168 }
2169 
2170 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2171 void JEMALLOC_NOTHROW *
2172 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
2173 je_mallocx(size_t size, int flags)
2174 {
2175 	tsdn_t *tsdn;
2176 	void *p;
2177 	size_t usize;
2178 
2179 	assert(size != 0);
2180 
2181 	if (likely(!malloc_slow)) {
2182 		p = imallocx_body(size, flags, &tsdn, &usize, false);
2183 		ialloc_post_check(p, tsdn, usize, "mallocx", false, false);
2184 	} else {
2185 		p = imallocx_body(size, flags, &tsdn, &usize, true);
2186 		ialloc_post_check(p, tsdn, usize, "mallocx", false, true);
2187 		UTRACE(0, size, p);
2188 		JEMALLOC_VALGRIND_MALLOC(p != NULL, tsdn, p, usize,
2189 		    MALLOCX_ZERO_GET(flags));
2190 	}
2191 
2192 	return (p);
2193 }
2194 
2195 static void *
2196 irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize,
2197     size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
2198     prof_tctx_t *tctx)
2199 {
2200 	void *p;
2201 
2202 	if (tctx == NULL)
2203 		return (NULL);
2204 	if (usize <= SMALL_MAXCLASS) {
2205 		p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment,
2206 		    zero, tcache, arena);
2207 		if (p == NULL)
2208 			return (NULL);
2209 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
2210 	} else {
2211 		p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero,
2212 		    tcache, arena);
2213 	}
2214 
2215 	return (p);
2216 }
2217 
2218 JEMALLOC_ALWAYS_INLINE_C void *
2219 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
2220     size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
2221     arena_t *arena)
2222 {
2223 	void *p;
2224 	bool prof_active;
2225 	prof_tctx_t *old_tctx, *tctx;
2226 
2227 	prof_active = prof_active_get_unlocked();
2228 	old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
2229 	tctx = prof_alloc_prep(tsd, *usize, prof_active, false);
2230 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
2231 		p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize,
2232 		    alignment, zero, tcache, arena, tctx);
2233 	} else {
2234 		p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero,
2235 		    tcache, arena);
2236 	}
2237 	if (unlikely(p == NULL)) {
2238 		prof_alloc_rollback(tsd, tctx, false);
2239 		return (NULL);
2240 	}
2241 
2242 	if (p == old_ptr && alignment != 0) {
2243 		/*
2244 		 * The allocation did not move, so it is possible that the size
2245 		 * class is smaller than would guarantee the requested
2246 		 * alignment, and that the alignment constraint was
2247 		 * serendipitously satisfied.  Additionally, old_usize may not
2248 		 * be the same as the current usize because of in-place large
2249 		 * reallocation.  Therefore, query the actual value of usize.
2250 		 */
2251 		*usize = isalloc(tsd_tsdn(tsd), p, config_prof);
2252 	}
2253 	prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr,
2254 	    old_usize, old_tctx);
2255 
2256 	return (p);
2257 }
2258 
2259 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2260 void JEMALLOC_NOTHROW *
2261 JEMALLOC_ALLOC_SIZE(2)
2262 je_rallocx(void *ptr, size_t size, int flags)
2263 {
2264 	void *p;
2265 	tsd_t *tsd;
2266 	size_t usize;
2267 	size_t old_usize;
2268 	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
2269 	size_t alignment = MALLOCX_ALIGN_GET(flags);
2270 	bool zero = flags & MALLOCX_ZERO;
2271 	arena_t *arena;
2272 	tcache_t *tcache;
2273 
2274 	assert(ptr != NULL);
2275 	assert(size != 0);
2276 	assert(malloc_initialized() || IS_INITIALIZER);
2277 	malloc_thread_init();
2278 	tsd = tsd_fetch();
2279 	witness_assert_lockless(tsd_tsdn(tsd));
2280 
2281 	if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
2282 		unsigned arena_ind = MALLOCX_ARENA_GET(flags);
2283 		arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
2284 		if (unlikely(arena == NULL))
2285 			goto label_oom;
2286 	} else
2287 		arena = NULL;
2288 
2289 	if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
2290 		if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
2291 			tcache = NULL;
2292 		else
2293 			tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2294 	} else
2295 		tcache = tcache_get(tsd, true);
2296 
2297 	old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
2298 	if (config_valgrind && unlikely(in_valgrind))
2299 		old_rzsize = u2rz(old_usize);
2300 
2301 	if (config_prof && opt_prof) {
2302 		usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
2303 		if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
2304 			goto label_oom;
2305 		p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
2306 		    zero, tcache, arena);
2307 		if (unlikely(p == NULL))
2308 			goto label_oom;
2309 	} else {
2310 		p = iralloct(tsd, ptr, old_usize, size, alignment, zero,
2311 		     tcache, arena);
2312 		if (unlikely(p == NULL))
2313 			goto label_oom;
2314 		if (config_stats || (config_valgrind && unlikely(in_valgrind)))
2315 			usize = isalloc(tsd_tsdn(tsd), p, config_prof);
2316 	}
2317 	assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
2318 
2319 	if (config_stats) {
2320 		*tsd_thread_allocatedp_get(tsd) += usize;
2321 		*tsd_thread_deallocatedp_get(tsd) += old_usize;
2322 	}
2323 	UTRACE(ptr, size, p);
2324 	JEMALLOC_VALGRIND_REALLOC(true, tsd_tsdn(tsd), p, usize, false, ptr,
2325 	    old_usize, old_rzsize, false, zero);
2326 	witness_assert_lockless(tsd_tsdn(tsd));
2327 	return (p);
2328 label_oom:
2329 	if (config_xmalloc && unlikely(opt_xmalloc)) {
2330 		malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
2331 		abort();
2332 	}
2333 	UTRACE(ptr, size, 0);
2334 	witness_assert_lockless(tsd_tsdn(tsd));
2335 	return (NULL);
2336 }
2337 
2338 JEMALLOC_ALWAYS_INLINE_C size_t
2339 ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
2340     size_t extra, size_t alignment, bool zero)
2341 {
2342 	size_t usize;
2343 
2344 	if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero))
2345 		return (old_usize);
2346 	usize = isalloc(tsdn, ptr, config_prof);
2347 
2348 	return (usize);
2349 }
2350 
2351 static size_t
2352 ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
2353     size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx)
2354 {
2355 	size_t usize;
2356 
2357 	if (tctx == NULL)
2358 		return (old_usize);
2359 	usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
2360 	    zero);
2361 
2362 	return (usize);
2363 }
2364 
2365 JEMALLOC_ALWAYS_INLINE_C size_t
2366 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
2367     size_t extra, size_t alignment, bool zero)
2368 {
2369 	size_t usize_max, usize;
2370 	bool prof_active;
2371 	prof_tctx_t *old_tctx, *tctx;
2372 
2373 	prof_active = prof_active_get_unlocked();
2374 	old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
2375 	/*
2376 	 * usize isn't knowable before ixalloc() returns when extra is non-zero.
2377 	 * Therefore, compute its maximum possible value and use that in
2378 	 * prof_alloc_prep() to decide whether to capture a backtrace.
2379 	 * prof_realloc() will use the actual usize to decide whether to sample.
2380 	 */
2381 	if (alignment == 0) {
2382 		usize_max = s2u(size+extra);
2383 		assert(usize_max > 0 && usize_max <= HUGE_MAXCLASS);
2384 	} else {
2385 		usize_max = sa2u(size+extra, alignment);
2386 		if (unlikely(usize_max == 0 || usize_max > HUGE_MAXCLASS)) {
2387 			/*
2388 			 * usize_max is out of range, and chances are that
2389 			 * allocation will fail, but use the maximum possible
2390 			 * value and carry on with prof_alloc_prep(), just in
2391 			 * case allocation succeeds.
2392 			 */
2393 			usize_max = HUGE_MAXCLASS;
2394 		}
2395 	}
2396 	tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
2397 
2398 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
2399 		usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
2400 		    size, extra, alignment, zero, tctx);
2401 	} else {
2402 		usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
2403 		    extra, alignment, zero);
2404 	}
2405 	if (usize == old_usize) {
2406 		prof_alloc_rollback(tsd, tctx, false);
2407 		return (usize);
2408 	}
2409 	prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
2410 	    old_tctx);
2411 
2412 	return (usize);
2413 }
2414 
2415 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
2416 je_xallocx(void *ptr, size_t size, size_t extra, int flags)
2417 {
2418 	tsd_t *tsd;
2419 	size_t usize, old_usize;
2420 	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
2421 	size_t alignment = MALLOCX_ALIGN_GET(flags);
2422 	bool zero = flags & MALLOCX_ZERO;
2423 
2424 	assert(ptr != NULL);
2425 	assert(size != 0);
2426 	assert(SIZE_T_MAX - size >= extra);
2427 	assert(malloc_initialized() || IS_INITIALIZER);
2428 	malloc_thread_init();
2429 	tsd = tsd_fetch();
2430 	witness_assert_lockless(tsd_tsdn(tsd));
2431 
2432 	old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
2433 
2434 	/*
2435 	 * The API explicitly absolves itself of protecting against (size +
2436 	 * extra) numerical overflow, but we may need to clamp extra to avoid
2437 	 * exceeding HUGE_MAXCLASS.
2438 	 *
2439 	 * Ordinarily, size limit checking is handled deeper down, but here we
2440 	 * have to check as part of (size + extra) clamping, since we need the
2441 	 * clamped value in the above helper functions.
2442 	 */
2443 	if (unlikely(size > HUGE_MAXCLASS)) {
2444 		usize = old_usize;
2445 		goto label_not_resized;
2446 	}
2447 	if (unlikely(HUGE_MAXCLASS - size < extra))
2448 		extra = HUGE_MAXCLASS - size;
2449 
2450 	if (config_valgrind && unlikely(in_valgrind))
2451 		old_rzsize = u2rz(old_usize);
2452 
2453 	if (config_prof && opt_prof) {
2454 		usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
2455 		    alignment, zero);
2456 	} else {
2457 		usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
2458 		    extra, alignment, zero);
2459 	}
2460 	if (unlikely(usize == old_usize))
2461 		goto label_not_resized;
2462 
2463 	if (config_stats) {
2464 		*tsd_thread_allocatedp_get(tsd) += usize;
2465 		*tsd_thread_deallocatedp_get(tsd) += old_usize;
2466 	}
2467 	JEMALLOC_VALGRIND_REALLOC(false, tsd_tsdn(tsd), ptr, usize, false, ptr,
2468 	    old_usize, old_rzsize, false, zero);
2469 label_not_resized:
2470 	UTRACE(ptr, size, ptr);
2471 	witness_assert_lockless(tsd_tsdn(tsd));
2472 	return (usize);
2473 }
2474 
2475 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
2476 JEMALLOC_ATTR(pure)
2477 je_sallocx(const void *ptr, int flags)
2478 {
2479 	size_t usize;
2480 	tsdn_t *tsdn;
2481 
2482 	assert(malloc_initialized() || IS_INITIALIZER);
2483 	malloc_thread_init();
2484 
2485 	tsdn = tsdn_fetch();
2486 	witness_assert_lockless(tsdn);
2487 
2488 	if (config_ivsalloc)
2489 		usize = ivsalloc(tsdn, ptr, config_prof);
2490 	else
2491 		usize = isalloc(tsdn, ptr, config_prof);
2492 
2493 	witness_assert_lockless(tsdn);
2494 	return (usize);
2495 }
2496 
2497 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
2498 je_dallocx(void *ptr, int flags)
2499 {
2500 	tsd_t *tsd;
2501 	tcache_t *tcache;
2502 
2503 	assert(ptr != NULL);
2504 	assert(malloc_initialized() || IS_INITIALIZER);
2505 
2506 	tsd = tsd_fetch();
2507 	witness_assert_lockless(tsd_tsdn(tsd));
2508 	if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
2509 		if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
2510 			tcache = NULL;
2511 		else
2512 			tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2513 	} else
2514 		tcache = tcache_get(tsd, false);
2515 
2516 	UTRACE(ptr, 0, 0);
2517 	if (likely(!malloc_slow))
2518 		ifree(tsd, ptr, tcache, false);
2519 	else
2520 		ifree(tsd, ptr, tcache, true);
2521 	witness_assert_lockless(tsd_tsdn(tsd));
2522 }
2523 
2524 JEMALLOC_ALWAYS_INLINE_C size_t
2525 inallocx(tsdn_t *tsdn, size_t size, int flags)
2526 {
2527 	size_t usize;
2528 
2529 	witness_assert_lockless(tsdn);
2530 
2531 	if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0))
2532 		usize = s2u(size);
2533 	else
2534 		usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
2535 	witness_assert_lockless(tsdn);
2536 	return (usize);
2537 }
2538 
2539 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
2540 je_sdallocx(void *ptr, size_t size, int flags)
2541 {
2542 	tsd_t *tsd;
2543 	tcache_t *tcache;
2544 	size_t usize;
2545 
2546 	assert(ptr != NULL);
2547 	assert(malloc_initialized() || IS_INITIALIZER);
2548 	tsd = tsd_fetch();
2549 	usize = inallocx(tsd_tsdn(tsd), size, flags);
2550 	assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof));
2551 
2552 	witness_assert_lockless(tsd_tsdn(tsd));
2553 	if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
2554 		if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
2555 			tcache = NULL;
2556 		else
2557 			tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2558 	} else
2559 		tcache = tcache_get(tsd, false);
2560 
2561 	UTRACE(ptr, 0, 0);
2562 	if (likely(!malloc_slow))
2563 		isfree(tsd, ptr, usize, tcache, false);
2564 	else
2565 		isfree(tsd, ptr, usize, tcache, true);
2566 	witness_assert_lockless(tsd_tsdn(tsd));
2567 }
2568 
2569 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
2570 JEMALLOC_ATTR(pure)
2571 je_nallocx(size_t size, int flags)
2572 {
2573 	size_t usize;
2574 	tsdn_t *tsdn;
2575 
2576 	assert(size != 0);
2577 
2578 	if (unlikely(malloc_init()))
2579 		return (0);
2580 
2581 	tsdn = tsdn_fetch();
2582 	witness_assert_lockless(tsdn);
2583 
2584 	usize = inallocx(tsdn, size, flags);
2585 	if (unlikely(usize > HUGE_MAXCLASS))
2586 		return (0);
2587 
2588 	witness_assert_lockless(tsdn);
2589 	return (usize);
2590 }
2591 
2592 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2593 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
2594     size_t newlen)
2595 {
2596 	int ret;
2597 	tsd_t *tsd;
2598 
2599 	if (unlikely(malloc_init()))
2600 		return (EAGAIN);
2601 
2602 	tsd = tsd_fetch();
2603 	witness_assert_lockless(tsd_tsdn(tsd));
2604 	ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
2605 	witness_assert_lockless(tsd_tsdn(tsd));
2606 	return (ret);
2607 }
2608 
2609 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2610 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
2611 {
2612 	int ret;
2613 	tsdn_t *tsdn;
2614 
2615 	if (unlikely(malloc_init()))
2616 		return (EAGAIN);
2617 
2618 	tsdn = tsdn_fetch();
2619 	witness_assert_lockless(tsdn);
2620 	ret = ctl_nametomib(tsdn, name, mibp, miblenp);
2621 	witness_assert_lockless(tsdn);
2622 	return (ret);
2623 }
2624 
2625 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2626 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
2627   void *newp, size_t newlen)
2628 {
2629 	int ret;
2630 	tsd_t *tsd;
2631 
2632 	if (unlikely(malloc_init()))
2633 		return (EAGAIN);
2634 
2635 	tsd = tsd_fetch();
2636 	witness_assert_lockless(tsd_tsdn(tsd));
2637 	ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
2638 	witness_assert_lockless(tsd_tsdn(tsd));
2639 	return (ret);
2640 }
2641 
2642 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
2643 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
2644     const char *opts)
2645 {
2646 	tsdn_t *tsdn;
2647 
2648 	tsdn = tsdn_fetch();
2649 	witness_assert_lockless(tsdn);
2650 	stats_print(write_cb, cbopaque, opts);
2651 	witness_assert_lockless(tsdn);
2652 }
2653 
2654 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
2655 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
2656 {
2657 	size_t ret;
2658 	tsdn_t *tsdn;
2659 
2660 	assert(malloc_initialized() || IS_INITIALIZER);
2661 	malloc_thread_init();
2662 
2663 	tsdn = tsdn_fetch();
2664 	witness_assert_lockless(tsdn);
2665 
2666 	if (config_ivsalloc)
2667 		ret = ivsalloc(tsdn, ptr, config_prof);
2668 	else
2669 		ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof);
2670 
2671 	witness_assert_lockless(tsdn);
2672 	return (ret);
2673 }
2674 
2675 /*
2676  * End non-standard functions.
2677  */
2678 /******************************************************************************/
2679 /*
2680  * Begin compatibility functions.
2681  */
2682 
2683 #define	ALLOCM_LG_ALIGN(la)	(la)
2684 #define	ALLOCM_ALIGN(a)		(ffsl(a)-1)
2685 #define	ALLOCM_ZERO		((int)0x40)
2686 #define	ALLOCM_NO_MOVE		((int)0x80)
2687 
2688 #define	ALLOCM_SUCCESS		0
2689 #define	ALLOCM_ERR_OOM		1
2690 #define	ALLOCM_ERR_NOT_MOVED	2
2691 
2692 int
2693 je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
2694 {
2695 	void *p;
2696 
2697 	assert(ptr != NULL);
2698 
2699 	p = je_mallocx(size, flags);
2700 	if (p == NULL)
2701 		return (ALLOCM_ERR_OOM);
2702 	if (rsize != NULL)
2703 		*rsize = isalloc(tsdn_fetch(), p, config_prof);
2704 	*ptr = p;
2705 	return (ALLOCM_SUCCESS);
2706 }
2707 
2708 int
2709 je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
2710 {
2711 	int ret;
2712 	bool no_move = flags & ALLOCM_NO_MOVE;
2713 
2714 	assert(ptr != NULL);
2715 	assert(*ptr != NULL);
2716 	assert(size != 0);
2717 	assert(SIZE_T_MAX - size >= extra);
2718 
2719 	if (no_move) {
2720 		size_t usize = je_xallocx(*ptr, size, extra, flags);
2721 		ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
2722 		if (rsize != NULL)
2723 			*rsize = usize;
2724 	} else {
2725 		void *p = je_rallocx(*ptr, size+extra, flags);
2726 		if (p != NULL) {
2727 			*ptr = p;
2728 			ret = ALLOCM_SUCCESS;
2729 		} else
2730 			ret = ALLOCM_ERR_OOM;
2731 		if (rsize != NULL)
2732 			*rsize = isalloc(tsdn_fetch(), *ptr, config_prof);
2733 	}
2734 	return (ret);
2735 }
2736 
2737 int
2738 je_sallocm(const void *ptr, size_t *rsize, int flags)
2739 {
2740 
2741 	assert(rsize != NULL);
2742 	*rsize = je_sallocx(ptr, flags);
2743 	return (ALLOCM_SUCCESS);
2744 }
2745 
2746 int
2747 je_dallocm(void *ptr, int flags)
2748 {
2749 
2750 	je_dallocx(ptr, flags);
2751 	return (ALLOCM_SUCCESS);
2752 }
2753 
2754 int
2755 je_nallocm(size_t *rsize, size_t size, int flags)
2756 {
2757 	size_t usize;
2758 
2759 	usize = je_nallocx(size, flags);
2760 	if (usize == 0)
2761 		return (ALLOCM_ERR_OOM);
2762 	if (rsize != NULL)
2763 		*rsize = usize;
2764 	return (ALLOCM_SUCCESS);
2765 }
2766 
2767 #undef ALLOCM_LG_ALIGN
2768 #undef ALLOCM_ALIGN
2769 #undef ALLOCM_ZERO
2770 #undef ALLOCM_NO_MOVE
2771 
2772 #undef ALLOCM_SUCCESS
2773 #undef ALLOCM_ERR_OOM
2774 #undef ALLOCM_ERR_NOT_MOVED
2775 
2776 /*
2777  * End compatibility functions.
2778  */
2779 /******************************************************************************/
2780 /*
2781  * The following functions are used by threading libraries for protection of
2782  * malloc during fork().
2783  */
2784 
2785 /*
2786  * If an application creates a thread before doing any allocation in the main
2787  * thread, then calls fork(2) in the main thread followed by memory allocation
2788  * in the child process, a race can occur that results in deadlock within the
2789  * child: the main thread may have forked while the created thread had
2790  * partially initialized the allocator.  Ordinarily jemalloc prevents
2791  * fork/malloc races via the following functions it registers during
2792  * initialization using pthread_atfork(), but of course that does no good if
2793  * the allocator isn't fully initialized at fork time.  The following library
2794  * constructor is a partial solution to this problem.  It may still be possible
2795  * to trigger the deadlock described above, but doing so would involve forking
2796  * via a library constructor that runs before jemalloc's runs.
2797  */
2798 #ifndef JEMALLOC_JET
2799 JEMALLOC_ATTR(constructor)
2800 static void
2801 jemalloc_constructor(void)
2802 {
2803 
2804 	malloc_init();
2805 }
2806 #endif
2807 
2808 #ifndef JEMALLOC_MUTEX_INIT_CB
2809 void
2810 jemalloc_prefork(void)
2811 #else
2812 JEMALLOC_EXPORT void
2813 _malloc_prefork(void)
2814 #endif
2815 {
2816 	tsd_t *tsd;
2817 	unsigned i, j, narenas;
2818 	arena_t *arena;
2819 
2820 #ifdef JEMALLOC_MUTEX_INIT_CB
2821 	if (!malloc_initialized())
2822 		return;
2823 #endif
2824 	assert(malloc_initialized());
2825 
2826 	tsd = tsd_fetch();
2827 
2828 	narenas = narenas_total_get();
2829 
2830 	witness_prefork(tsd);
2831 	/* Acquire all mutexes in a safe order. */
2832 	ctl_prefork(tsd_tsdn(tsd));
2833 	malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
2834 	prof_prefork0(tsd_tsdn(tsd));
2835 	for (i = 0; i < 3; i++) {
2836 		for (j = 0; j < narenas; j++) {
2837 			if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
2838 			    NULL) {
2839 				switch (i) {
2840 				case 0:
2841 					arena_prefork0(tsd_tsdn(tsd), arena);
2842 					break;
2843 				case 1:
2844 					arena_prefork1(tsd_tsdn(tsd), arena);
2845 					break;
2846 				case 2:
2847 					arena_prefork2(tsd_tsdn(tsd), arena);
2848 					break;
2849 				default: not_reached();
2850 				}
2851 			}
2852 		}
2853 	}
2854 	base_prefork(tsd_tsdn(tsd));
2855 	chunk_prefork(tsd_tsdn(tsd));
2856 	for (i = 0; i < narenas; i++) {
2857 		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
2858 			arena_prefork3(tsd_tsdn(tsd), arena);
2859 	}
2860 	prof_prefork1(tsd_tsdn(tsd));
2861 }
2862 
2863 #ifndef JEMALLOC_MUTEX_INIT_CB
2864 void
2865 jemalloc_postfork_parent(void)
2866 #else
2867 JEMALLOC_EXPORT void
2868 _malloc_postfork(void)
2869 #endif
2870 {
2871 	tsd_t *tsd;
2872 	unsigned i, narenas;
2873 
2874 #ifdef JEMALLOC_MUTEX_INIT_CB
2875 	if (!malloc_initialized())
2876 		return;
2877 #endif
2878 	assert(malloc_initialized());
2879 
2880 	tsd = tsd_fetch();
2881 
2882 	witness_postfork_parent(tsd);
2883 	/* Release all mutexes, now that fork() has completed. */
2884 	chunk_postfork_parent(tsd_tsdn(tsd));
2885 	base_postfork_parent(tsd_tsdn(tsd));
2886 	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
2887 		arena_t *arena;
2888 
2889 		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
2890 			arena_postfork_parent(tsd_tsdn(tsd), arena);
2891 	}
2892 	prof_postfork_parent(tsd_tsdn(tsd));
2893 	malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
2894 	ctl_postfork_parent(tsd_tsdn(tsd));
2895 }
2896 
2897 void
2898 jemalloc_postfork_child(void)
2899 {
2900 	tsd_t *tsd;
2901 	unsigned i, narenas;
2902 
2903 	assert(malloc_initialized());
2904 
2905 	tsd = tsd_fetch();
2906 
2907 	witness_postfork_child(tsd);
2908 	/* Release all mutexes, now that fork() has completed. */
2909 	chunk_postfork_child(tsd_tsdn(tsd));
2910 	base_postfork_child(tsd_tsdn(tsd));
2911 	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
2912 		arena_t *arena;
2913 
2914 		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
2915 			arena_postfork_child(tsd_tsdn(tsd), arena);
2916 	}
2917 	prof_postfork_child(tsd_tsdn(tsd));
2918 	malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
2919 	ctl_postfork_child(tsd_tsdn(tsd));
2920 }
2921 
2922 void
2923 _malloc_first_thread(void)
2924 {
2925 
2926 	(void)malloc_mutex_first_thread();
2927 }
2928 
2929 /******************************************************************************/
2930