xref: /freebsd/contrib/jemalloc/FREEBSD-diffs (revision f5e9c916afed4a948fe5c03bfaee038d165e12ab)
1diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
2index bc5dbd1..ba182da 100644
3--- a/doc/jemalloc.xml.in
4+++ b/doc/jemalloc.xml.in
5@@ -53,11 +53,23 @@
6     <para>This manual describes jemalloc @jemalloc_version@.  More information
7     can be found at the <ulink
8     url="http://www.canonware.com/jemalloc/">jemalloc website</ulink>.</para>
9+
10+    <para>The following configuration options are enabled in libc's built-in
11+    jemalloc: <option>--enable-fill</option>,
12+    <option>--enable-lazy-lock</option>, <option>--enable-munmap</option>,
13+    <option>--enable-stats</option>, <option>--enable-tcache</option>,
14+    <option>--enable-tls</option>, <option>--enable-utrace</option>, and
15+    <option>--enable-xmalloc</option>.  Additionally,
16+    <option>--enable-debug</option> is enabled in development versions of
17+    FreeBSD (controlled by the <constant>MALLOC_PRODUCTION</constant> make
18+    variable).</para>
19+
20   </refsect1>
21   <refsynopsisdiv>
22     <title>SYNOPSIS</title>
23     <funcsynopsis>
24-      <funcsynopsisinfo>#include &lt;<filename class="headerfile">jemalloc/jemalloc.h</filename>&gt;</funcsynopsisinfo>
25+      <funcsynopsisinfo>#include &lt;<filename class="headerfile">stdlib.h</filename>&gt;
26+#include &lt;<filename class="headerfile">malloc_np.h</filename>&gt;</funcsynopsisinfo>
27       <refsect2>
28         <title>Standard API</title>
29         <funcprototype>
30@@ -2905,4 +2917,18 @@ malloc_conf = "lg_chunk:24";]]></programlisting></para>
31     <para>The <function>posix_memalign<parameter/></function> function conforms
32     to IEEE Std 1003.1-2001 (&ldquo;POSIX.1&rdquo;).</para>
33   </refsect1>
34+  <refsect1 id="history">
35+    <title>HISTORY</title>
36+    <para>The <function>malloc_usable_size<parameter/></function> and
37+    <function>posix_memalign<parameter/></function> functions first appeared in
38+    FreeBSD 7.0.</para>
39+
40+    <para>The <function>aligned_alloc<parameter/></function>,
41+    <function>malloc_stats_print<parameter/></function>, and
42+    <function>mallctl*<parameter/></function> functions first appeared in
43+    FreeBSD 10.0.</para>
44+
45+    <para>The <function>*allocx<parameter/></function> functions first appeared
46+    in FreeBSD 11.0.</para>
47+  </refsect1>
48 </refentry>
49diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
50index 3f54391..d240256 100644
51--- a/include/jemalloc/internal/jemalloc_internal.h.in
52+++ b/include/jemalloc/internal/jemalloc_internal.h.in
53@@ -8,6 +8,9 @@
54 #include <sys/ktrace.h>
55 #endif
56
57+#include "un-namespace.h"
58+#include "libc_private.h"
59+
60 #define	JEMALLOC_NO_DEMANGLE
61 #ifdef JEMALLOC_JET
62 #  define JEMALLOC_N(n) jet_##n
63@@ -42,13 +45,7 @@ static const bool config_fill =
64     false
65 #endif
66     ;
67-static const bool config_lazy_lock =
68-#ifdef JEMALLOC_LAZY_LOCK
69-    true
70-#else
71-    false
72-#endif
73-    ;
74+static const bool config_lazy_lock = true;
75 static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF;
76 static const bool config_prof =
77 #ifdef JEMALLOC_PROF
78diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h
79index 2b8ca5d..42d97f2 100644
80--- a/include/jemalloc/internal/jemalloc_internal_decls.h
81+++ b/include/jemalloc/internal/jemalloc_internal_decls.h
82@@ -1,6 +1,9 @@
83 #ifndef JEMALLOC_INTERNAL_DECLS_H
84 #define	JEMALLOC_INTERNAL_DECLS_H
85
86+#include "libc_private.h"
87+#include "namespace.h"
88+
89 #include <math.h>
90 #ifdef _WIN32
91 #  include <windows.h>
92diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h
93index f051f29..561378f 100644
94--- a/include/jemalloc/internal/mutex.h
95+++ b/include/jemalloc/internal/mutex.h
96@@ -47,15 +47,13 @@ struct malloc_mutex_s {
97
98 #ifdef JEMALLOC_LAZY_LOCK
99 extern bool isthreaded;
100-#else
101-#  undef isthreaded /* Undo private_namespace.h definition. */
102-#  define isthreaded true
103 #endif
104
105 bool	malloc_mutex_init(malloc_mutex_t *mutex);
106 void	malloc_mutex_prefork(malloc_mutex_t *mutex);
107 void	malloc_mutex_postfork_parent(malloc_mutex_t *mutex);
108 void	malloc_mutex_postfork_child(malloc_mutex_t *mutex);
109+bool	malloc_mutex_first_thread(void);
110 bool	mutex_boot(void);
111
112 #endif /* JEMALLOC_H_EXTERNS */
113diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
114index 5880996..6e94e03 100644
115--- a/include/jemalloc/internal/private_symbols.txt
116+++ b/include/jemalloc/internal/private_symbols.txt
117@@ -296,7 +296,6 @@ iralloct_realign
118 isalloc
119 isdalloct
120 isqalloc
121-isthreaded
122 ivsalloc
123 ixalloc
124 jemalloc_postfork_child
125diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h
126new file mode 100644
127index 0000000..433dab5
128--- /dev/null
129+++ b/include/jemalloc/jemalloc_FreeBSD.h
130@@ -0,0 +1,160 @@
131+/*
132+ * Override settings that were generated in jemalloc_defs.h as necessary.
133+ */
134+
135+#undef JEMALLOC_OVERRIDE_VALLOC
136+
137+#ifndef MALLOC_PRODUCTION
138+#define	JEMALLOC_DEBUG
139+#endif
140+
141+/*
142+ * The following are architecture-dependent, so conditionally define them for
143+ * each supported architecture.
144+ */
145+#undef JEMALLOC_TLS_MODEL
146+#undef STATIC_PAGE_SHIFT
147+#undef LG_SIZEOF_PTR
148+#undef LG_SIZEOF_INT
149+#undef LG_SIZEOF_LONG
150+#undef LG_SIZEOF_INTMAX_T
151+
152+#ifdef __i386__
153+#  define LG_SIZEOF_PTR		2
154+#  define JEMALLOC_TLS_MODEL	__attribute__((tls_model("initial-exec")))
155+#endif
156+#ifdef __ia64__
157+#  define LG_SIZEOF_PTR		3
158+#endif
159+#ifdef __sparc64__
160+#  define LG_SIZEOF_PTR		3
161+#  define JEMALLOC_TLS_MODEL	__attribute__((tls_model("initial-exec")))
162+#endif
163+#ifdef __amd64__
164+#  define LG_SIZEOF_PTR		3
165+#  define JEMALLOC_TLS_MODEL	__attribute__((tls_model("initial-exec")))
166+#endif
167+#ifdef __arm__
168+#  define LG_SIZEOF_PTR		2
169+#endif
170+#ifdef __aarch64__
171+#  define LG_SIZEOF_PTR		3
172+#endif
173+#ifdef __mips__
174+#ifdef __mips_n64
175+#  define LG_SIZEOF_PTR		3
176+#else
177+#  define LG_SIZEOF_PTR		2
178+#endif
179+#endif
180+#ifdef __powerpc64__
181+#  define LG_SIZEOF_PTR		3
182+#elif defined(__powerpc__)
183+#  define LG_SIZEOF_PTR		2
184+#endif
185+#ifdef __riscv__
186+#  define LG_SIZEOF_PTR		3
187+#endif
188+
189+#ifndef JEMALLOC_TLS_MODEL
190+#  define JEMALLOC_TLS_MODEL	/* Default. */
191+#endif
192+
193+#define	STATIC_PAGE_SHIFT	PAGE_SHIFT
194+#define	LG_SIZEOF_INT		2
195+#define	LG_SIZEOF_LONG		LG_SIZEOF_PTR
196+#define	LG_SIZEOF_INTMAX_T	3
197+
198+#undef CPU_SPINWAIT
199+#include <machine/cpu.h>
200+#include <machine/cpufunc.h>
201+#define	CPU_SPINWAIT		cpu_spinwait()
202+
203+/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */
204+#undef JEMALLOC_LAZY_LOCK
205+extern int __isthreaded;
206+#define	isthreaded		((bool)__isthreaded)
207+
208+/* Mangle. */
209+#undef je_malloc
210+#undef je_calloc
211+#undef je_posix_memalign
212+#undef je_aligned_alloc
213+#undef je_realloc
214+#undef je_free
215+#undef je_malloc_usable_size
216+#undef je_mallocx
217+#undef je_rallocx
218+#undef je_xallocx
219+#undef je_sallocx
220+#undef je_dallocx
221+#undef je_sdallocx
222+#undef je_nallocx
223+#undef je_mallctl
224+#undef je_mallctlnametomib
225+#undef je_mallctlbymib
226+#undef je_malloc_stats_print
227+#undef je_allocm
228+#undef je_rallocm
229+#undef je_sallocm
230+#undef je_dallocm
231+#undef je_nallocm
232+#define	je_malloc		__malloc
233+#define	je_calloc		__calloc
234+#define	je_posix_memalign	__posix_memalign
235+#define	je_aligned_alloc	__aligned_alloc
236+#define	je_realloc		__realloc
237+#define	je_free			__free
238+#define	je_malloc_usable_size	__malloc_usable_size
239+#define	je_mallocx		__mallocx
240+#define	je_rallocx		__rallocx
241+#define	je_xallocx		__xallocx
242+#define	je_sallocx		__sallocx
243+#define	je_dallocx		__dallocx
244+#define	je_sdallocx		__sdallocx
245+#define	je_nallocx		__nallocx
246+#define	je_mallctl		__mallctl
247+#define	je_mallctlnametomib	__mallctlnametomib
248+#define	je_mallctlbymib		__mallctlbymib
249+#define	je_malloc_stats_print	__malloc_stats_print
250+#define	je_allocm		__allocm
251+#define	je_rallocm		__rallocm
252+#define	je_sallocm		__sallocm
253+#define	je_dallocm		__dallocm
254+#define	je_nallocm		__nallocm
255+#define	open			_open
256+#define	read			_read
257+#define	write			_write
258+#define	close			_close
259+#define	pthread_mutex_lock	_pthread_mutex_lock
260+#define	pthread_mutex_unlock	_pthread_mutex_unlock
261+
262+#ifdef JEMALLOC_C_
263+/*
264+ * Define 'weak' symbols so that an application can have its own versions
265+ * of malloc, calloc, realloc, free, et al.
266+ */
267+__weak_reference(__malloc, malloc);
268+__weak_reference(__calloc, calloc);
269+__weak_reference(__posix_memalign, posix_memalign);
270+__weak_reference(__aligned_alloc, aligned_alloc);
271+__weak_reference(__realloc, realloc);
272+__weak_reference(__free, free);
273+__weak_reference(__malloc_usable_size, malloc_usable_size);
274+__weak_reference(__mallocx, mallocx);
275+__weak_reference(__rallocx, rallocx);
276+__weak_reference(__xallocx, xallocx);
277+__weak_reference(__sallocx, sallocx);
278+__weak_reference(__dallocx, dallocx);
279+__weak_reference(__sdallocx, sdallocx);
280+__weak_reference(__nallocx, nallocx);
281+__weak_reference(__mallctl, mallctl);
282+__weak_reference(__mallctlnametomib, mallctlnametomib);
283+__weak_reference(__mallctlbymib, mallctlbymib);
284+__weak_reference(__malloc_stats_print, malloc_stats_print);
285+__weak_reference(__allocm, allocm);
286+__weak_reference(__rallocm, rallocm);
287+__weak_reference(__sallocm, sallocm);
288+__weak_reference(__dallocm, dallocm);
289+__weak_reference(__nallocm, nallocm);
290+#endif
291diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh
292index f943891..47d032c 100755
293--- a/include/jemalloc/jemalloc_rename.sh
294+++ b/include/jemalloc/jemalloc_rename.sh
295@@ -19,4 +19,6 @@ done
296
297 cat <<EOF
298 #endif
299+
300+#include "jemalloc_FreeBSD.h"
301 EOF
302diff --git a/src/jemalloc.c b/src/jemalloc.c
303index 0735376..a34b85c 100644
304--- a/src/jemalloc.c
305+++ b/src/jemalloc.c
306@@ -4,6 +4,10 @@
307 /******************************************************************************/
308 /* Data. */
309
310+/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
311+const char	*__malloc_options_1_0 = NULL;
312+__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
313+
314 /* Runtime configuration options. */
315 const char	*je_malloc_conf JEMALLOC_ATTR(weak);
316 bool	opt_abort =
317@@ -2611,6 +2615,107 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
318  */
319 /******************************************************************************/
320 /*
321+ * Begin compatibility functions.
322+ */
323+
324+#define	ALLOCM_LG_ALIGN(la)	(la)
325+#define	ALLOCM_ALIGN(a)		(ffsl(a)-1)
326+#define	ALLOCM_ZERO		((int)0x40)
327+#define	ALLOCM_NO_MOVE		((int)0x80)
328+
329+#define	ALLOCM_SUCCESS		0
330+#define	ALLOCM_ERR_OOM		1
331+#define	ALLOCM_ERR_NOT_MOVED	2
332+
333+int
334+je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
335+{
336+	void *p;
337+
338+	assert(ptr != NULL);
339+
340+	p = je_mallocx(size, flags);
341+	if (p == NULL)
342+		return (ALLOCM_ERR_OOM);
343+	if (rsize != NULL)
344+		*rsize = isalloc(p, config_prof);
345+	*ptr = p;
346+	return (ALLOCM_SUCCESS);
347+}
348+
349+int
350+je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
351+{
352+	int ret;
353+	bool no_move = flags & ALLOCM_NO_MOVE;
354+
355+	assert(ptr != NULL);
356+	assert(*ptr != NULL);
357+	assert(size != 0);
358+	assert(SIZE_T_MAX - size >= extra);
359+
360+	if (no_move) {
361+		size_t usize = je_xallocx(*ptr, size, extra, flags);
362+		ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
363+		if (rsize != NULL)
364+			*rsize = usize;
365+	} else {
366+		void *p = je_rallocx(*ptr, size+extra, flags);
367+		if (p != NULL) {
368+			*ptr = p;
369+			ret = ALLOCM_SUCCESS;
370+		} else
371+			ret = ALLOCM_ERR_OOM;
372+		if (rsize != NULL)
373+			*rsize = isalloc(*ptr, config_prof);
374+	}
375+	return (ret);
376+}
377+
378+int
379+je_sallocm(const void *ptr, size_t *rsize, int flags)
380+{
381+
382+	assert(rsize != NULL);
383+	*rsize = je_sallocx(ptr, flags);
384+	return (ALLOCM_SUCCESS);
385+}
386+
387+int
388+je_dallocm(void *ptr, int flags)
389+{
390+
391+	je_dallocx(ptr, flags);
392+	return (ALLOCM_SUCCESS);
393+}
394+
395+int
396+je_nallocm(size_t *rsize, size_t size, int flags)
397+{
398+	size_t usize;
399+
400+	usize = je_nallocx(size, flags);
401+	if (usize == 0)
402+		return (ALLOCM_ERR_OOM);
403+	if (rsize != NULL)
404+		*rsize = usize;
405+	return (ALLOCM_SUCCESS);
406+}
407+
408+#undef ALLOCM_LG_ALIGN
409+#undef ALLOCM_ALIGN
410+#undef ALLOCM_ZERO
411+#undef ALLOCM_NO_MOVE
412+
413+#undef ALLOCM_SUCCESS
414+#undef ALLOCM_ERR_OOM
415+#undef ALLOCM_ERR_NOT_MOVED
416+
417+/*
418+ * End compatibility functions.
419+ */
420+/******************************************************************************/
421+/*
422  * The following functions are used by threading libraries for protection of
423  * malloc during fork().
424  */
425@@ -2717,4 +2822,11 @@ jemalloc_postfork_child(void)
426 	ctl_postfork_child();
427 }
428
429+void
430+_malloc_first_thread(void)
431+{
432+
433+	(void)malloc_mutex_first_thread();
434+}
435+
436 /******************************************************************************/
437diff --git a/src/mutex.c b/src/mutex.c
438index 2d47af9..934d5aa 100644
439--- a/src/mutex.c
440+++ b/src/mutex.c
441@@ -66,6 +66,17 @@ pthread_create(pthread_t *__restrict thread,
442 #ifdef JEMALLOC_MUTEX_INIT_CB
443 JEMALLOC_EXPORT int	_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
444     void *(calloc_cb)(size_t, size_t));
445+
446+#pragma weak _pthread_mutex_init_calloc_cb
447+int
448+_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
449+    void *(calloc_cb)(size_t, size_t))
450+{
451+
452+	return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t)))
453+	    __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex,
454+	    calloc_cb));
455+}
456 #endif
457
458 bool
459@@ -137,7 +148,7 @@ malloc_mutex_postfork_child(malloc_mutex_t *mutex)
460 }
461
462 bool
463-mutex_boot(void)
464+malloc_mutex_first_thread(void)
465 {
466
467 #ifdef JEMALLOC_MUTEX_INIT_CB
468@@ -151,3 +162,14 @@ mutex_boot(void)
469 #endif
470 	return (false);
471 }
472+
473+bool
474+mutex_boot(void)
475+{
476+
477+#ifndef JEMALLOC_MUTEX_INIT_CB
478+	return (malloc_mutex_first_thread());
479+#else
480+	return (false);
481+#endif
482+}
483diff --git a/src/util.c b/src/util.c
484index 02673c7..116e981 100644
485--- a/src/util.c
486+++ b/src/util.c
487@@ -66,6 +66,22 @@ wrtmessage(void *cbopaque, const char *s)
488
489 JEMALLOC_EXPORT void	(*je_malloc_message)(void *, const char *s);
490
491+JEMALLOC_ATTR(visibility("hidden"))
492+void
493+wrtmessage_1_0(const char *s1, const char *s2, const char *s3,
494+    const char *s4)
495+{
496+
497+	wrtmessage(NULL, s1);
498+	wrtmessage(NULL, s2);
499+	wrtmessage(NULL, s3);
500+	wrtmessage(NULL, s4);
501+}
502+
503+void	(*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
504+    const char *s4) = wrtmessage_1_0;
505+__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
506+
507 /*
508  * Wrapper around malloc_message() that avoids the need for
509  * je_malloc_message(...) throughout the code.
510