xref: /freebsd/contrib/jemalloc/FREEBSD-diffs (revision bde951447fbed7c2669f80b5c7b6d3d16bb144f8)

diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index 3d2e721..b361db2 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -53,11 +53,23 @@
     <para>This manual describes jemalloc @jemalloc_version@.  More information
     can be found at the <ulink
     url="http://jemalloc.net/">jemalloc website</ulink>.</para>
+
+    <para>The following configuration options are enabled in libc's built-in
+    jemalloc: <option>--enable-fill</option>,
+    <option>--enable-lazy-lock</option>, <option>--enable-munmap</option>,
+    <option>--enable-stats</option>, <option>--enable-tcache</option>,
+    <option>--enable-tls</option>, <option>--enable-utrace</option>, and
+    <option>--enable-xmalloc</option>.  Additionally,
+    <option>--enable-debug</option> is enabled in development versions of
+    FreeBSD (controlled by the <constant>MALLOC_PRODUCTION</constant> make
+    variable).</para>
+
   </refsect1>
   <refsynopsisdiv>
     <title>SYNOPSIS</title>
     <funcsynopsis>
-      <funcsynopsisinfo>#include &lt;<filename class="headerfile">jemalloc/jemalloc.h</filename>&gt;</funcsynopsisinfo>
+      <funcsynopsisinfo>#include &lt;<filename class="headerfile">stdlib.h</filename>&gt;
+#include &lt;<filename class="headerfile">malloc_np.h</filename>&gt;</funcsynopsisinfo>
       <refsect2>
         <title>Standard API</title>
         <funcprototype>
@@ -2963,4 +2975,18 @@ malloc_conf = "lg_chunk:24";]]></programlisting></para>
     <para>The <function>posix_memalign()</function> function conforms
     to IEEE Std 1003.1-2001 (<quote>POSIX.1</quote>).</para>
   </refsect1>
+  <refsect1 id="history">
+    <title>HISTORY</title>
+    <para>The <function>malloc_usable_size()</function> and
+    <function>posix_memalign()</function> functions first appeared in FreeBSD
+    7.0.</para>
+
+    <para>The <function>aligned_alloc()</function>,
+    <function>malloc_stats_print()</function>, and
+    <function>mallctl*()</function> functions first appeared in FreeBSD
+    10.0.</para>
+
+    <para>The <function>*allocx()</function> functions first appeared in FreeBSD
+    11.0.</para>
+  </refsect1>
 </refentry>
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index f39ce54..a3ba55d 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -719,8 +719,13 @@ arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind)
 JEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t *
 arena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind)
 {
+#if 1 /* Work around gcc bug. */
+	arena_chunk_t *mchunk = (arena_chunk_t *)chunk;

+	return (arena_miscelm_get_mutable(mchunk, pageind));
+#else
 	return (arena_miscelm_get_mutable((arena_chunk_t *)chunk, pageind));
+#endif
 }

 JEMALLOC_ALWAYS_INLINE size_t
@@ -779,8 +784,13 @@ arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind)
 JEMALLOC_ALWAYS_INLINE const size_t *
 arena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind)
 {
+#if 1 /* Work around gcc bug. */
+	arena_chunk_t *mchunk = (arena_chunk_t *)chunk;

+	return (arena_mapbitsp_get_mutable(mchunk, pageind));
+#else
 	return (arena_mapbitsp_get_mutable((arena_chunk_t *)chunk, pageind));
+#endif
 }

 JEMALLOC_ALWAYS_INLINE size_t
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index fdc8fef..56a35a4 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -8,6 +8,9 @@
 #include <sys/ktrace.h>
 #endif

+#include "un-namespace.h"
+#include "libc_private.h"
+
 #define	JEMALLOC_NO_DEMANGLE
 #ifdef JEMALLOC_JET
 #  define JEMALLOC_N(n) jet_##n
@@ -42,13 +45,7 @@ static const bool config_fill =
     false
 #endif
     ;
-static const bool config_lazy_lock =
-#ifdef JEMALLOC_LAZY_LOCK
-    true
-#else
-    false
-#endif
-    ;
+static const bool config_lazy_lock = true;
 static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF;
 static const bool config_prof =
 #ifdef JEMALLOC_PROF
diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h
index c907d91..4626632 100644
--- a/include/jemalloc/internal/jemalloc_internal_decls.h
+++ b/include/jemalloc/internal/jemalloc_internal_decls.h
@@ -1,6 +1,9 @@
 #ifndef JEMALLOC_INTERNAL_DECLS_H
 #define	JEMALLOC_INTERNAL_DECLS_H

+#include "libc_private.h"
+#include "namespace.h"
+
 #include <math.h>
 #ifdef _WIN32
 #  include <windows.h>
diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h
index b442d2d..76518db 100644
--- a/include/jemalloc/internal/mutex.h
+++ b/include/jemalloc/internal/mutex.h
@@ -57,9 +57,6 @@ struct malloc_mutex_s {

 #ifdef JEMALLOC_LAZY_LOCK
 extern bool isthreaded;
-#else
-#  undef isthreaded /* Undo private_namespace.h definition. */
-#  define isthreaded true
 #endif

 bool	malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
@@ -67,6 +64,7 @@ bool	malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
 void	malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
 void	malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
 void	malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
+bool	malloc_mutex_first_thread(void);
 bool	malloc_mutex_boot(void);

 #endif /* JEMALLOC_H_EXTERNS */
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index 87c8c9b..df576f6 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -307,7 +307,6 @@ iralloct_realign
 isalloc
 isdalloct
 isqalloc
-isthreaded
 ivsalloc
 ixalloc
 jemalloc_postfork_child
diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h
new file mode 100644
index 0000000..c58a8f3
--- /dev/null
+++ b/include/jemalloc/jemalloc_FreeBSD.h
@@ -0,0 +1,162 @@
+/*
+ * Override settings that were generated in jemalloc_defs.h as necessary.
+ */
+
+#undef JEMALLOC_OVERRIDE_VALLOC
+
+#ifndef MALLOC_PRODUCTION
+#define	JEMALLOC_DEBUG
+#endif
+
+#undef JEMALLOC_DSS
+
+/*
+ * The following are architecture-dependent, so conditionally define them for
+ * each supported architecture.
+ */
+#undef JEMALLOC_TLS_MODEL
+#undef STATIC_PAGE_SHIFT
+#undef LG_SIZEOF_PTR
+#undef LG_SIZEOF_INT
+#undef LG_SIZEOF_LONG
+#undef LG_SIZEOF_INTMAX_T
+
+#ifdef __i386__
+#  define LG_SIZEOF_PTR		2
+#  define JEMALLOC_TLS_MODEL	__attribute__((tls_model("initial-exec")))
+#endif
+#ifdef __ia64__
+#  define LG_SIZEOF_PTR		3
+#endif
+#ifdef __sparc64__
+#  define LG_SIZEOF_PTR		3
+#  define JEMALLOC_TLS_MODEL	__attribute__((tls_model("initial-exec")))
+#endif
+#ifdef __amd64__
+#  define LG_SIZEOF_PTR		3
+#  define JEMALLOC_TLS_MODEL	__attribute__((tls_model("initial-exec")))
+#endif
+#ifdef __arm__
+#  define LG_SIZEOF_PTR		2
+#endif
+#ifdef __aarch64__
+#  define LG_SIZEOF_PTR		3
+#endif
+#ifdef __mips__
+#ifdef __mips_n64
+#  define LG_SIZEOF_PTR		3
+#else
+#  define LG_SIZEOF_PTR		2
+#endif
+#endif
+#ifdef __powerpc64__
+#  define LG_SIZEOF_PTR		3
+#elif defined(__powerpc__)
+#  define LG_SIZEOF_PTR		2
+#endif
+#ifdef __riscv__
+#  define LG_SIZEOF_PTR		3
+#endif
+
+#ifndef JEMALLOC_TLS_MODEL
+#  define JEMALLOC_TLS_MODEL	/* Default. */
+#endif
+
+#define	STATIC_PAGE_SHIFT	PAGE_SHIFT
+#define	LG_SIZEOF_INT		2
+#define	LG_SIZEOF_LONG		LG_SIZEOF_PTR
+#define	LG_SIZEOF_INTMAX_T	3
+
+#undef CPU_SPINWAIT
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+#define	CPU_SPINWAIT		cpu_spinwait()
+
+/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */
+#undef JEMALLOC_LAZY_LOCK
+extern int __isthreaded;
+#define	isthreaded		((bool)__isthreaded)
+
+/* Mangle. */
+#undef je_malloc
+#undef je_calloc
+#undef je_posix_memalign
+#undef je_aligned_alloc
+#undef je_realloc
+#undef je_free
+#undef je_malloc_usable_size
+#undef je_mallocx
+#undef je_rallocx
+#undef je_xallocx
+#undef je_sallocx
+#undef je_dallocx
+#undef je_sdallocx
+#undef je_nallocx
+#undef je_mallctl
+#undef je_mallctlnametomib
+#undef je_mallctlbymib
+#undef je_malloc_stats_print
+#undef je_allocm
+#undef je_rallocm
+#undef je_sallocm
+#undef je_dallocm
+#undef je_nallocm
+#define	je_malloc		__malloc
+#define	je_calloc		__calloc
+#define	je_posix_memalign	__posix_memalign
+#define	je_aligned_alloc	__aligned_alloc
+#define	je_realloc		__realloc
+#define	je_free			__free
+#define	je_malloc_usable_size	__malloc_usable_size
+#define	je_mallocx		__mallocx
+#define	je_rallocx		__rallocx
+#define	je_xallocx		__xallocx
+#define	je_sallocx		__sallocx
+#define	je_dallocx		__dallocx
+#define	je_sdallocx		__sdallocx
+#define	je_nallocx		__nallocx
+#define	je_mallctl		__mallctl
+#define	je_mallctlnametomib	__mallctlnametomib
+#define	je_mallctlbymib		__mallctlbymib
+#define	je_malloc_stats_print	__malloc_stats_print
+#define	je_allocm		__allocm
+#define	je_rallocm		__rallocm
+#define	je_sallocm		__sallocm
+#define	je_dallocm		__dallocm
+#define	je_nallocm		__nallocm
+#define	open			_open
+#define	read			_read
+#define	write			_write
+#define	close			_close
+#define	pthread_mutex_lock	_pthread_mutex_lock
+#define	pthread_mutex_unlock	_pthread_mutex_unlock
+
+#ifdef JEMALLOC_C_
+/*
+ * Define 'weak' symbols so that an application can have its own versions
+ * of malloc, calloc, realloc, free, et al.
+ */
+__weak_reference(__malloc, malloc);
+__weak_reference(__calloc, calloc);
+__weak_reference(__posix_memalign, posix_memalign);
+__weak_reference(__aligned_alloc, aligned_alloc);
+__weak_reference(__realloc, realloc);
+__weak_reference(__free, free);
+__weak_reference(__malloc_usable_size, malloc_usable_size);
+__weak_reference(__mallocx, mallocx);
+__weak_reference(__rallocx, rallocx);
+__weak_reference(__xallocx, xallocx);
+__weak_reference(__sallocx, sallocx);
+__weak_reference(__dallocx, dallocx);
+__weak_reference(__sdallocx, sdallocx);
+__weak_reference(__nallocx, nallocx);
+__weak_reference(__mallctl, mallctl);
+__weak_reference(__mallctlnametomib, mallctlnametomib);
+__weak_reference(__mallctlbymib, mallctlbymib);
+__weak_reference(__malloc_stats_print, malloc_stats_print);
+__weak_reference(__allocm, allocm);
+__weak_reference(__rallocm, rallocm);
+__weak_reference(__sallocm, sallocm);
+__weak_reference(__dallocm, dallocm);
+__weak_reference(__nallocm, nallocm);
+#endif
diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh
index f943891..47d032c 100755
--- a/include/jemalloc/jemalloc_rename.sh
+++ b/include/jemalloc/jemalloc_rename.sh
@@ -19,4 +19,6 @@ done

 cat <<EOF
 #endif
+
+#include "jemalloc_FreeBSD.h"
 EOF
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 38650ff..f659b55 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -4,6 +4,10 @@
 /******************************************************************************/
 /* Data. */

+/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
+const char	*__malloc_options_1_0 = NULL;
+__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
+
 /* Runtime configuration options. */
 const char	*je_malloc_conf
 #ifndef _WIN32
@@ -2756,6 +2760,107 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
  */
 /******************************************************************************/
 /*
+ * Begin compatibility functions.
+ */
+
+#define	ALLOCM_LG_ALIGN(la)	(la)
+#define	ALLOCM_ALIGN(a)		(ffsl(a)-1)
+#define	ALLOCM_ZERO		((int)0x40)
+#define	ALLOCM_NO_MOVE		((int)0x80)
+
+#define	ALLOCM_SUCCESS		0
+#define	ALLOCM_ERR_OOM		1
+#define	ALLOCM_ERR_NOT_MOVED	2
+
+int
+je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
+{
+	void *p;
+
+	assert(ptr != NULL);
+
+	p = je_mallocx(size, flags);
+	if (p == NULL)
+		return (ALLOCM_ERR_OOM);
+	if (rsize != NULL)
+		*rsize = isalloc(tsdn_fetch(), p, config_prof);
+	*ptr = p;
+	return (ALLOCM_SUCCESS);
+}
+
+int
+je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
+{
+	int ret;
+	bool no_move = flags & ALLOCM_NO_MOVE;
+
+	assert(ptr != NULL);
+	assert(*ptr != NULL);
+	assert(size != 0);
+	assert(SIZE_T_MAX - size >= extra);
+
+	if (no_move) {
+		size_t usize = je_xallocx(*ptr, size, extra, flags);
+		ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
+		if (rsize != NULL)
+			*rsize = usize;
+	} else {
+		void *p = je_rallocx(*ptr, size+extra, flags);
+		if (p != NULL) {
+			*ptr = p;
+			ret = ALLOCM_SUCCESS;
+		} else
+			ret = ALLOCM_ERR_OOM;
+		if (rsize != NULL)
+			*rsize = isalloc(tsdn_fetch(), *ptr, config_prof);
+	}
+	return (ret);
+}
+
+int
+je_sallocm(const void *ptr, size_t *rsize, int flags)
+{
+
+	assert(rsize != NULL);
+	*rsize = je_sallocx(ptr, flags);
+	return (ALLOCM_SUCCESS);
+}
+
+int
+je_dallocm(void *ptr, int flags)
+{
+
+	je_dallocx(ptr, flags);
+	return (ALLOCM_SUCCESS);
+}
+
+int
+je_nallocm(size_t *rsize, size_t size, int flags)
+{
+	size_t usize;
+
+	usize = je_nallocx(size, flags);
+	if (usize == 0)
+		return (ALLOCM_ERR_OOM);
+	if (rsize != NULL)
+		*rsize = usize;
+	return (ALLOCM_SUCCESS);
+}
+
+#undef ALLOCM_LG_ALIGN
+#undef ALLOCM_ALIGN
+#undef ALLOCM_ZERO
+#undef ALLOCM_NO_MOVE
+
+#undef ALLOCM_SUCCESS
+#undef ALLOCM_ERR_OOM
+#undef ALLOCM_ERR_NOT_MOVED
+
+/*
+ * End compatibility functions.
+ */
+/******************************************************************************/
+/*
  * The following functions are used by threading libraries for protection of
  * malloc during fork().
  */
@@ -2894,4 +2999,11 @@ jemalloc_postfork_child(void)
 	ctl_postfork_child(tsd_tsdn(tsd));
 }

+void
+_malloc_first_thread(void)
+{
+
+	(void)malloc_mutex_first_thread();
+}
+
 /******************************************************************************/
diff --git a/src/mutex.c b/src/mutex.c
index 6333e73..13f8d79 100644
--- a/src/mutex.c
+++ b/src/mutex.c
@@ -66,6 +66,17 @@ pthread_create(pthread_t *__restrict thread,
 #ifdef JEMALLOC_MUTEX_INIT_CB
 JEMALLOC_EXPORT int	_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
     void *(calloc_cb)(size_t, size_t));
+
+#pragma weak _pthread_mutex_init_calloc_cb
+int
+_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
+    void *(calloc_cb)(size_t, size_t))
+{
+
+	return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t)))
+	    __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex,
+	    calloc_cb));
+}
 #endif

 bool
@@ -142,7 +153,7 @@ malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex)
 }

 bool
-malloc_mutex_boot(void)
+malloc_mutex_first_thread(void)
 {

 #ifdef JEMALLOC_MUTEX_INIT_CB
@@ -156,3 +167,14 @@ malloc_mutex_boot(void)
 #endif
 	return (false);
 }
+
+bool
+malloc_mutex_boot(void)
+{
+
+#ifndef JEMALLOC_MUTEX_INIT_CB
+	return (malloc_mutex_first_thread());
+#else
+	return (false);
+#endif
+}
diff --git a/src/util.c b/src/util.c
index 7905267..bee1c77 100644
--- a/src/util.c
+++ b/src/util.c
@@ -67,6 +67,22 @@ wrtmessage(void *cbopaque, const char *s)

 JEMALLOC_EXPORT void	(*je_malloc_message)(void *, const char *s);

+JEMALLOC_ATTR(visibility("hidden"))
+void
+wrtmessage_1_0(const char *s1, const char *s2, const char *s3,
+    const char *s4)
+{
+
+	wrtmessage(NULL, s1);
+	wrtmessage(NULL, s2);
+	wrtmessage(NULL, s3);
+	wrtmessage(NULL, s4);
+}
+
+void	(*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
+    const char *s4) = wrtmessage_1_0;
+__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
+
 /*
  * Wrapper around malloc_message() that avoids the need for
  * je_malloc_message(...) throughout the code.