xref: /freebsd/contrib/jemalloc/FREEBSD-diffs (revision 7d4374f65f7b3df3d2567029c510f2e1576f0f69)

diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index 7fecda7c..d5ca5e86 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -53,11 +53,22 @@
     <para>This manual describes jemalloc @jemalloc_version@.  More information
     can be found at the <ulink
     url="http://jemalloc.net/">jemalloc website</ulink>.</para>
+
+    <para>The following configuration options are enabled in libc's built-in
+    jemalloc: <option>--enable-fill</option>,
+    <option>--enable-lazy-lock</option>, <option>--enable-stats</option>,
+    <option>--enable-utrace</option>, <option>--enable-xmalloc</option>, and
+    <option>--with-malloc-conf=abort_conf:false</option>.
+    Additionally, <option>--enable-debug</option> is enabled in development
+    versions of FreeBSD (controlled by the
+    <constant>MK_MALLOC_PRODUCTION</constant> make variable).</para>
+
   </refsect1>
   <refsynopsisdiv>
     <title>SYNOPSIS</title>
     <funcsynopsis>
-      <funcsynopsisinfo>#include &lt;<filename class="headerfile">jemalloc/jemalloc.h</filename>&gt;</funcsynopsisinfo>
+      <funcsynopsisinfo>#include &lt;<filename class="headerfile">stdlib.h</filename>&gt;
+#include &lt;<filename class="headerfile">malloc_np.h</filename>&gt;</funcsynopsisinfo>
       <refsect2>
         <title>Standard API</title>
         <funcprototype>
@@ -3510,4 +3521,18 @@ malloc_conf = "narenas:1";]]></programlisting></para>
     <para>The <function>posix_memalign()</function> function conforms
     to IEEE Std 1003.1-2001 (<quote>POSIX.1</quote>).</para>
   </refsect1>
+  <refsect1 id="history">
+    <title>HISTORY</title>
+    <para>The <function>malloc_usable_size()</function> and
+    <function>posix_memalign()</function> functions first appeared in FreeBSD
+    7.0.</para>
+
+    <para>The <function>aligned_alloc()</function>,
+    <function>malloc_stats_print()</function>, and
+    <function>mallctl*()</function> functions first appeared in FreeBSD
+    10.0.</para>
+
+    <para>The <function>*allocx()</function> functions first appeared in FreeBSD
+    11.0.</para>
+  </refsect1>
 </refentry>
diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h
index 7d6053e2..a0e4f5af 100644
--- a/include/jemalloc/internal/jemalloc_internal_decls.h
+++ b/include/jemalloc/internal/jemalloc_internal_decls.h
@@ -1,6 +1,9 @@
 #ifndef JEMALLOC_INTERNAL_DECLS_H
 #define JEMALLOC_INTERNAL_DECLS_H

+#include "libc_private.h"
+#include "namespace.h"
+
 #include <math.h>
 #ifdef _WIN32
 #  include <windows.h>
diff --git a/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h
new file mode 100644
index 00000000..0dab1296
--- /dev/null
+++ b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h
@@ -0,0 +1,9 @@
+#ifndef __clang__
+#  undef JEMALLOC_INTERNAL_UNREACHABLE
+#  define JEMALLOC_INTERNAL_UNREACHABLE abort
+
+#  undef JEMALLOC_C11_ATOMICS
+#  undef JEMALLOC_GCC_ATOMIC_ATOMICS
+#  undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS
+#  undef JEMALLOC_GCC_U8_SYNC_ATOMICS
+#endif
diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in
index 3418cbfa..53e30dc4 100644
--- a/include/jemalloc/internal/jemalloc_preamble.h.in
+++ b/include/jemalloc/internal/jemalloc_preamble.h.in
@@ -8,6 +8,9 @@
 #include <sys/ktrace.h>
 #endif

+#include "un-namespace.h"
+#include "libc_private.h"
+
 #define JEMALLOC_NO_DEMANGLE
 #ifdef JEMALLOC_JET
 #  undef JEMALLOC_IS_MALLOC
@@ -79,13 +82,7 @@ static const bool config_fill =
     false
 #endif
     ;
-static const bool config_lazy_lock =
-#ifdef JEMALLOC_LAZY_LOCK
-    true
-#else
-    false
-#endif
-    ;
+static const bool config_lazy_lock = true;
 static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF;
 static const bool config_prof =
 #ifdef JEMALLOC_PROF
diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h
index 7c24f072..94af1618 100644
--- a/include/jemalloc/internal/mutex.h
+++ b/include/jemalloc/internal/mutex.h
@@ -135,9 +135,6 @@ struct malloc_mutex_s {

 #ifdef JEMALLOC_LAZY_LOCK
 extern bool isthreaded;
-#else
-#  undef isthreaded /* Undo private_namespace.h definition. */
-#  define isthreaded true
 #endif

 bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
@@ -145,6 +142,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
 void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
 void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
 void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
+bool malloc_mutex_first_thread(void);
 bool malloc_mutex_boot(void);
 void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);

diff --git a/include/jemalloc/internal/test_hooks.h b/include/jemalloc/internal/test_hooks.h
index a6351e59..0780c52f 100644
--- a/include/jemalloc/internal/test_hooks.h
+++ b/include/jemalloc/internal/test_hooks.h
@@ -6,13 +6,6 @@ extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)();

 #define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn)

-#define open JEMALLOC_HOOK(open, test_hooks_libc_hook)
-#define read JEMALLOC_HOOK(read, test_hooks_libc_hook)
-#define write JEMALLOC_HOOK(write, test_hooks_libc_hook)
-#define readlink JEMALLOC_HOOK(readlink, test_hooks_libc_hook)
-#define close JEMALLOC_HOOK(close, test_hooks_libc_hook)
-#define creat JEMALLOC_HOOK(creat, test_hooks_libc_hook)
-#define secure_getenv JEMALLOC_HOOK(secure_getenv, test_hooks_libc_hook)
 /* Note that this is undef'd and re-define'd in src/prof.c. */
 #define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)

diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h
index 9ba26004..ecfda5d6 100644
--- a/include/jemalloc/internal/tsd.h
+++ b/include/jemalloc/internal/tsd.h
@@ -198,7 +198,8 @@ struct tsd_s {
 	t TSD_MANGLE(n);
 MALLOC_TSD
 #undef O
-};
+/* AddressSanitizer requires TLS data to be aligned to at least 8 bytes. */
+} JEMALLOC_ALIGNED(16);

 JEMALLOC_ALWAYS_INLINE uint8_t
 tsd_state_get(tsd_t *tsd) {
diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h
new file mode 100644
index 00000000..b752b0e7
--- /dev/null
+++ b/include/jemalloc/jemalloc_FreeBSD.h
@@ -0,0 +1,185 @@
+/*
+ * Override settings that were generated in jemalloc_defs.h as necessary.
+ */
+
+#undef JEMALLOC_OVERRIDE_VALLOC
+
+#ifndef MALLOC_PRODUCTION
+#define	JEMALLOC_DEBUG
+#endif
+
+#undef JEMALLOC_DSS
+
+#undef JEMALLOC_BACKGROUND_THREAD
+
+/*
+ * The following are architecture-dependent, so conditionally define them for
+ * each supported architecture.
+ */
+#undef JEMALLOC_TLS_MODEL
+#undef LG_PAGE
+#undef LG_VADDR
+#undef LG_SIZEOF_PTR
+#undef LG_SIZEOF_INT
+#undef LG_SIZEOF_LONG
+#undef LG_SIZEOF_INTMAX_T
+
+#ifdef __i386__
+#  define LG_VADDR		32
+#  define LG_SIZEOF_PTR		2
+#  define JEMALLOC_TLS_MODEL	__attribute__((tls_model("initial-exec")))
+#endif
+#ifdef __ia64__
+#  define LG_VADDR		64
+#  define LG_SIZEOF_PTR		3
+#endif
+#ifdef __sparc64__
+#  define LG_VADDR		64
+#  define LG_SIZEOF_PTR		3
+#  define JEMALLOC_TLS_MODEL	__attribute__((tls_model("initial-exec")))
+#endif
+#ifdef __amd64__
+#  define LG_VADDR		48
+#  define LG_SIZEOF_PTR		3
+#  define JEMALLOC_TLS_MODEL	__attribute__((tls_model("initial-exec")))
+#endif
+#ifdef __arm__
+#  define LG_VADDR		32
+#  define LG_SIZEOF_PTR		2
+#endif
+#ifdef __aarch64__
+#  define LG_VADDR		48
+#  define LG_SIZEOF_PTR		3
+#endif
+#ifdef __mips__
+#ifdef __mips_n64
+#  define LG_VADDR		64
+#  define LG_SIZEOF_PTR		3
+#else
+#  define LG_VADDR		32
+#  define LG_SIZEOF_PTR		2
+#endif
+#endif
+#ifdef __powerpc64__
+#  define LG_VADDR		64
+#  define LG_SIZEOF_PTR		3
+#elif defined(__powerpc__)
+#  define LG_VADDR		32
+#  define LG_SIZEOF_PTR		2
+#endif
+#ifdef __riscv
+#  define LG_VADDR		48
+#  define LG_SIZEOF_PTR		3
+#endif
+
+#ifndef JEMALLOC_TLS_MODEL
+#  define JEMALLOC_TLS_MODEL	/* Default. */
+#endif
+
+#define	LG_PAGE			PAGE_SHIFT
+#define	LG_SIZEOF_INT		2
+#define	LG_SIZEOF_LONG		LG_SIZEOF_PTR
+#define	LG_SIZEOF_INTMAX_T	3
+
+#undef CPU_SPINWAIT
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+#define	CPU_SPINWAIT		cpu_spinwait()
+
+/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */
+#undef JEMALLOC_LAZY_LOCK
+extern int __isthreaded;
+#define	isthreaded		((bool)__isthreaded)
+
+/* Mangle. */
+#undef je_malloc
+#undef je_calloc
+#undef je_posix_memalign
+#undef je_aligned_alloc
+#undef je_realloc
+#undef je_free
+#undef je_malloc_usable_size
+#undef je_mallocx
+#undef je_rallocx
+#undef je_xallocx
+#undef je_sallocx
+#undef je_dallocx
+#undef je_sdallocx
+#undef je_nallocx
+#undef je_mallctl
+#undef je_mallctlnametomib
+#undef je_mallctlbymib
+#undef je_malloc_stats_print
+#undef je_allocm
+#undef je_rallocm
+#undef je_sallocm
+#undef je_dallocm
+#undef je_nallocm
+#define	je_malloc		__malloc
+#define	je_calloc		__calloc
+#define	je_posix_memalign	__posix_memalign
+#define	je_aligned_alloc	__aligned_alloc
+#define	je_realloc		__realloc
+#define	je_free			__free
+#define	je_malloc_usable_size	__malloc_usable_size
+#define	je_mallocx		__mallocx
+#define	je_rallocx		__rallocx
+#define	je_xallocx		__xallocx
+#define	je_sallocx		__sallocx
+#define	je_dallocx		__dallocx
+#define	je_sdallocx		__sdallocx
+#define	je_nallocx		__nallocx
+#define	je_mallctl		__mallctl
+#define	je_mallctlnametomib	__mallctlnametomib
+#define	je_mallctlbymib		__mallctlbymib
+#define	je_malloc_stats_print	__malloc_stats_print
+#define	je_allocm		__allocm
+#define	je_rallocm		__rallocm
+#define	je_sallocm		__sallocm
+#define	je_dallocm		__dallocm
+#define	je_nallocm		__nallocm
+#define	open			_open
+#define	read			_read
+#define	write			_write
+#define	close			_close
+#define	pthread_join		_pthread_join
+#define	pthread_once		_pthread_once
+#define	pthread_self		_pthread_self
+#define	pthread_equal		_pthread_equal
+#define	pthread_mutex_lock	_pthread_mutex_lock
+#define	pthread_mutex_trylock	_pthread_mutex_trylock
+#define	pthread_mutex_unlock	_pthread_mutex_unlock
+#define	pthread_cond_init	_pthread_cond_init
+#define	pthread_cond_wait	_pthread_cond_wait
+#define	pthread_cond_timedwait	_pthread_cond_timedwait
+#define	pthread_cond_signal	_pthread_cond_signal
+
+#ifdef JEMALLOC_C_
+/*
+ * Define 'weak' symbols so that an application can have its own versions
+ * of malloc, calloc, realloc, free, et al.
+ */
+__weak_reference(__malloc, malloc);
+__weak_reference(__calloc, calloc);
+__weak_reference(__posix_memalign, posix_memalign);
+__weak_reference(__aligned_alloc, aligned_alloc);
+__weak_reference(__realloc, realloc);
+__weak_reference(__free, free);
+__weak_reference(__malloc_usable_size, malloc_usable_size);
+__weak_reference(__mallocx, mallocx);
+__weak_reference(__rallocx, rallocx);
+__weak_reference(__xallocx, xallocx);
+__weak_reference(__sallocx, sallocx);
+__weak_reference(__dallocx, dallocx);
+__weak_reference(__sdallocx, sdallocx);
+__weak_reference(__nallocx, nallocx);
+__weak_reference(__mallctl, mallctl);
+__weak_reference(__mallctlnametomib, mallctlnametomib);
+__weak_reference(__mallctlbymib, mallctlbymib);
+__weak_reference(__malloc_stats_print, malloc_stats_print);
+__weak_reference(__allocm, allocm);
+__weak_reference(__rallocm, rallocm);
+__weak_reference(__sallocm, sallocm);
+__weak_reference(__dallocm, dallocm);
+__weak_reference(__nallocm, nallocm);
+#endif
diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh
index f9438912..47d032c1 100755
--- a/include/jemalloc/jemalloc_rename.sh
+++ b/include/jemalloc/jemalloc_rename.sh
@@ -19,4 +19,6 @@ done

 cat <<EOF
 #endif
+
+#include "jemalloc_FreeBSD.h"
 EOF
diff --git a/src/jemalloc.c b/src/jemalloc.c
index ed13718d..fefb719a 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -23,6 +23,10 @@
 /******************************************************************************/
 /* Data. */

+/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
+const char	*__malloc_options_1_0 = NULL;
+__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
+
 /* Runtime configuration options. */
 const char	*je_malloc_conf
 #ifndef _WIN32
@@ -2660,25 +2664,6 @@ je_realloc(void *ptr, size_t arg_size) {
 	LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);

 	if (unlikely(size == 0)) {
-		if (ptr != NULL) {
-			/* realloc(ptr, 0) is equivalent to free(ptr). */
-			UTRACE(ptr, 0, 0);
-			tcache_t *tcache;
-			tsd_t *tsd = tsd_fetch();
-			if (tsd_reentrancy_level_get(tsd) == 0) {
-				tcache = tcache_get(tsd);
-			} else {
-				tcache = NULL;
-			}
-
-			uintptr_t args[3] = {(uintptr_t)ptr, size};
-			hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
-
-			ifree(tsd, ptr, tcache, true);
-
-			LOG("core.realloc.exit", "result: %p", NULL);
-			return NULL;
-		}
 		size = 1;
 	}

@@ -3750,6 +3735,103 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
  * End non-standard functions.
  */
 /******************************************************************************/
+/*
+ * Begin compatibility functions.
+ */
+
+#define	ALLOCM_LG_ALIGN(la)	(la)
+#define	ALLOCM_ALIGN(a)		(ffsl(a)-1)
+#define	ALLOCM_ZERO		((int)0x40)
+#define	ALLOCM_NO_MOVE		((int)0x80)
+
+#define	ALLOCM_SUCCESS		0
+#define	ALLOCM_ERR_OOM		1
+#define	ALLOCM_ERR_NOT_MOVED	2
+
+int
+je_allocm(void **ptr, size_t *rsize, size_t size, int flags) {
+	assert(ptr != NULL);
+
+	void *p = je_mallocx(size, flags);
+	if (p == NULL) {
+		return (ALLOCM_ERR_OOM);
+	}
+	if (rsize != NULL) {
+		*rsize = isalloc(tsdn_fetch(), p);
+	}
+	*ptr = p;
+	return ALLOCM_SUCCESS;
+}
+
+int
+je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) {
+	assert(ptr != NULL);
+	assert(*ptr != NULL);
+	assert(size != 0);
+	assert(SIZE_T_MAX - size >= extra);
+
+	int ret;
+	bool no_move = flags & ALLOCM_NO_MOVE;
+
+	if (no_move) {
+		size_t usize = je_xallocx(*ptr, size, extra, flags);
+		ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
+		if (rsize != NULL) {
+			*rsize = usize;
+		}
+	} else {
+		void *p = je_rallocx(*ptr, size+extra, flags);
+		if (p != NULL) {
+			*ptr = p;
+			ret = ALLOCM_SUCCESS;
+		} else {
+			ret = ALLOCM_ERR_OOM;
+		}
+		if (rsize != NULL) {
+			*rsize = isalloc(tsdn_fetch(), *ptr);
+		}
+	}
+	return ret;
+}
+
+int
+je_sallocm(const void *ptr, size_t *rsize, int flags) {
+	assert(rsize != NULL);
+	*rsize = je_sallocx(ptr, flags);
+	return ALLOCM_SUCCESS;
+}
+
+int
+je_dallocm(void *ptr, int flags) {
+	je_dallocx(ptr, flags);
+	return ALLOCM_SUCCESS;
+}
+
+int
+je_nallocm(size_t *rsize, size_t size, int flags) {
+	size_t usize = je_nallocx(size, flags);
+	if (usize == 0) {
+		return ALLOCM_ERR_OOM;
+	}
+	if (rsize != NULL) {
+		*rsize = usize;
+	}
+	return ALLOCM_SUCCESS;
+}
+
+#undef ALLOCM_LG_ALIGN
+#undef ALLOCM_ALIGN
+#undef ALLOCM_ZERO
+#undef ALLOCM_NO_MOVE
+
+#undef ALLOCM_SUCCESS
+#undef ALLOCM_ERR_OOM
+#undef ALLOCM_ERR_NOT_MOVED
+
+/*
+ * End compatibility functions.
+ */
+/******************************************************************************/
 /*
  * The following functions are used by threading libraries for protection of
  * malloc during fork().
@@ -3919,4 +4001,11 @@ jemalloc_postfork_child(void) {
 	ctl_postfork_child(tsd_tsdn(tsd));
 }

+void
+_malloc_first_thread(void)
+{
+
+	(void)malloc_mutex_first_thread();
+}
+
 /******************************************************************************/
diff --git a/src/malloc_io.c b/src/malloc_io.c
index d7cb0f52..cda589c4 100644
--- a/src/malloc_io.c
+++ b/src/malloc_io.c
@@ -75,6 +75,20 @@ wrtmessage(void *cbopaque, const char *s) {

 JEMALLOC_EXPORT void	(*je_malloc_message)(void *, const char *s);

+JEMALLOC_ATTR(visibility("hidden"))
+void
+wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) {
+
+	wrtmessage(NULL, s1);
+	wrtmessage(NULL, s2);
+	wrtmessage(NULL, s3);
+	wrtmessage(NULL, s4);
+}
+
+void	(*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
+    const char *s4) = wrtmessage_1_0;
+__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
+
 /*
  * Wrapper around malloc_message() that avoids the need for
  * je_malloc_message(...) throughout the code.
diff --git a/src/mutex.c b/src/mutex.c
index 3f920f5b..88a7730c 100644
--- a/src/mutex.c
+++ b/src/mutex.c
@@ -41,6 +41,17 @@ pthread_create(pthread_t *__restrict thread,
 #ifdef JEMALLOC_MUTEX_INIT_CB
 JEMALLOC_EXPORT int	_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
     void *(calloc_cb)(size_t, size_t));
+
+#pragma weak _pthread_mutex_init_calloc_cb
+int
+_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
+    void *(calloc_cb)(size_t, size_t))
+{
+
+	return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t)))
+	    __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex,
+	    calloc_cb));
+}
 #endif

 void
@@ -131,6 +142,16 @@ mutex_addr_comp(const witness_t *witness1, void *mutex1,
 	}
 }

+bool
+malloc_mutex_first_thread(void) {
+
+#ifndef JEMALLOC_MUTEX_INIT_CB
+	return (malloc_mutex_first_thread());
+#else
+	return (false);
+#endif
+}
+
 bool
 malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
     witness_rank_t rank, malloc_mutex_lock_order_t lock_order) {