1diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in 2index 1e12fd3a..c42a7e10 100644 3--- a/doc/jemalloc.xml.in 4+++ b/doc/jemalloc.xml.in 5@@ -53,11 +53,22 @@ 6 <para>This manual describes jemalloc @jemalloc_version@. More information 7 can be found at the <ulink 8 url="http://jemalloc.net/">jemalloc website</ulink>.</para> 9+ 10+ <para>The following configuration options are enabled in libc's built-in 11+ jemalloc: <option>--enable-fill</option>, 12+ <option>--enable-lazy-lock</option>, <option>--enable-stats</option>, 13+ <option>--enable-utrace</option>, <option>--enable-xmalloc</option>, and 14+ <option>--with-malloc-conf=abort_conf:false</option>. 15+ Additionally, <option>--enable-debug</option> is enabled in development 16+ versions of FreeBSD (controlled by the 17+ <constant>MALLOC_PRODUCTION</constant> make variable).</para> 18+ 19 </refsect1> 20 <refsynopsisdiv> 21 <title>SYNOPSIS</title> 22 <funcsynopsis> 23- <funcsynopsisinfo>#include <<filename class="headerfile">jemalloc/jemalloc.h</filename>></funcsynopsisinfo> 24+ <funcsynopsisinfo>#include <<filename class="headerfile">stdlib.h</filename>> 25+#include <<filename class="headerfile">malloc_np.h</filename>></funcsynopsisinfo> 26 <refsect2> 27 <title>Standard API</title> 28 <funcprototype> 29@@ -3376,4 +3387,18 @@ malloc_conf = "narenas:1";]]></programlisting></para> 30 <para>The <function>posix_memalign()</function> function conforms 31 to IEEE Std 1003.1-2001 (<quote>POSIX.1</quote>).</para> 32 </refsect1> 33+ <refsect1 id="history"> 34+ <title>HISTORY</title> 35+ <para>The <function>malloc_usable_size()</function> and 36+ <function>posix_memalign()</function> functions first appeared in FreeBSD 37+ 7.0.</para> 38+ 39+ <para>The <function>aligned_alloc()</function>, 40+ <function>malloc_stats_print()</function>, and 41+ <function>mallctl*()</function> functions first appeared in FreeBSD 42+ 10.0.</para> 43+ 44+ <para>The <function>*allocx()</function> functions first appeared in FreeBSD 45+ 11.0.</para> 46+ </refsect1> 47 </refentry> 48diff --git a/include/jemalloc/internal/hooks.h b/include/jemalloc/internal/hooks.h 49index cd49afcb..85e2a991 100644 50--- a/include/jemalloc/internal/hooks.h 51+++ b/include/jemalloc/internal/hooks.h 52@@ -6,13 +6,6 @@ extern JEMALLOC_EXPORT void (*hooks_libc_hook)(); 53 54 #define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) 55 56-#define open JEMALLOC_HOOK(open, hooks_libc_hook) 57-#define read JEMALLOC_HOOK(read, hooks_libc_hook) 58-#define write JEMALLOC_HOOK(write, hooks_libc_hook) 59-#define readlink JEMALLOC_HOOK(readlink, hooks_libc_hook) 60-#define close JEMALLOC_HOOK(close, hooks_libc_hook) 61-#define creat JEMALLOC_HOOK(creat, hooks_libc_hook) 62-#define secure_getenv JEMALLOC_HOOK(secure_getenv, hooks_libc_hook) 63 /* Note that this is undef'd and re-define'd in src/prof.c. */ 64 #define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) 65 66diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h 67index be70df51..84cd70da 100644 68--- a/include/jemalloc/internal/jemalloc_internal_decls.h 69+++ b/include/jemalloc/internal/jemalloc_internal_decls.h 70@@ -1,6 +1,9 @@ 71 #ifndef JEMALLOC_INTERNAL_DECLS_H 72 #define JEMALLOC_INTERNAL_DECLS_H 73 74+#include "libc_private.h" 75+#include "namespace.h" 76+ 77 #include <math.h> 78 #ifdef _WIN32 79 # include <windows.h> 80diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in 81index e621fbc8..dbdd5d6b 100644 82--- a/include/jemalloc/internal/jemalloc_preamble.h.in 83+++ b/include/jemalloc/internal/jemalloc_preamble.h.in 84@@ -8,6 +8,9 @@ 85 #include <sys/ktrace.h> 86 #endif 87 88+#include "un-namespace.h" 89+#include "libc_private.h" 90+ 91 #define JEMALLOC_NO_DEMANGLE 92 #ifdef JEMALLOC_JET 93 # undef JEMALLOC_IS_MALLOC 94@@ -79,13 +82,7 @@ static const bool config_fill = 95 false 96 #endif 97 ; 98-static const bool config_lazy_lock = 99-#ifdef JEMALLOC_LAZY_LOCK 100- true 101-#else 102- false 103-#endif 104- ; 105+static const bool config_lazy_lock = true; 106 static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; 107 static const bool config_prof = 108 #ifdef JEMALLOC_PROF 109diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h 110index 6520c251..0013cbe9 100644 111--- a/include/jemalloc/internal/mutex.h 112+++ b/include/jemalloc/internal/mutex.h 113@@ -121,9 +121,6 @@ struct malloc_mutex_s { 114 115 #ifdef JEMALLOC_LAZY_LOCK 116 extern bool isthreaded; 117-#else 118-# undef isthreaded /* Undo private_namespace.h definition. */ 119-# define isthreaded true 120 #endif 121 122 bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 123@@ -131,6 +128,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 124 void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); 125 void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); 126 void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); 127+bool malloc_mutex_first_thread(void); 128 bool malloc_mutex_boot(void); 129 void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex); 130 131diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h 132index 0b9841aa..f03eee61 100644 133--- a/include/jemalloc/internal/tsd.h 134+++ b/include/jemalloc/internal/tsd.h 135@@ -122,7 +122,8 @@ struct tsd_s { 136 t use_a_getter_or_setter_instead_##n; 137 MALLOC_TSD 138 #undef O 139-}; 140+/* AddressSanitizer requires TLS data to be aligned to at least 8 bytes. */ 141+} JEMALLOC_ALIGNED(16); 142 143 /* 144 * Wrapper around tsd_t that makes it possible to avoid implicit conversion 145diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h 146new file mode 100644 147index 00000000..b752b0e7 148--- /dev/null 149+++ b/include/jemalloc/jemalloc_FreeBSD.h 150@@ -0,0 +1,185 @@ 151+/* 152+ * Override settings that were generated in jemalloc_defs.h as necessary. 153+ */ 154+ 155+#undef JEMALLOC_OVERRIDE_VALLOC 156+ 157+#ifndef MALLOC_PRODUCTION 158+#define JEMALLOC_DEBUG 159+#endif 160+ 161+#undef JEMALLOC_DSS 162+ 163+#undef JEMALLOC_BACKGROUND_THREAD 164+ 165+/* 166+ * The following are architecture-dependent, so conditionally define them for 167+ * each supported architecture. 168+ */ 169+#undef JEMALLOC_TLS_MODEL 170+#undef LG_PAGE 171+#undef LG_VADDR 172+#undef LG_SIZEOF_PTR 173+#undef LG_SIZEOF_INT 174+#undef LG_SIZEOF_LONG 175+#undef LG_SIZEOF_INTMAX_T 176+ 177+#ifdef __i386__ 178+# define LG_VADDR 32 179+# define LG_SIZEOF_PTR 2 180+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) 181+#endif 182+#ifdef __ia64__ 183+# define LG_VADDR 64 184+# define LG_SIZEOF_PTR 3 185+#endif 186+#ifdef __sparc64__ 187+# define LG_VADDR 64 188+# define LG_SIZEOF_PTR 3 189+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) 190+#endif 191+#ifdef __amd64__ 192+# define LG_VADDR 48 193+# define LG_SIZEOF_PTR 3 194+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) 195+#endif 196+#ifdef __arm__ 197+# define LG_VADDR 32 198+# define LG_SIZEOF_PTR 2 199+#endif 200+#ifdef __aarch64__ 201+# define LG_VADDR 48 202+# define LG_SIZEOF_PTR 3 203+#endif 204+#ifdef __mips__ 205+#ifdef __mips_n64 206+# define LG_VADDR 64 207+# define LG_SIZEOF_PTR 3 208+#else 209+# define LG_VADDR 32 210+# define LG_SIZEOF_PTR 2 211+#endif 212+#endif 213+#ifdef __powerpc64__ 214+# define LG_VADDR 64 215+# define LG_SIZEOF_PTR 3 216+#elif defined(__powerpc__) 217+# define LG_VADDR 32 218+# define LG_SIZEOF_PTR 2 219+#endif 220+#ifdef __riscv 221+# define LG_VADDR 64 222+# define LG_SIZEOF_PTR 3 223+#endif 224+ 225+#ifndef JEMALLOC_TLS_MODEL 226+# define JEMALLOC_TLS_MODEL /* Default. */ 227+#endif 228+ 229+#define LG_PAGE PAGE_SHIFT 230+#define LG_SIZEOF_INT 2 231+#define LG_SIZEOF_LONG LG_SIZEOF_PTR 232+#define LG_SIZEOF_INTMAX_T 3 233+ 234+#undef CPU_SPINWAIT 235+#include <machine/cpu.h> 236+#include <machine/cpufunc.h> 237+#define CPU_SPINWAIT cpu_spinwait() 238+ 239+/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */ 240+#undef JEMALLOC_LAZY_LOCK 241+extern int __isthreaded; 242+#define isthreaded ((bool)__isthreaded) 243+ 244+/* Mangle. */ 245+#undef je_malloc 246+#undef je_calloc 247+#undef je_posix_memalign 248+#undef je_aligned_alloc 249+#undef je_realloc 250+#undef je_free 251+#undef je_malloc_usable_size 252+#undef je_mallocx 253+#undef je_rallocx 254+#undef je_xallocx 255+#undef je_sallocx 256+#undef je_dallocx 257+#undef je_sdallocx 258+#undef je_nallocx 259+#undef je_mallctl 260+#undef je_mallctlnametomib 261+#undef je_mallctlbymib 262+#undef je_malloc_stats_print 263+#undef je_allocm 264+#undef je_rallocm 265+#undef je_sallocm 266+#undef je_dallocm 267+#undef je_nallocm 268+#define je_malloc __malloc 269+#define je_calloc __calloc 270+#define je_posix_memalign __posix_memalign 271+#define je_aligned_alloc __aligned_alloc 272+#define je_realloc __realloc 273+#define je_free __free 274+#define je_malloc_usable_size __malloc_usable_size 275+#define je_mallocx __mallocx 276+#define je_rallocx __rallocx 277+#define je_xallocx __xallocx 278+#define je_sallocx __sallocx 279+#define je_dallocx __dallocx 280+#define je_sdallocx __sdallocx 281+#define je_nallocx __nallocx 282+#define je_mallctl __mallctl 283+#define je_mallctlnametomib __mallctlnametomib 284+#define je_mallctlbymib __mallctlbymib 285+#define je_malloc_stats_print __malloc_stats_print 286+#define je_allocm __allocm 287+#define je_rallocm __rallocm 288+#define je_sallocm __sallocm 289+#define je_dallocm __dallocm 290+#define je_nallocm __nallocm 291+#define open _open 292+#define read _read 293+#define write _write 294+#define close _close 295+#define pthread_join _pthread_join 296+#define pthread_once _pthread_once 297+#define pthread_self _pthread_self 298+#define pthread_equal _pthread_equal 299+#define pthread_mutex_lock _pthread_mutex_lock 300+#define pthread_mutex_trylock _pthread_mutex_trylock 301+#define pthread_mutex_unlock _pthread_mutex_unlock 302+#define pthread_cond_init _pthread_cond_init 303+#define pthread_cond_wait _pthread_cond_wait 304+#define pthread_cond_timedwait _pthread_cond_timedwait 305+#define pthread_cond_signal _pthread_cond_signal 306+ 307+#ifdef JEMALLOC_C_ 308+/* 309+ * Define 'weak' symbols so that an application can have its own versions 310+ * of malloc, calloc, realloc, free, et al. 311+ */ 312+__weak_reference(__malloc, malloc); 313+__weak_reference(__calloc, calloc); 314+__weak_reference(__posix_memalign, posix_memalign); 315+__weak_reference(__aligned_alloc, aligned_alloc); 316+__weak_reference(__realloc, realloc); 317+__weak_reference(__free, free); 318+__weak_reference(__malloc_usable_size, malloc_usable_size); 319+__weak_reference(__mallocx, mallocx); 320+__weak_reference(__rallocx, rallocx); 321+__weak_reference(__xallocx, xallocx); 322+__weak_reference(__sallocx, sallocx); 323+__weak_reference(__dallocx, dallocx); 324+__weak_reference(__sdallocx, sdallocx); 325+__weak_reference(__nallocx, nallocx); 326+__weak_reference(__mallctl, mallctl); 327+__weak_reference(__mallctlnametomib, mallctlnametomib); 328+__weak_reference(__mallctlbymib, mallctlbymib); 329+__weak_reference(__malloc_stats_print, malloc_stats_print); 330+__weak_reference(__allocm, allocm); 331+__weak_reference(__rallocm, rallocm); 332+__weak_reference(__sallocm, sallocm); 333+__weak_reference(__dallocm, dallocm); 334+__weak_reference(__nallocm, nallocm); 335+#endif 336diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh 337index f9438912..47d032c1 100755 338--- a/include/jemalloc/jemalloc_rename.sh 339+++ b/include/jemalloc/jemalloc_rename.sh 340@@ -19,4 +19,6 @@ done 341 342 cat <<EOF 343 #endif 344+ 345+#include "jemalloc_FreeBSD.h" 346 EOF 347diff --git a/src/jemalloc.c b/src/jemalloc.c 348index f93c16fa..e0ad297b 100644 349--- a/src/jemalloc.c 350+++ b/src/jemalloc.c 351@@ -21,6 +21,10 @@ 352 /******************************************************************************/ 353 /* Data. */ 354 355+/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */ 356+const char *__malloc_options_1_0 = NULL; 357+__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 358+ 359 /* Runtime configuration options. */ 360 const char *je_malloc_conf 361 #ifndef _WIN32 362@@ -3160,6 +3164,103 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { 363 */ 364 /******************************************************************************/ 365 /* 366+ * Begin compatibility functions. 367+ */ 368+ 369+#define ALLOCM_LG_ALIGN(la) (la) 370+#define ALLOCM_ALIGN(a) (ffsl(a)-1) 371+#define ALLOCM_ZERO ((int)0x40) 372+#define ALLOCM_NO_MOVE ((int)0x80) 373+ 374+#define ALLOCM_SUCCESS 0 375+#define ALLOCM_ERR_OOM 1 376+#define ALLOCM_ERR_NOT_MOVED 2 377+ 378+int 379+je_allocm(void **ptr, size_t *rsize, size_t size, int flags) { 380+ assert(ptr != NULL); 381+ 382+ void *p = je_mallocx(size, flags); 383+ if (p == NULL) { 384+ return (ALLOCM_ERR_OOM); 385+ } 386+ if (rsize != NULL) { 387+ *rsize = isalloc(tsdn_fetch(), p); 388+ } 389+ *ptr = p; 390+ return ALLOCM_SUCCESS; 391+} 392+ 393+int 394+je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) { 395+ assert(ptr != NULL); 396+ assert(*ptr != NULL); 397+ assert(size != 0); 398+ assert(SIZE_T_MAX - size >= extra); 399+ 400+ int ret; 401+ bool no_move = flags & ALLOCM_NO_MOVE; 402+ 403+ if (no_move) { 404+ size_t usize = je_xallocx(*ptr, size, extra, flags); 405+ ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; 406+ if (rsize != NULL) { 407+ *rsize = usize; 408+ } 409+ } else { 410+ void *p = je_rallocx(*ptr, size+extra, flags); 411+ if (p != NULL) { 412+ *ptr = p; 413+ ret = ALLOCM_SUCCESS; 414+ } else { 415+ ret = ALLOCM_ERR_OOM; 416+ } 417+ if (rsize != NULL) { 418+ *rsize = isalloc(tsdn_fetch(), *ptr); 419+ } 420+ } 421+ return ret; 422+} 423+ 424+int 425+je_sallocm(const void *ptr, size_t *rsize, int flags) { 426+ assert(rsize != NULL); 427+ *rsize = je_sallocx(ptr, flags); 428+ return ALLOCM_SUCCESS; 429+} 430+ 431+int 432+je_dallocm(void *ptr, int flags) { 433+ je_dallocx(ptr, flags); 434+ return ALLOCM_SUCCESS; 435+} 436+ 437+int 438+je_nallocm(size_t *rsize, size_t size, int flags) { 439+ size_t usize = je_nallocx(size, flags); 440+ if (usize == 0) { 441+ return ALLOCM_ERR_OOM; 442+ } 443+ if (rsize != NULL) { 444+ *rsize = usize; 445+ } 446+ return ALLOCM_SUCCESS; 447+} 448+ 449+#undef ALLOCM_LG_ALIGN 450+#undef ALLOCM_ALIGN 451+#undef ALLOCM_ZERO 452+#undef ALLOCM_NO_MOVE 453+ 454+#undef ALLOCM_SUCCESS 455+#undef ALLOCM_ERR_OOM 456+#undef ALLOCM_ERR_NOT_MOVED 457+ 458+/* 459+ * End compatibility functions. 460+ */ 461+/******************************************************************************/ 462+/* 463 * The following functions are used by threading libraries for protection of 464 * malloc during fork(). 465 */ 466@@ -3323,4 +3424,11 @@ jemalloc_postfork_child(void) { 467 ctl_postfork_child(tsd_tsdn(tsd)); 468 } 469 470+void 471+_malloc_first_thread(void) 472+{ 473+ 474+ (void)malloc_mutex_first_thread(); 475+} 476+ 477 /******************************************************************************/ 478diff --git a/src/malloc_io.c b/src/malloc_io.c 479index 7bdc13f9..c8802c70 100644 480--- a/src/malloc_io.c 481+++ b/src/malloc_io.c 482@@ -75,6 +75,20 @@ wrtmessage(void *cbopaque, const char *s) { 483 484 JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); 485 486+JEMALLOC_ATTR(visibility("hidden")) 487+void 488+wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) { 489+ 490+ wrtmessage(NULL, s1); 491+ wrtmessage(NULL, s2); 492+ wrtmessage(NULL, s3); 493+ wrtmessage(NULL, s4); 494+} 495+ 496+void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3, 497+ const char *s4) = wrtmessage_1_0; 498+__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); 499+ 500 /* 501 * Wrapper around malloc_message() that avoids the need for 502 * je_malloc_message(...) throughout the code. 503diff --git a/src/mutex.c b/src/mutex.c 504index 30222b3e..b2c36283 100644 505--- a/src/mutex.c 506+++ b/src/mutex.c 507@@ -41,6 +41,17 @@ pthread_create(pthread_t *__restrict thread, 508 #ifdef JEMALLOC_MUTEX_INIT_CB 509 JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 510 void *(calloc_cb)(size_t, size_t)); 511+ 512+#pragma weak _pthread_mutex_init_calloc_cb 513+int 514+_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 515+ void *(calloc_cb)(size_t, size_t)) 516+{ 517+ 518+ return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t))) 519+ __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex, 520+ calloc_cb)); 521+} 522 #endif 523 524 void 525@@ -131,6 +142,16 @@ mutex_addr_comp(const witness_t *witness1, void *mutex1, 526 } 527 528 bool 529+malloc_mutex_first_thread(void) { 530+ 531+#ifndef JEMALLOC_MUTEX_INIT_CB 532+ return (malloc_mutex_first_thread()); 533+#else 534+ return (false); 535+#endif 536+} 537+ 538+bool 539 malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 540 witness_rank_t rank, malloc_mutex_lock_order_t lock_order) { 541 mutex_prof_data_init(&mutex->prof_data); 542