1diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in 2index 3d2e721..b361db2 100644 3--- a/doc/jemalloc.xml.in 4+++ b/doc/jemalloc.xml.in 5@@ -53,11 +53,23 @@ 6 <para>This manual describes jemalloc @jemalloc_version@. More information 7 can be found at the <ulink 8 url="http://jemalloc.net/">jemalloc website</ulink>.</para> 9+ 10+ <para>The following configuration options are enabled in libc's built-in 11+ jemalloc: <option>--enable-fill</option>, 12+ <option>--enable-lazy-lock</option>, <option>--enable-munmap</option>, 13+ <option>--enable-stats</option>, <option>--enable-tcache</option>, 14+ <option>--enable-tls</option>, <option>--enable-utrace</option>, and 15+ <option>--enable-xmalloc</option>. Additionally, 16+ <option>--enable-debug</option> is enabled in development versions of 17+ FreeBSD (controlled by the <constant>MALLOC_PRODUCTION</constant> make 18+ variable).</para> 19+ 20 </refsect1> 21 <refsynopsisdiv> 22 <title>SYNOPSIS</title> 23 <funcsynopsis> 24- <funcsynopsisinfo>#include <<filename class="headerfile">jemalloc/jemalloc.h</filename>></funcsynopsisinfo> 25+ <funcsynopsisinfo>#include <<filename class="headerfile">stdlib.h</filename>> 26+#include <<filename class="headerfile">malloc_np.h</filename>></funcsynopsisinfo> 27 <refsect2> 28 <title>Standard API</title> 29 <funcprototype> 30@@ -2963,4 +2975,18 @@ malloc_conf = "lg_chunk:24";]]></programlisting></para> 31 <para>The <function>posix_memalign()</function> function conforms 32 to IEEE Std 1003.1-2001 (<quote>POSIX.1</quote>).</para> 33 </refsect1> 34+ <refsect1 id="history"> 35+ <title>HISTORY</title> 36+ <para>The <function>malloc_usable_size()</function> and 37+ <function>posix_memalign()</function> functions first appeared in FreeBSD 38+ 7.0.</para> 39+ 40+ <para>The <function>aligned_alloc()</function>, 41+ <function>malloc_stats_print()</function>, and 42+ <function>mallctl*()</function> functions first appeared in FreeBSD 43+ 10.0.</para> 44+ 45+ <para>The <function>*allocx()</function> functions first appeared in FreeBSD 46+ 11.0.</para> 47+ </refsect1> 48 </refentry> 49diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h 50index f39ce54..a3ba55d 100644 51--- a/include/jemalloc/internal/arena.h 52+++ b/include/jemalloc/internal/arena.h 53@@ -719,8 +719,13 @@ arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind) 54 JEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t * 55 arena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind) 56 { 57+#if 1 /* Work around gcc bug. */ 58+ arena_chunk_t *mchunk = (arena_chunk_t *)chunk; 59 60+ return (arena_miscelm_get_mutable(mchunk, pageind)); 61+#else 62 return (arena_miscelm_get_mutable((arena_chunk_t *)chunk, pageind)); 63+#endif 64 } 65 66 JEMALLOC_ALWAYS_INLINE size_t 67@@ -779,8 +784,13 @@ arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind) 68 JEMALLOC_ALWAYS_INLINE const size_t * 69 arena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind) 70 { 71+#if 1 /* Work around gcc bug. */ 72+ arena_chunk_t *mchunk = (arena_chunk_t *)chunk; 73 74+ return (arena_mapbitsp_get_mutable(mchunk, pageind)); 75+#else 76 return (arena_mapbitsp_get_mutable((arena_chunk_t *)chunk, pageind)); 77+#endif 78 } 79 80 JEMALLOC_ALWAYS_INLINE size_t 81diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in 82index fdc8fef..56a35a4 100644 83--- a/include/jemalloc/internal/jemalloc_internal.h.in 84+++ b/include/jemalloc/internal/jemalloc_internal.h.in 85@@ -8,6 +8,9 @@ 86 #include <sys/ktrace.h> 87 #endif 88 89+#include "un-namespace.h" 90+#include "libc_private.h" 91+ 92 #define JEMALLOC_NO_DEMANGLE 93 #ifdef JEMALLOC_JET 94 # define JEMALLOC_N(n) jet_##n 95@@ -42,13 +45,7 @@ static const bool config_fill = 96 false 97 #endif 98 ; 99-static const bool config_lazy_lock = 100-#ifdef JEMALLOC_LAZY_LOCK 101- true 102-#else 103- false 104-#endif 105- ; 106+static const bool config_lazy_lock = true; 107 static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; 108 static const bool config_prof = 109 #ifdef JEMALLOC_PROF 110diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h 111index c907d91..4626632 100644 112--- a/include/jemalloc/internal/jemalloc_internal_decls.h 113+++ b/include/jemalloc/internal/jemalloc_internal_decls.h 114@@ -1,6 +1,9 @@ 115 #ifndef JEMALLOC_INTERNAL_DECLS_H 116 #define JEMALLOC_INTERNAL_DECLS_H 117 118+#include "libc_private.h" 119+#include "namespace.h" 120+ 121 #include <math.h> 122 #ifdef _WIN32 123 # include <windows.h> 124diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h 125index b442d2d..76518db 100644 126--- a/include/jemalloc/internal/mutex.h 127+++ b/include/jemalloc/internal/mutex.h 128@@ -57,9 +57,6 @@ struct malloc_mutex_s { 129 130 #ifdef JEMALLOC_LAZY_LOCK 131 extern bool isthreaded; 132-#else 133-# undef isthreaded /* Undo private_namespace.h definition. */ 134-# define isthreaded true 135 #endif 136 137 bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 138@@ -67,6 +64,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 139 void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); 140 void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); 141 void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); 142+bool malloc_mutex_first_thread(void); 143 bool malloc_mutex_boot(void); 144 145 #endif /* JEMALLOC_H_EXTERNS */ 146diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt 147index 87c8c9b..df576f6 100644 148--- a/include/jemalloc/internal/private_symbols.txt 149+++ b/include/jemalloc/internal/private_symbols.txt 150@@ -307,7 +307,6 @@ iralloct_realign 151 isalloc 152 isdalloct 153 isqalloc 154-isthreaded 155 ivsalloc 156 ixalloc 157 jemalloc_postfork_child 158diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h 159new file mode 100644 160index 0000000..c58a8f3 161--- /dev/null 162+++ b/include/jemalloc/jemalloc_FreeBSD.h 163@@ -0,0 +1,162 @@ 164+/* 165+ * Override settings that were generated in jemalloc_defs.h as necessary. 166+ */ 167+ 168+#undef JEMALLOC_OVERRIDE_VALLOC 169+ 170+#ifndef MALLOC_PRODUCTION 171+#define JEMALLOC_DEBUG 172+#endif 173+ 174+#undef JEMALLOC_DSS 175+ 176+/* 177+ * The following are architecture-dependent, so conditionally define them for 178+ * each supported architecture. 179+ */ 180+#undef JEMALLOC_TLS_MODEL 181+#undef STATIC_PAGE_SHIFT 182+#undef LG_SIZEOF_PTR 183+#undef LG_SIZEOF_INT 184+#undef LG_SIZEOF_LONG 185+#undef LG_SIZEOF_INTMAX_T 186+ 187+#ifdef __i386__ 188+# define LG_SIZEOF_PTR 2 189+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) 190+#endif 191+#ifdef __ia64__ 192+# define LG_SIZEOF_PTR 3 193+#endif 194+#ifdef __sparc64__ 195+# define LG_SIZEOF_PTR 3 196+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) 197+#endif 198+#ifdef __amd64__ 199+# define LG_SIZEOF_PTR 3 200+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) 201+#endif 202+#ifdef __arm__ 203+# define LG_SIZEOF_PTR 2 204+#endif 205+#ifdef __aarch64__ 206+# define LG_SIZEOF_PTR 3 207+#endif 208+#ifdef __mips__ 209+#ifdef __mips_n64 210+# define LG_SIZEOF_PTR 3 211+#else 212+# define LG_SIZEOF_PTR 2 213+#endif 214+#endif 215+#ifdef __powerpc64__ 216+# define LG_SIZEOF_PTR 3 217+#elif defined(__powerpc__) 218+# define LG_SIZEOF_PTR 2 219+#endif 220+#ifdef __riscv__ 221+# define LG_SIZEOF_PTR 3 222+#endif 223+ 224+#ifndef JEMALLOC_TLS_MODEL 225+# define JEMALLOC_TLS_MODEL /* Default. */ 226+#endif 227+ 228+#define STATIC_PAGE_SHIFT PAGE_SHIFT 229+#define LG_SIZEOF_INT 2 230+#define LG_SIZEOF_LONG LG_SIZEOF_PTR 231+#define LG_SIZEOF_INTMAX_T 3 232+ 233+#undef CPU_SPINWAIT 234+#include <machine/cpu.h> 235+#include <machine/cpufunc.h> 236+#define CPU_SPINWAIT cpu_spinwait() 237+ 238+/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */ 239+#undef JEMALLOC_LAZY_LOCK 240+extern int __isthreaded; 241+#define isthreaded ((bool)__isthreaded) 242+ 243+/* Mangle. */ 244+#undef je_malloc 245+#undef je_calloc 246+#undef je_posix_memalign 247+#undef je_aligned_alloc 248+#undef je_realloc 249+#undef je_free 250+#undef je_malloc_usable_size 251+#undef je_mallocx 252+#undef je_rallocx 253+#undef je_xallocx 254+#undef je_sallocx 255+#undef je_dallocx 256+#undef je_sdallocx 257+#undef je_nallocx 258+#undef je_mallctl 259+#undef je_mallctlnametomib 260+#undef je_mallctlbymib 261+#undef je_malloc_stats_print 262+#undef je_allocm 263+#undef je_rallocm 264+#undef je_sallocm 265+#undef je_dallocm 266+#undef je_nallocm 267+#define je_malloc __malloc 268+#define je_calloc __calloc 269+#define je_posix_memalign __posix_memalign 270+#define je_aligned_alloc __aligned_alloc 271+#define je_realloc __realloc 272+#define je_free __free 273+#define je_malloc_usable_size __malloc_usable_size 274+#define je_mallocx __mallocx 275+#define je_rallocx __rallocx 276+#define je_xallocx __xallocx 277+#define je_sallocx __sallocx 278+#define je_dallocx __dallocx 279+#define je_sdallocx __sdallocx 280+#define je_nallocx __nallocx 281+#define je_mallctl __mallctl 282+#define je_mallctlnametomib __mallctlnametomib 283+#define je_mallctlbymib __mallctlbymib 284+#define je_malloc_stats_print __malloc_stats_print 285+#define je_allocm __allocm 286+#define je_rallocm __rallocm 287+#define je_sallocm __sallocm 288+#define je_dallocm __dallocm 289+#define je_nallocm __nallocm 290+#define open _open 291+#define read _read 292+#define write _write 293+#define close _close 294+#define pthread_mutex_lock _pthread_mutex_lock 295+#define pthread_mutex_unlock _pthread_mutex_unlock 296+ 297+#ifdef JEMALLOC_C_ 298+/* 299+ * Define 'weak' symbols so that an application can have its own versions 300+ * of malloc, calloc, realloc, free, et al. 301+ */ 302+__weak_reference(__malloc, malloc); 303+__weak_reference(__calloc, calloc); 304+__weak_reference(__posix_memalign, posix_memalign); 305+__weak_reference(__aligned_alloc, aligned_alloc); 306+__weak_reference(__realloc, realloc); 307+__weak_reference(__free, free); 308+__weak_reference(__malloc_usable_size, malloc_usable_size); 309+__weak_reference(__mallocx, mallocx); 310+__weak_reference(__rallocx, rallocx); 311+__weak_reference(__xallocx, xallocx); 312+__weak_reference(__sallocx, sallocx); 313+__weak_reference(__dallocx, dallocx); 314+__weak_reference(__sdallocx, sdallocx); 315+__weak_reference(__nallocx, nallocx); 316+__weak_reference(__mallctl, mallctl); 317+__weak_reference(__mallctlnametomib, mallctlnametomib); 318+__weak_reference(__mallctlbymib, mallctlbymib); 319+__weak_reference(__malloc_stats_print, malloc_stats_print); 320+__weak_reference(__allocm, allocm); 321+__weak_reference(__rallocm, rallocm); 322+__weak_reference(__sallocm, sallocm); 323+__weak_reference(__dallocm, dallocm); 324+__weak_reference(__nallocm, nallocm); 325+#endif 326diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh 327index f943891..47d032c 100755 328--- a/include/jemalloc/jemalloc_rename.sh 329+++ b/include/jemalloc/jemalloc_rename.sh 330@@ -19,4 +19,6 @@ done 331 332 cat <<EOF 333 #endif 334+ 335+#include "jemalloc_FreeBSD.h" 336 EOF 337diff --git a/src/jemalloc.c b/src/jemalloc.c 338index 38650ff..f659b55 100644 339--- a/src/jemalloc.c 340+++ b/src/jemalloc.c 341@@ -4,6 +4,10 @@ 342 /******************************************************************************/ 343 /* Data. */ 344 345+/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */ 346+const char *__malloc_options_1_0 = NULL; 347+__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 348+ 349 /* Runtime configuration options. */ 350 const char *je_malloc_conf 351 #ifndef _WIN32 352@@ -2756,6 +2760,107 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 353 */ 354 /******************************************************************************/ 355 /* 356+ * Begin compatibility functions. 357+ */ 358+ 359+#define ALLOCM_LG_ALIGN(la) (la) 360+#define ALLOCM_ALIGN(a) (ffsl(a)-1) 361+#define ALLOCM_ZERO ((int)0x40) 362+#define ALLOCM_NO_MOVE ((int)0x80) 363+ 364+#define ALLOCM_SUCCESS 0 365+#define ALLOCM_ERR_OOM 1 366+#define ALLOCM_ERR_NOT_MOVED 2 367+ 368+int 369+je_allocm(void **ptr, size_t *rsize, size_t size, int flags) 370+{ 371+ void *p; 372+ 373+ assert(ptr != NULL); 374+ 375+ p = je_mallocx(size, flags); 376+ if (p == NULL) 377+ return (ALLOCM_ERR_OOM); 378+ if (rsize != NULL) 379+ *rsize = isalloc(tsdn_fetch(), p, config_prof); 380+ *ptr = p; 381+ return (ALLOCM_SUCCESS); 382+} 383+ 384+int 385+je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) 386+{ 387+ int ret; 388+ bool no_move = flags & ALLOCM_NO_MOVE; 389+ 390+ assert(ptr != NULL); 391+ assert(*ptr != NULL); 392+ assert(size != 0); 393+ assert(SIZE_T_MAX - size >= extra); 394+ 395+ if (no_move) { 396+ size_t usize = je_xallocx(*ptr, size, extra, flags); 397+ ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; 398+ if (rsize != NULL) 399+ *rsize = usize; 400+ } else { 401+ void *p = je_rallocx(*ptr, size+extra, flags); 402+ if (p != NULL) { 403+ *ptr = p; 404+ ret = ALLOCM_SUCCESS; 405+ } else 406+ ret = ALLOCM_ERR_OOM; 407+ if (rsize != NULL) 408+ *rsize = isalloc(tsdn_fetch(), *ptr, config_prof); 409+ } 410+ return (ret); 411+} 412+ 413+int 414+je_sallocm(const void *ptr, size_t *rsize, int flags) 415+{ 416+ 417+ assert(rsize != NULL); 418+ *rsize = je_sallocx(ptr, flags); 419+ return (ALLOCM_SUCCESS); 420+} 421+ 422+int 423+je_dallocm(void *ptr, int flags) 424+{ 425+ 426+ je_dallocx(ptr, flags); 427+ return (ALLOCM_SUCCESS); 428+} 429+ 430+int 431+je_nallocm(size_t *rsize, size_t size, int flags) 432+{ 433+ size_t usize; 434+ 435+ usize = je_nallocx(size, flags); 436+ if (usize == 0) 437+ return (ALLOCM_ERR_OOM); 438+ if (rsize != NULL) 439+ *rsize = usize; 440+ return (ALLOCM_SUCCESS); 441+} 442+ 443+#undef ALLOCM_LG_ALIGN 444+#undef ALLOCM_ALIGN 445+#undef ALLOCM_ZERO 446+#undef ALLOCM_NO_MOVE 447+ 448+#undef ALLOCM_SUCCESS 449+#undef ALLOCM_ERR_OOM 450+#undef ALLOCM_ERR_NOT_MOVED 451+ 452+/* 453+ * End compatibility functions. 454+ */ 455+/******************************************************************************/ 456+/* 457 * The following functions are used by threading libraries for protection of 458 * malloc during fork(). 459 */ 460@@ -2894,4 +2999,11 @@ jemalloc_postfork_child(void) 461 ctl_postfork_child(tsd_tsdn(tsd)); 462 } 463 464+void 465+_malloc_first_thread(void) 466+{ 467+ 468+ (void)malloc_mutex_first_thread(); 469+} 470+ 471 /******************************************************************************/ 472diff --git a/src/mutex.c b/src/mutex.c 473index 6333e73..13f8d79 100644 474--- a/src/mutex.c 475+++ b/src/mutex.c 476@@ -66,6 +66,17 @@ pthread_create(pthread_t *__restrict thread, 477 #ifdef JEMALLOC_MUTEX_INIT_CB 478 JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 479 void *(calloc_cb)(size_t, size_t)); 480+ 481+#pragma weak _pthread_mutex_init_calloc_cb 482+int 483+_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 484+ void *(calloc_cb)(size_t, size_t)) 485+{ 486+ 487+ return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t))) 488+ __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex, 489+ calloc_cb)); 490+} 491 #endif 492 493 bool 494@@ -142,7 +153,7 @@ malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex) 495 } 496 497 bool 498-malloc_mutex_boot(void) 499+malloc_mutex_first_thread(void) 500 { 501 502 #ifdef JEMALLOC_MUTEX_INIT_CB 503@@ -156,3 +167,14 @@ malloc_mutex_boot(void) 504 #endif 505 return (false); 506 } 507+ 508+bool 509+malloc_mutex_boot(void) 510+{ 511+ 512+#ifndef JEMALLOC_MUTEX_INIT_CB 513+ return (malloc_mutex_first_thread()); 514+#else 515+ return (false); 516+#endif 517+} 518diff --git a/src/util.c b/src/util.c 519index 7905267..bee1c77 100644 520--- a/src/util.c 521+++ b/src/util.c 522@@ -67,6 +67,22 @@ wrtmessage(void *cbopaque, const char *s) 523 524 JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); 525 526+JEMALLOC_ATTR(visibility("hidden")) 527+void 528+wrtmessage_1_0(const char *s1, const char *s2, const char *s3, 529+ const char *s4) 530+{ 531+ 532+ wrtmessage(NULL, s1); 533+ wrtmessage(NULL, s2); 534+ wrtmessage(NULL, s3); 535+ wrtmessage(NULL, s4); 536+} 537+ 538+void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3, 539+ const char *s4) = wrtmessage_1_0; 540+__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); 541+ 542 /* 543 * Wrapper around malloc_message() that avoids the need for 544 * je_malloc_message(...) throughout the code. 545