Lines Matching +full:clip +full:- +full:x +full:- +full:high
96 /* Below three are read-only after initialization. */
105 malloc_init_initialized = 0 /* Common case --> jnz. */
264 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive
389 unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1, in arena_bind()
395 bins->binshard[i] = shard % bin_infos[i].n_shards; in arena_bind()
489 * (narenas_tdata - narenas_actual)); in arena_tdata_get_hard()
521 * Determine binding for both non-internal and internal in arena_choose_hard()
640 /* Prevent tsd->arenas_tdata from being (re)created. */ in arenas_tdata_cleanup()
670 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); in stats_print_atexit()
671 ql_foreach(tcache, &arena->tcache_ql, link) { in stats_print_atexit()
675 &arena->tcache_ql_mtx); in stats_print_atexit()
754 return ((result == -1) ? 1 : (unsigned)result); in malloc_ncpus()
782 /* Reads the next size pair in a multi-sized option. */
792 /* First number, then '-' */ in malloc_conf_multi_sizes_next()
794 if (get_errno() != 0 || *end != '-') { in malloc_conf_multi_sizes_next()
820 *vlen_left -= end - *slab_size_segment_cur; in malloc_conf_multi_sizes_next()
839 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': in malloc_conf_next()
844 case 's': case 't': case 'u': case 'v': case 'w': case 'x': in malloc_conf_next()
853 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; in malloc_conf_next()
884 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; in malloc_conf_next()
888 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; in malloc_conf_next()
988 if (linklen == -1) { in obtain_malloc_conf()
1029 "string specified via --with-malloc-conf", in malloc_conf_init_helper()
1066 malloc_printf("<jemalloc>: -- " \ in malloc_conf_init_helper()
1074 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) in malloc_conf_init_helper()
1076 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) in malloc_conf_init_helper()
1092 * GCC -Wtype-limits warning, which we disable here and re-enable below. in malloc_conf_init_helper()
1101 #define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ in malloc_conf_init_helper() argument
1108 if (get_errno() != 0 || (uintptr_t)end -\ in malloc_conf_init_helper()
1112 } else if (clip) { \ in malloc_conf_init_helper()
1125 "Out-of-range " \ in malloc_conf_init_helper()
1135 clip) \ in malloc_conf_init_helper() argument
1137 check_min, check_max, clip) in malloc_conf_init_helper()
1138 #define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \ in malloc_conf_init_helper() argument
1140 check_min, check_max, clip) in malloc_conf_init_helper()
1148 if (get_errno() != 0 || (uintptr_t)end -\ in malloc_conf_init_helper()
1155 "Out-of-range conf value", \ in malloc_conf_init_helper()
1165 sizeof(o)-1) ? vlen : \ in malloc_conf_init_helper()
1166 sizeof(o)-1; \ in malloc_conf_init_helper()
1249 "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < in malloc_conf_init_helper()
1253 "muzzy_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < in malloc_conf_init_helper()
1296 -1, (sizeof(size_t) << 3) - 1) in malloc_conf_init_helper()
1376 - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, in malloc_conf_init_helper()
1380 "lg_prof_interval", -1, in malloc_conf_init_helper()
1381 (sizeof(uint64_t) << 3) - 1) in malloc_conf_init_helper()
1391 vlen : sizeof(log_var_names) - 1); in malloc_conf_init_helper()
1433 /* Re-enable diagnostic "-Wtype-limits" */ in malloc_conf_init_helper()
1469 /* Busy-wait until the initializing thread completes. */ in malloc_init_hard_needed()
1657 "configuration -- per physical CPU arena " in malloc_init_narenas()
1694 narenas_auto = MALLOCX_ARENA_LIMIT - 1; in malloc_init_narenas()
1813 * Begin allocation-path internal functions and data structures.
1850 /* The error string to use if the passed-in alignment is invalid. */
1854 * False if we're configured to skip some time-consuming operations.
1869 static_opts->may_overflow = false; in static_opts_init()
1870 static_opts->bump_empty_aligned_alloc = false; in static_opts_init()
1871 static_opts->assert_nonempty_alloc = false; in static_opts_init()
1872 static_opts->null_out_result_on_error = false; in static_opts_init()
1873 static_opts->set_errno_on_error = false; in static_opts_init()
1874 static_opts->min_alignment = 0; in static_opts_init()
1875 static_opts->oom_string = ""; in static_opts_init()
1876 static_opts->invalid_alignment_string = ""; in static_opts_init()
1877 static_opts->slow = false; in static_opts_init()
1878 static_opts->usize = false; in static_opts_init()
1886 #define TCACHE_IND_NONE ((unsigned)-1)
1887 #define TCACHE_IND_AUTOMATIC ((unsigned)-2)
1888 #define ARENA_IND_AUTOMATIC ((unsigned)-1)
1904 dynamic_opts->result = NULL; in dynamic_opts_init()
1905 dynamic_opts->usize = 0; in dynamic_opts_init()
1906 dynamic_opts->num_items = 0; in dynamic_opts_init()
1907 dynamic_opts->item_size = 0; in dynamic_opts_init()
1908 dynamic_opts->alignment = 0; in dynamic_opts_init()
1909 dynamic_opts->zero = false; in dynamic_opts_init()
1910 dynamic_opts->tcache_ind = TCACHE_IND_AUTOMATIC; in dynamic_opts_init()
1911 dynamic_opts->arena_ind = ARENA_IND_AUTOMATIC; in dynamic_opts_init()
1914 /* ind is ignored if dopts->alignment > 0. */
1922 if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) { in imalloc_no_sample()
1923 if (likely(!sopts->slow)) { in imalloc_no_sample()
1930 } else if (dopts->tcache_ind == TCACHE_IND_NONE) { in imalloc_no_sample()
1933 tcache = tcaches_get(tsd, dopts->tcache_ind); in imalloc_no_sample()
1937 if (dopts->arena_ind == ARENA_IND_AUTOMATIC) { in imalloc_no_sample()
1945 arena = arena_get(tsd_tsdn(tsd), dopts->arena_ind, true); in imalloc_no_sample()
1948 if (unlikely(dopts->alignment != 0)) { in imalloc_no_sample()
1949 return ipalloct(tsd_tsdn(tsd), usize, dopts->alignment, in imalloc_no_sample()
1950 dopts->zero, tcache, arena); in imalloc_no_sample()
1953 return iallocztm(tsd_tsdn(tsd), size, ind, dopts->zero, tcache, false, in imalloc_no_sample()
1954 arena, sopts->slow); in imalloc_no_sample()
1970 assert(((dopts->alignment == 0) ? in imalloc_sample()
1972 sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment)) in imalloc_sample()
2002 assert(dopts->num_items == 1); in compute_size_with_overflow()
2003 *size = dopts->item_size; in compute_size_with_overflow()
2007 /* A size_t with its high-half bits all set to 1. */ in compute_size_with_overflow()
2010 *size = dopts->item_size * dopts->num_items; in compute_size_with_overflow()
2013 return (dopts->num_items != 0 && dopts->item_size != 0); in compute_size_with_overflow()
2017 * We got a non-zero size, but we don't know if we overflowed to get in compute_size_with_overflow()
2022 if (likely((high_bits & (dopts->num_items | dopts->item_size)) == 0)) { in compute_size_with_overflow()
2025 if (likely(*size / dopts->item_size == dopts->num_items)) { in compute_size_with_overflow()
2053 if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts, in imalloc_body()
2058 if (unlikely(dopts->alignment < sopts->min_alignment in imalloc_body()
2059 || (dopts->alignment & (dopts->alignment - 1)) != 0)) { in imalloc_body()
2065 if (dopts->alignment == 0) { in imalloc_body()
2070 if (config_stats || (config_prof && opt_prof) || sopts->usize) { in imalloc_body()
2072 dopts->usize = usize; in imalloc_body()
2077 if (sopts->bump_empty_aligned_alloc) { in imalloc_body()
2082 usize = sz_sa2u(size, dopts->alignment); in imalloc_body()
2083 dopts->usize = usize; in imalloc_body()
2090 if (sopts->assert_nonempty_alloc) { in imalloc_body()
2098 * known-initialized arena (i.e. arena 0). in imalloc_body()
2101 if (sopts->slow && unlikely(reentrancy_level > 0)) { in imalloc_body()
2106 assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC || in imalloc_body()
2107 dopts->tcache_ind == TCACHE_IND_NONE); in imalloc_body()
2108 assert(dopts->arena_ind == ARENA_IND_AUTOMATIC); in imalloc_body()
2109 dopts->tcache_ind = TCACHE_IND_NONE; in imalloc_body()
2111 dopts->arena_ind = 0; in imalloc_body()
2132 * imalloc_sample ignores ind if dopts->alignment > 0. in imalloc_body()
2148 * If dopts->alignment > 0, then ind is still 0, but usize was in imalloc_body()
2162 * post-allocation work to do though. in imalloc_body()
2164 assert(dopts->alignment == 0 in imalloc_body()
2165 || ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0)); in imalloc_body()
2172 if (sopts->slow) { in imalloc_body()
2178 *dopts->result = allocation; in imalloc_body()
2182 if (unlikely(sopts->slow) && config_xmalloc && unlikely(opt_xmalloc)) { in imalloc_body()
2183 malloc_write(sopts->oom_string); in imalloc_body()
2187 if (sopts->slow) { in imalloc_body()
2193 if (sopts->set_errno_on_error) { in imalloc_body()
2197 if (sopts->null_out_result_on_error) { in imalloc_body()
2198 *dopts->result = NULL; in imalloc_body()
2205 * anyways to avoid obscuring the non-error paths, and for symmetry with in imalloc_body()
2210 malloc_write(sopts->invalid_alignment_string); in imalloc_body()
2214 if (sopts->set_errno_on_error) { in imalloc_body()
2218 if (sopts->slow) { in imalloc_body()
2224 if (sopts->null_out_result_on_error) { in imalloc_body()
2225 *dopts->result = NULL; in imalloc_body()
2235 malloc_write(sopts->oom_string); in imalloc_init_check()
2238 UTRACE(NULL, dopts->num_items * dopts->item_size, NULL); in imalloc_init_check()
2240 *dopts->result = NULL; in imalloc_init_check()
2248 /* Returns the errno-style error code of the allocation. */
2261 sopts->slow = false; in imalloc()
2268 sopts->slow = true; in imalloc()
2295 * Note that this branch gets optimized away -- it immediately follows in malloc_default()
2310 * Begin malloc(3)-compatible functions.
2317 * tcache. If either of these is false, we tail-call to the slowpath,
2318 * malloc_default(). Tail-calling is used to avoid any caller-saved
2322 * tail-call to the slowpath if they fire.
2341 if (unlikely(ticker_trytick(&tcache->gc_ticker))) { in JEMALLOC_ATTR()
2356 bytes_until_sample -= usize; in JEMALLOC_ATTR()
2380 bin->tstats.nrequests++; in JEMALLOC_ATTR()
2383 tcache->prof_accumbytes += usize; in JEMALLOC_ATTR()
2615 * the allocation was not sampled -- usize can be used to in isfree()
2761 * based on only free() calls -- other activities trigger the in free_default()
2765 * fully-setup tsd won't be destructed properly. in free_default()
2820 * Sampled objects are always page-aligned. The sampled object check in free_fastpath()
2829 if (unlikely(ticker_trytick(&tcache->gc_ticker))) { in free_fastpath()
2859 * End malloc(3)-compatible functions.
2863 * Begin non-standard override functions.
2946 * to inconsistently reference libc's malloc(3)-compatible functions
2997 * End non-standard override functions.
3001 * Begin non-standard functions.
3006 #define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y argument
3007 #define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \ argument
3008 JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y)
3019 * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488
3187 * be the same as the current usize because of in-place large in irallocx_prof()
3273 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); in je_rallocx()
3333 * usize isn't knowable before ixalloc() returns when extra is non-zero. in ixallocx_prof()
3386 assert(SIZE_T_MAX - size >= extra); in je_xallocx()
3411 if (unlikely(SC_LARGE_MAXCLASS - size < extra)) { in je_xallocx()
3412 extra = SC_LARGE_MAXCLASS - size; in je_xallocx()
3735 * End non-standard functions.
3743 #define ALLOCM_ALIGN(a) (ffsl(a)-1)
3771 assert(SIZE_T_MAX - size >= extra); in je_rallocm()