1 #ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H
2 #define JEMALLOC_INTERNAL_ARENA_INLINES_B_H
3
4 #include "jemalloc/internal/jemalloc_internal_types.h"
5 #include "jemalloc/internal/mutex.h"
6 #include "jemalloc/internal/rtree.h"
7 #include "jemalloc/internal/sc.h"
8 #include "jemalloc/internal/sz.h"
9 #include "jemalloc/internal/ticker.h"
10
11 JEMALLOC_ALWAYS_INLINE bool
arena_has_default_hooks(arena_t * arena)12 arena_has_default_hooks(arena_t *arena) {
13 return (extent_hooks_get(arena) == &extent_hooks_default);
14 }
15
16 JEMALLOC_ALWAYS_INLINE arena_t *
arena_choose_maybe_huge(tsd_t * tsd,arena_t * arena,size_t size)17 arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
18 if (arena != NULL) {
19 return arena;
20 }
21
22 /*
23 * For huge allocations, use the dedicated huge arena if both are true:
24 * 1) is using auto arena selection (i.e. arena == NULL), and 2) the
25 * thread is not assigned to a manual arena.
26 */
27 if (unlikely(size >= oversize_threshold)) {
28 arena_t *tsd_arena = tsd_arena_get(tsd);
29 if (tsd_arena == NULL || arena_is_auto(tsd_arena)) {
30 return arena_choose_huge(tsd);
31 }
32 }
33
34 return arena_choose(tsd, NULL);
35 }
36
37 JEMALLOC_ALWAYS_INLINE prof_tctx_t *
arena_prof_tctx_get(tsdn_t * tsdn,const void * ptr,alloc_ctx_t * alloc_ctx)38 arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) {
39 cassert(config_prof);
40 assert(ptr != NULL);
41
42 /* Static check. */
43 if (alloc_ctx == NULL) {
44 const extent_t *extent = iealloc(tsdn, ptr);
45 if (unlikely(!extent_slab_get(extent))) {
46 return large_prof_tctx_get(tsdn, extent);
47 }
48 } else {
49 if (unlikely(!alloc_ctx->slab)) {
50 return large_prof_tctx_get(tsdn, iealloc(tsdn, ptr));
51 }
52 }
53 return (prof_tctx_t *)(uintptr_t)1U;
54 }
55
56 JEMALLOC_ALWAYS_INLINE void
arena_prof_tctx_set(tsdn_t * tsdn,const void * ptr,size_t usize,alloc_ctx_t * alloc_ctx,prof_tctx_t * tctx)57 arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
58 alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) {
59 cassert(config_prof);
60 assert(ptr != NULL);
61
62 /* Static check. */
63 if (alloc_ctx == NULL) {
64 extent_t *extent = iealloc(tsdn, ptr);
65 if (unlikely(!extent_slab_get(extent))) {
66 large_prof_tctx_set(tsdn, extent, tctx);
67 }
68 } else {
69 if (unlikely(!alloc_ctx->slab)) {
70 large_prof_tctx_set(tsdn, iealloc(tsdn, ptr), tctx);
71 }
72 }
73 }
74
75 static inline void
arena_prof_tctx_reset(tsdn_t * tsdn,const void * ptr,prof_tctx_t * tctx)76 arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) {
77 cassert(config_prof);
78 assert(ptr != NULL);
79
80 extent_t *extent = iealloc(tsdn, ptr);
81 assert(!extent_slab_get(extent));
82
83 large_prof_tctx_reset(tsdn, extent);
84 }
85
86 JEMALLOC_ALWAYS_INLINE nstime_t
arena_prof_alloc_time_get(tsdn_t * tsdn,const void * ptr,alloc_ctx_t * alloc_ctx)87 arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr,
88 alloc_ctx_t *alloc_ctx) {
89 cassert(config_prof);
90 assert(ptr != NULL);
91
92 extent_t *extent = iealloc(tsdn, ptr);
93 /*
94 * Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're
95 * sure we have a sampled allocation.
96 */
97 assert(!extent_slab_get(extent));
98 return large_prof_alloc_time_get(extent);
99 }
100
101 JEMALLOC_ALWAYS_INLINE void
arena_prof_alloc_time_set(tsdn_t * tsdn,const void * ptr,alloc_ctx_t * alloc_ctx,nstime_t t)102 arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx,
103 nstime_t t) {
104 cassert(config_prof);
105 assert(ptr != NULL);
106
107 extent_t *extent = iealloc(tsdn, ptr);
108 assert(!extent_slab_get(extent));
109 large_prof_alloc_time_set(extent, t);
110 }
111
112 JEMALLOC_ALWAYS_INLINE void
arena_decay_ticks(tsdn_t * tsdn,arena_t * arena,unsigned nticks)113 arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
114 tsd_t *tsd;
115 ticker_t *decay_ticker;
116
117 if (unlikely(tsdn_null(tsdn))) {
118 return;
119 }
120 tsd = tsdn_tsd(tsdn);
121 decay_ticker = decay_ticker_get(tsd, arena_ind_get(arena));
122 if (unlikely(decay_ticker == NULL)) {
123 return;
124 }
125 if (unlikely(ticker_ticks(decay_ticker, nticks))) {
126 arena_decay(tsdn, arena, false, false);
127 }
128 }
129
130 JEMALLOC_ALWAYS_INLINE void
arena_decay_tick(tsdn_t * tsdn,arena_t * arena)131 arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
132 malloc_mutex_assert_not_owner(tsdn, &arena->decay_dirty.mtx);
133 malloc_mutex_assert_not_owner(tsdn, &arena->decay_muzzy.mtx);
134
135 arena_decay_ticks(tsdn, arena, 1);
136 }
137
138 /* Purge a single extent to retained / unmapped directly. */
139 JEMALLOC_ALWAYS_INLINE void
arena_decay_extent(tsdn_t * tsdn,arena_t * arena,extent_hooks_t ** r_extent_hooks,extent_t * extent)140 arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t **r_extent_hooks,
141 extent_t *extent) {
142 size_t extent_size = extent_size_get(extent);
143 extent_dalloc_wrapper(tsdn, arena,
144 r_extent_hooks, extent);
145 if (config_stats) {
146 /* Update stats accordingly. */
147 arena_stats_lock(tsdn, &arena->stats);
148 arena_stats_add_u64(tsdn, &arena->stats,
149 &arena->decay_dirty.stats->nmadvise, 1);
150 arena_stats_add_u64(tsdn, &arena->stats,
151 &arena->decay_dirty.stats->purged, extent_size >> LG_PAGE);
152 arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped,
153 extent_size);
154 arena_stats_unlock(tsdn, &arena->stats);
155 }
156 }
157
158 JEMALLOC_ALWAYS_INLINE void *
arena_malloc(tsdn_t * tsdn,arena_t * arena,size_t size,szind_t ind,bool zero,tcache_t * tcache,bool slow_path)159 arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
160 tcache_t *tcache, bool slow_path) {
161 assert(!tsdn_null(tsdn) || tcache == NULL);
162
163 if (likely(tcache != NULL)) {
164 if (likely(size <= SC_SMALL_MAXCLASS)) {
165 return tcache_alloc_small(tsdn_tsd(tsdn), arena,
166 tcache, size, ind, zero, slow_path);
167 }
168 if (likely(size <= tcache_maxclass)) {
169 return tcache_alloc_large(tsdn_tsd(tsdn), arena,
170 tcache, size, ind, zero, slow_path);
171 }
172 /* (size > tcache_maxclass) case falls through. */
173 assert(size > tcache_maxclass);
174 }
175
176 return arena_malloc_hard(tsdn, arena, size, ind, zero);
177 }
178
179 JEMALLOC_ALWAYS_INLINE arena_t *
arena_aalloc(tsdn_t * tsdn,const void * ptr)180 arena_aalloc(tsdn_t *tsdn, const void *ptr) {
181 return extent_arena_get(iealloc(tsdn, ptr));
182 }
183
184 JEMALLOC_ALWAYS_INLINE size_t
arena_salloc(tsdn_t * tsdn,const void * ptr)185 arena_salloc(tsdn_t *tsdn, const void *ptr) {
186 assert(ptr != NULL);
187
188 rtree_ctx_t rtree_ctx_fallback;
189 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
190
191 szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx,
192 (uintptr_t)ptr, true);
193 assert(szind != SC_NSIZES);
194
195 return sz_index2size(szind);
196 }
197
198 JEMALLOC_ALWAYS_INLINE size_t
arena_vsalloc(tsdn_t * tsdn,const void * ptr)199 arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
200 /*
201 * Return 0 if ptr is not within an extent managed by jemalloc. This
202 * function has two extra costs relative to isalloc():
203 * - The rtree calls cannot claim to be dependent lookups, which induces
204 * rtree lookup load dependencies.
205 * - The lookup may fail, so there is an extra branch to check for
206 * failure.
207 */
208
209 rtree_ctx_t rtree_ctx_fallback;
210 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
211
212 extent_t *extent;
213 szind_t szind;
214 if (rtree_extent_szind_read(tsdn, &extents_rtree, rtree_ctx,
215 (uintptr_t)ptr, false, &extent, &szind)) {
216 return 0;
217 }
218
219 if (extent == NULL) {
220 return 0;
221 }
222 assert(extent_state_get(extent) == extent_state_active);
223 /* Only slab members should be looked up via interior pointers. */
224 assert(extent_addr_get(extent) == ptr || extent_slab_get(extent));
225
226 assert(szind != SC_NSIZES);
227
228 return sz_index2size(szind);
229 }
230
231 static inline void
arena_dalloc_large_no_tcache(tsdn_t * tsdn,void * ptr,szind_t szind)232 arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind) {
233 if (config_prof && unlikely(szind < SC_NBINS)) {
234 arena_dalloc_promoted(tsdn, ptr, NULL, true);
235 } else {
236 extent_t *extent = iealloc(tsdn, ptr);
237 large_dalloc(tsdn, extent);
238 }
239 }
240
241 static inline void
arena_dalloc_no_tcache(tsdn_t * tsdn,void * ptr)242 arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
243 assert(ptr != NULL);
244
245 rtree_ctx_t rtree_ctx_fallback;
246 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
247
248 szind_t szind;
249 bool slab;
250 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
251 true, &szind, &slab);
252
253 if (config_debug) {
254 extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
255 rtree_ctx, (uintptr_t)ptr, true);
256 assert(szind == extent_szind_get(extent));
257 assert(szind < SC_NSIZES);
258 assert(slab == extent_slab_get(extent));
259 }
260
261 if (likely(slab)) {
262 /* Small allocation. */
263 arena_dalloc_small(tsdn, ptr);
264 } else {
265 arena_dalloc_large_no_tcache(tsdn, ptr, szind);
266 }
267 }
268
269 JEMALLOC_ALWAYS_INLINE void
arena_dalloc_large(tsdn_t * tsdn,void * ptr,tcache_t * tcache,szind_t szind,bool slow_path)270 arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
271 bool slow_path) {
272 if (szind < nhbins) {
273 if (config_prof && unlikely(szind < SC_NBINS)) {
274 arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
275 } else {
276 tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind,
277 slow_path);
278 }
279 } else {
280 extent_t *extent = iealloc(tsdn, ptr);
281 large_dalloc(tsdn, extent);
282 }
283 }
284
285 JEMALLOC_ALWAYS_INLINE void
arena_dalloc(tsdn_t * tsdn,void * ptr,tcache_t * tcache,alloc_ctx_t * alloc_ctx,bool slow_path)286 arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
287 alloc_ctx_t *alloc_ctx, bool slow_path) {
288 assert(!tsdn_null(tsdn) || tcache == NULL);
289 assert(ptr != NULL);
290
291 if (unlikely(tcache == NULL)) {
292 arena_dalloc_no_tcache(tsdn, ptr);
293 return;
294 }
295
296 szind_t szind;
297 bool slab;
298 rtree_ctx_t *rtree_ctx;
299 if (alloc_ctx != NULL) {
300 szind = alloc_ctx->szind;
301 slab = alloc_ctx->slab;
302 assert(szind != SC_NSIZES);
303 } else {
304 rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
305 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
306 (uintptr_t)ptr, true, &szind, &slab);
307 }
308
309 if (config_debug) {
310 rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
311 extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
312 rtree_ctx, (uintptr_t)ptr, true);
313 assert(szind == extent_szind_get(extent));
314 assert(szind < SC_NSIZES);
315 assert(slab == extent_slab_get(extent));
316 }
317
318 if (likely(slab)) {
319 /* Small allocation. */
320 tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
321 slow_path);
322 } else {
323 arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path);
324 }
325 }
326
327 static inline void
arena_sdalloc_no_tcache(tsdn_t * tsdn,void * ptr,size_t size)328 arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
329 assert(ptr != NULL);
330 assert(size <= SC_LARGE_MAXCLASS);
331
332 szind_t szind;
333 bool slab;
334 if (!config_prof || !opt_prof) {
335 /*
336 * There is no risk of being confused by a promoted sampled
337 * object, so base szind and slab on the given size.
338 */
339 szind = sz_size2index(size);
340 slab = (szind < SC_NBINS);
341 }
342
343 if ((config_prof && opt_prof) || config_debug) {
344 rtree_ctx_t rtree_ctx_fallback;
345 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
346 &rtree_ctx_fallback);
347
348 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
349 (uintptr_t)ptr, true, &szind, &slab);
350
351 assert(szind == sz_size2index(size));
352 assert((config_prof && opt_prof) || slab == (szind < SC_NBINS));
353
354 if (config_debug) {
355 extent_t *extent = rtree_extent_read(tsdn,
356 &extents_rtree, rtree_ctx, (uintptr_t)ptr, true);
357 assert(szind == extent_szind_get(extent));
358 assert(slab == extent_slab_get(extent));
359 }
360 }
361
362 if (likely(slab)) {
363 /* Small allocation. */
364 arena_dalloc_small(tsdn, ptr);
365 } else {
366 arena_dalloc_large_no_tcache(tsdn, ptr, szind);
367 }
368 }
369
370 JEMALLOC_ALWAYS_INLINE void
arena_sdalloc(tsdn_t * tsdn,void * ptr,size_t size,tcache_t * tcache,alloc_ctx_t * alloc_ctx,bool slow_path)371 arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
372 alloc_ctx_t *alloc_ctx, bool slow_path) {
373 assert(!tsdn_null(tsdn) || tcache == NULL);
374 assert(ptr != NULL);
375 assert(size <= SC_LARGE_MAXCLASS);
376
377 if (unlikely(tcache == NULL)) {
378 arena_sdalloc_no_tcache(tsdn, ptr, size);
379 return;
380 }
381
382 szind_t szind;
383 bool slab;
384 alloc_ctx_t local_ctx;
385 if (config_prof && opt_prof) {
386 if (alloc_ctx == NULL) {
387 /* Uncommon case and should be a static check. */
388 rtree_ctx_t rtree_ctx_fallback;
389 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
390 &rtree_ctx_fallback);
391 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
392 (uintptr_t)ptr, true, &local_ctx.szind,
393 &local_ctx.slab);
394 assert(local_ctx.szind == sz_size2index(size));
395 alloc_ctx = &local_ctx;
396 }
397 slab = alloc_ctx->slab;
398 szind = alloc_ctx->szind;
399 } else {
400 /*
401 * There is no risk of being confused by a promoted sampled
402 * object, so base szind and slab on the given size.
403 */
404 szind = sz_size2index(size);
405 slab = (szind < SC_NBINS);
406 }
407
408 if (config_debug) {
409 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
410 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
411 (uintptr_t)ptr, true, &szind, &slab);
412 extent_t *extent = rtree_extent_read(tsdn,
413 &extents_rtree, rtree_ctx, (uintptr_t)ptr, true);
414 assert(szind == extent_szind_get(extent));
415 assert(slab == extent_slab_get(extent));
416 }
417
418 if (likely(slab)) {
419 /* Small allocation. */
420 tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
421 slow_path);
422 } else {
423 arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path);
424 }
425 }
426
427 #endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */
428