xref: /freebsd/contrib/jemalloc/src/tcache.c (revision 8d20be1e22095c27faf8fe8b2f0d089739cc742e)
1 #define	JEMALLOC_TCACHE_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3 
4 /******************************************************************************/
5 /* Data. */
6 
7 malloc_tsd_data(, tcache, tcache_t *, NULL)
8 malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default)
9 
10 bool	opt_tcache = true;
11 ssize_t	opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
12 
13 tcache_bin_info_t	*tcache_bin_info;
14 static unsigned		stack_nelms; /* Total stack elms per tcache. */
15 
16 size_t			nhbins;
17 size_t			tcache_maxclass;
18 
19 /******************************************************************************/
20 
21 size_t	tcache_salloc(const void *ptr)
22 {
23 
24 	return (arena_salloc(ptr, false));
25 }
26 
27 void
28 tcache_event_hard(tcache_t *tcache)
29 {
30 	size_t binind = tcache->next_gc_bin;
31 	tcache_bin_t *tbin = &tcache->tbins[binind];
32 	tcache_bin_info_t *tbin_info = &tcache_bin_info[binind];
33 
34 	if (tbin->low_water > 0) {
35 		/*
36 		 * Flush (ceiling) 3/4 of the objects below the low water mark.
37 		 */
38 		if (binind < NBINS) {
39 			tcache_bin_flush_small(tbin, binind, tbin->ncached -
40 			    tbin->low_water + (tbin->low_water >> 2), tcache);
41 		} else {
42 			tcache_bin_flush_large(tbin, binind, tbin->ncached -
43 			    tbin->low_water + (tbin->low_water >> 2), tcache);
44 		}
45 		/*
46 		 * Reduce fill count by 2X.  Limit lg_fill_div such that the
47 		 * fill count is always at least 1.
48 		 */
49 		if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) >= 1)
50 			tbin->lg_fill_div++;
51 	} else if (tbin->low_water < 0) {
52 		/*
53 		 * Increase fill count by 2X.  Make sure lg_fill_div stays
54 		 * greater than 0.
55 		 */
56 		if (tbin->lg_fill_div > 1)
57 			tbin->lg_fill_div--;
58 	}
59 	tbin->low_water = tbin->ncached;
60 
61 	tcache->next_gc_bin++;
62 	if (tcache->next_gc_bin == nhbins)
63 		tcache->next_gc_bin = 0;
64 	tcache->ev_cnt = 0;
65 }
66 
67 void *
68 tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
69 {
70 	void *ret;
71 
72 	arena_tcache_fill_small(tcache->arena, tbin, binind,
73 	    config_prof ? tcache->prof_accumbytes : 0);
74 	if (config_prof)
75 		tcache->prof_accumbytes = 0;
76 	ret = tcache_alloc_easy(tbin);
77 
78 	return (ret);
79 }
80 
81 void
82 tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
83     tcache_t *tcache)
84 {
85 	void *ptr;
86 	unsigned i, nflush, ndeferred;
87 	bool merged_stats = false;
88 
89 	assert(binind < NBINS);
90 	assert(rem <= tbin->ncached);
91 
92 	for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
93 		/* Lock the arena bin associated with the first object. */
94 		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
95 		    tbin->avail[0]);
96 		arena_t *arena = chunk->arena;
97 		arena_bin_t *bin = &arena->bins[binind];
98 
99 		if (config_prof && arena == tcache->arena) {
100 			if (arena_prof_accum(arena, tcache->prof_accumbytes))
101 				prof_idump();
102 			tcache->prof_accumbytes = 0;
103 		}
104 
105 		malloc_mutex_lock(&bin->lock);
106 		if (config_stats && arena == tcache->arena) {
107 			assert(merged_stats == false);
108 			merged_stats = true;
109 			bin->stats.nflushes++;
110 			bin->stats.nrequests += tbin->tstats.nrequests;
111 			tbin->tstats.nrequests = 0;
112 		}
113 		ndeferred = 0;
114 		for (i = 0; i < nflush; i++) {
115 			ptr = tbin->avail[i];
116 			assert(ptr != NULL);
117 			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
118 			if (chunk->arena == arena) {
119 				size_t pageind = ((uintptr_t)ptr -
120 				    (uintptr_t)chunk) >> LG_PAGE;
121 				arena_chunk_map_t *mapelm =
122 				    arena_mapp_get(chunk, pageind);
123 				if (config_fill && opt_junk) {
124 					arena_alloc_junk_small(ptr,
125 					    &arena_bin_info[binind], true);
126 				}
127 				arena_dalloc_bin_locked(arena, chunk, ptr,
128 				    mapelm);
129 			} else {
130 				/*
131 				 * This object was allocated via a different
132 				 * arena bin than the one that is currently
133 				 * locked.  Stash the object, so that it can be
134 				 * handled in a future pass.
135 				 */
136 				tbin->avail[ndeferred] = ptr;
137 				ndeferred++;
138 			}
139 		}
140 		malloc_mutex_unlock(&bin->lock);
141 	}
142 	if (config_stats && merged_stats == false) {
143 		/*
144 		 * The flush loop didn't happen to flush to this thread's
145 		 * arena, so the stats didn't get merged.  Manually do so now.
146 		 */
147 		arena_bin_t *bin = &tcache->arena->bins[binind];
148 		malloc_mutex_lock(&bin->lock);
149 		bin->stats.nflushes++;
150 		bin->stats.nrequests += tbin->tstats.nrequests;
151 		tbin->tstats.nrequests = 0;
152 		malloc_mutex_unlock(&bin->lock);
153 	}
154 
155 	memmove(tbin->avail, &tbin->avail[tbin->ncached - rem],
156 	    rem * sizeof(void *));
157 	tbin->ncached = rem;
158 	if ((int)tbin->ncached < tbin->low_water)
159 		tbin->low_water = tbin->ncached;
160 }
161 
162 void
163 tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
164     tcache_t *tcache)
165 {
166 	void *ptr;
167 	unsigned i, nflush, ndeferred;
168 	bool merged_stats = false;
169 
170 	assert(binind < nhbins);
171 	assert(rem <= tbin->ncached);
172 
173 	for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
174 		/* Lock the arena associated with the first object. */
175 		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
176 		    tbin->avail[0]);
177 		arena_t *arena = chunk->arena;
178 		UNUSED bool idump;
179 
180 		if (config_prof)
181 			idump = false;
182 		malloc_mutex_lock(&arena->lock);
183 		if ((config_prof || config_stats) && arena == tcache->arena) {
184 			if (config_prof) {
185 				idump = arena_prof_accum_locked(arena,
186 				    tcache->prof_accumbytes);
187 				tcache->prof_accumbytes = 0;
188 			}
189 			if (config_stats) {
190 				merged_stats = true;
191 				arena->stats.nrequests_large +=
192 				    tbin->tstats.nrequests;
193 				arena->stats.lstats[binind - NBINS].nrequests +=
194 				    tbin->tstats.nrequests;
195 				tbin->tstats.nrequests = 0;
196 			}
197 		}
198 		ndeferred = 0;
199 		for (i = 0; i < nflush; i++) {
200 			ptr = tbin->avail[i];
201 			assert(ptr != NULL);
202 			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
203 			if (chunk->arena == arena)
204 				arena_dalloc_large_locked(arena, chunk, ptr);
205 			else {
206 				/*
207 				 * This object was allocated via a different
208 				 * arena than the one that is currently locked.
209 				 * Stash the object, so that it can be handled
210 				 * in a future pass.
211 				 */
212 				tbin->avail[ndeferred] = ptr;
213 				ndeferred++;
214 			}
215 		}
216 		malloc_mutex_unlock(&arena->lock);
217 		if (config_prof && idump)
218 			prof_idump();
219 	}
220 	if (config_stats && merged_stats == false) {
221 		/*
222 		 * The flush loop didn't happen to flush to this thread's
223 		 * arena, so the stats didn't get merged.  Manually do so now.
224 		 */
225 		arena_t *arena = tcache->arena;
226 		malloc_mutex_lock(&arena->lock);
227 		arena->stats.nrequests_large += tbin->tstats.nrequests;
228 		arena->stats.lstats[binind - NBINS].nrequests +=
229 		    tbin->tstats.nrequests;
230 		tbin->tstats.nrequests = 0;
231 		malloc_mutex_unlock(&arena->lock);
232 	}
233 
234 	memmove(tbin->avail, &tbin->avail[tbin->ncached - rem],
235 	    rem * sizeof(void *));
236 	tbin->ncached = rem;
237 	if ((int)tbin->ncached < tbin->low_water)
238 		tbin->low_water = tbin->ncached;
239 }
240 
241 void
242 tcache_arena_associate(tcache_t *tcache, arena_t *arena)
243 {
244 
245 	if (config_stats) {
246 		/* Link into list of extant tcaches. */
247 		malloc_mutex_lock(&arena->lock);
248 		ql_elm_new(tcache, link);
249 		ql_tail_insert(&arena->tcache_ql, tcache, link);
250 		malloc_mutex_unlock(&arena->lock);
251 	}
252 	tcache->arena = arena;
253 }
254 
255 void
256 tcache_arena_dissociate(tcache_t *tcache)
257 {
258 
259 	if (config_stats) {
260 		/* Unlink from list of extant tcaches. */
261 		malloc_mutex_lock(&tcache->arena->lock);
262 		ql_remove(&tcache->arena->tcache_ql, tcache, link);
263 		malloc_mutex_unlock(&tcache->arena->lock);
264 		tcache_stats_merge(tcache, tcache->arena);
265 	}
266 }
267 
268 tcache_t *
269 tcache_create(arena_t *arena)
270 {
271 	tcache_t *tcache;
272 	size_t size, stack_offset;
273 	unsigned i;
274 
275 	size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins);
276 	/* Naturally align the pointer stacks. */
277 	size = PTR_CEILING(size);
278 	stack_offset = size;
279 	size += stack_nelms * sizeof(void *);
280 	/*
281 	 * Round up to the nearest multiple of the cacheline size, in order to
282 	 * avoid the possibility of false cacheline sharing.
283 	 *
284 	 * That this works relies on the same logic as in ipalloc(), but we
285 	 * cannot directly call ipalloc() here due to tcache bootstrapping
286 	 * issues.
287 	 */
288 	size = (size + CACHELINE_MASK) & (-CACHELINE);
289 
290 	if (size <= SMALL_MAXCLASS)
291 		tcache = (tcache_t *)arena_malloc_small(arena, size, true);
292 	else if (size <= tcache_maxclass)
293 		tcache = (tcache_t *)arena_malloc_large(arena, size, true);
294 	else
295 		tcache = (tcache_t *)icallocx(size, false, arena);
296 
297 	if (tcache == NULL)
298 		return (NULL);
299 
300 	tcache_arena_associate(tcache, arena);
301 
302 	assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
303 	for (i = 0; i < nhbins; i++) {
304 		tcache->tbins[i].lg_fill_div = 1;
305 		tcache->tbins[i].avail = (void **)((uintptr_t)tcache +
306 		    (uintptr_t)stack_offset);
307 		stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
308 	}
309 
310 	tcache_tsd_set(&tcache);
311 
312 	return (tcache);
313 }
314 
315 void
316 tcache_destroy(tcache_t *tcache)
317 {
318 	unsigned i;
319 	size_t tcache_size;
320 
321 	tcache_arena_dissociate(tcache);
322 
323 	for (i = 0; i < NBINS; i++) {
324 		tcache_bin_t *tbin = &tcache->tbins[i];
325 		tcache_bin_flush_small(tbin, i, 0, tcache);
326 
327 		if (config_stats && tbin->tstats.nrequests != 0) {
328 			arena_t *arena = tcache->arena;
329 			arena_bin_t *bin = &arena->bins[i];
330 			malloc_mutex_lock(&bin->lock);
331 			bin->stats.nrequests += tbin->tstats.nrequests;
332 			malloc_mutex_unlock(&bin->lock);
333 		}
334 	}
335 
336 	for (; i < nhbins; i++) {
337 		tcache_bin_t *tbin = &tcache->tbins[i];
338 		tcache_bin_flush_large(tbin, i, 0, tcache);
339 
340 		if (config_stats && tbin->tstats.nrequests != 0) {
341 			arena_t *arena = tcache->arena;
342 			malloc_mutex_lock(&arena->lock);
343 			arena->stats.nrequests_large += tbin->tstats.nrequests;
344 			arena->stats.lstats[i - NBINS].nrequests +=
345 			    tbin->tstats.nrequests;
346 			malloc_mutex_unlock(&arena->lock);
347 		}
348 	}
349 
350 	if (config_prof && tcache->prof_accumbytes > 0 &&
351 	    arena_prof_accum(tcache->arena, tcache->prof_accumbytes))
352 		prof_idump();
353 
354 	tcache_size = arena_salloc(tcache, false);
355 	if (tcache_size <= SMALL_MAXCLASS) {
356 		arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
357 		arena_t *arena = chunk->arena;
358 		size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >>
359 		    LG_PAGE;
360 		arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind);
361 
362 		arena_dalloc_bin(arena, chunk, tcache, pageind, mapelm);
363 	} else if (tcache_size <= tcache_maxclass) {
364 		arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
365 		arena_t *arena = chunk->arena;
366 
367 		arena_dalloc_large(arena, chunk, tcache);
368 	} else
369 		idallocx(tcache, false);
370 }
371 
372 void
373 tcache_thread_cleanup(void *arg)
374 {
375 	tcache_t *tcache = *(tcache_t **)arg;
376 
377 	if (tcache == TCACHE_STATE_DISABLED) {
378 		/* Do nothing. */
379 	} else if (tcache == TCACHE_STATE_REINCARNATED) {
380 		/*
381 		 * Another destructor called an allocator function after this
382 		 * destructor was called.  Reset tcache to
383 		 * TCACHE_STATE_PURGATORY in order to receive another callback.
384 		 */
385 		tcache = TCACHE_STATE_PURGATORY;
386 		tcache_tsd_set(&tcache);
387 	} else if (tcache == TCACHE_STATE_PURGATORY) {
388 		/*
389 		 * The previous time this destructor was called, we set the key
390 		 * to TCACHE_STATE_PURGATORY so that other destructors wouldn't
391 		 * cause re-creation of the tcache.  This time, do nothing, so
392 		 * that the destructor will not be called again.
393 		 */
394 	} else if (tcache != NULL) {
395 		assert(tcache != TCACHE_STATE_PURGATORY);
396 		tcache_destroy(tcache);
397 		tcache = TCACHE_STATE_PURGATORY;
398 		tcache_tsd_set(&tcache);
399 	}
400 }
401 
402 void
403 tcache_stats_merge(tcache_t *tcache, arena_t *arena)
404 {
405 	unsigned i;
406 
407 	/* Merge and reset tcache stats. */
408 	for (i = 0; i < NBINS; i++) {
409 		arena_bin_t *bin = &arena->bins[i];
410 		tcache_bin_t *tbin = &tcache->tbins[i];
411 		malloc_mutex_lock(&bin->lock);
412 		bin->stats.nrequests += tbin->tstats.nrequests;
413 		malloc_mutex_unlock(&bin->lock);
414 		tbin->tstats.nrequests = 0;
415 	}
416 
417 	for (; i < nhbins; i++) {
418 		malloc_large_stats_t *lstats = &arena->stats.lstats[i - NBINS];
419 		tcache_bin_t *tbin = &tcache->tbins[i];
420 		arena->stats.nrequests_large += tbin->tstats.nrequests;
421 		lstats->nrequests += tbin->tstats.nrequests;
422 		tbin->tstats.nrequests = 0;
423 	}
424 }
425 
426 bool
427 tcache_boot0(void)
428 {
429 	unsigned i;
430 
431 	/*
432 	 * If necessary, clamp opt_lg_tcache_max, now that arena_maxclass is
433 	 * known.
434 	 */
435 	if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) < SMALL_MAXCLASS)
436 		tcache_maxclass = SMALL_MAXCLASS;
437 	else if ((1U << opt_lg_tcache_max) > arena_maxclass)
438 		tcache_maxclass = arena_maxclass;
439 	else
440 		tcache_maxclass = (1U << opt_lg_tcache_max);
441 
442 	nhbins = NBINS + (tcache_maxclass >> LG_PAGE);
443 
444 	/* Initialize tcache_bin_info. */
445 	tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins *
446 	    sizeof(tcache_bin_info_t));
447 	if (tcache_bin_info == NULL)
448 		return (true);
449 	stack_nelms = 0;
450 	for (i = 0; i < NBINS; i++) {
451 		if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) {
452 			tcache_bin_info[i].ncached_max =
453 			    (arena_bin_info[i].nregs << 1);
454 		} else {
455 			tcache_bin_info[i].ncached_max =
456 			    TCACHE_NSLOTS_SMALL_MAX;
457 		}
458 		stack_nelms += tcache_bin_info[i].ncached_max;
459 	}
460 	for (; i < nhbins; i++) {
461 		tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
462 		stack_nelms += tcache_bin_info[i].ncached_max;
463 	}
464 
465 	return (false);
466 }
467 
468 bool
469 tcache_boot1(void)
470 {
471 
472 	if (tcache_tsd_boot() || tcache_enabled_tsd_boot())
473 		return (true);
474 
475 	return (false);
476 }
477