xref: /illumos-gate/usr/src/lib/libumem/common/umem_impl.h (revision 831abf2c3ce98eddc86402eb1c97c92fa48c7349)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Copyright (c) 2012 Joyent, Inc.  All rights reserved.
29  * Copyright (c) 2015 by Delphix. All rights reserved.
30  */
31 
32 #ifndef _UMEM_IMPL_H
33 #define	_UMEM_IMPL_H
34 
35 #include <umem.h>
36 
37 #include <sys/sysmacros.h>
38 #include <sys/time.h>
39 #include <sys/vmem.h>
40 #include <thread.h>
41 
42 #ifdef	__cplusplus
43 extern "C" {
44 #endif
45 
46 /*
47  * umem memory allocator: implementation-private data structures
48  */
49 
50 /*
51  * Internal flags for umem_cache_create
52  */
53 #define	UMC_QCACHE	0x00100000
54 #define	UMC_INTERNAL	0x80000000
55 
56 /*
57  * Cache flags
58  */
59 #define	UMF_AUDIT	0x00000001	/* transaction auditing */
60 #define	UMF_DEADBEEF	0x00000002	/* deadbeef checking */
61 #define	UMF_REDZONE	0x00000004	/* redzone checking */
62 #define	UMF_CONTENTS	0x00000008	/* freed-buffer content logging */
63 #define	UMF_CHECKSIGNAL	0x00000010	/* abort when in signal context */
64 #define	UMF_NOMAGAZINE	0x00000020	/* disable per-cpu magazines */
65 #define	UMF_FIREWALL	0x00000040	/* put all bufs before unmapped pages */
66 #define	UMF_LITE	0x00000100	/* lightweight debugging */
67 
68 #define	UMF_HASH	0x00000200	/* cache has hash table */
69 #define	UMF_RANDOMIZE	0x00000400	/* randomize other umem_flags */
70 #define	UMF_PTC		0x00000800	/* cache has per-thread caching */
71 
72 #define	UMF_CHECKNULL	0x00001000	/* heap exhaustion checking */
73 
74 #define	UMF_BUFTAG	(UMF_DEADBEEF | UMF_REDZONE)
75 #define	UMF_TOUCH	(UMF_BUFTAG | UMF_LITE | UMF_CONTENTS)
76 #define	UMF_RANDOM	(UMF_TOUCH | UMF_AUDIT | UMF_NOMAGAZINE)
77 #define	UMF_DEBUG	(UMF_RANDOM | UMF_FIREWALL)
78 
79 #define	UMEM_STACK_DEPTH	umem_stack_depth
80 
81 #define	UMEM_FREE_PATTERN		0xdeadbeefdeadbeefULL
82 #define	UMEM_UNINITIALIZED_PATTERN	0xbaddcafebaddcafeULL
83 #define	UMEM_REDZONE_PATTERN		0xfeedfacefeedfaceULL
84 #define	UMEM_REDZONE_BYTE		0xbb
85 
86 #define	UMEM_FATAL_FLAGS	(UMEM_NOFAIL)
87 #define	UMEM_SLEEP_FLAGS	(0)
88 
89 /*
90  * Redzone size encodings for umem_alloc() / umem_free().  We encode the
91  * allocation size, rather than storing it directly, so that umem_free()
92  * can distinguish frees of the wrong size from redzone violations.
93  */
94 #define	UMEM_SIZE_ENCODE(x)	(251 * (x) + 1)
95 #define	UMEM_SIZE_DECODE(x)	((x) / 251)
96 #define	UMEM_SIZE_VALID(x)	((x) % 251 == 1)
97 
98 /*
99  * The bufctl (buffer control) structure keeps some minimal information
100  * about each buffer: its address, its slab, and its current linkage,
101  * which is either on the slab's freelist (if the buffer is free), or
102  * on the cache's buf-to-bufctl hash table (if the buffer is allocated).
103  * In the case of non-hashed, or "raw", caches (the common case), only
104  * the freelist linkage is necessary: the buffer address is at a fixed
105  * offset from the bufctl address, and the slab is at the end of the page.
106  *
107  * NOTE: bc_next must be the first field; raw buffers have linkage only.
108  */
109 typedef struct umem_bufctl {
110 	struct umem_bufctl	*bc_next;	/* next bufctl struct */
111 	void			*bc_addr;	/* address of buffer */
112 	struct umem_slab	*bc_slab;	/* controlling slab */
113 } umem_bufctl_t;
114 
115 /*
116  * The UMF_AUDIT version of the bufctl structure.  The beginning of this
117  * structure must be identical to the normal bufctl structure so that
118  * pointers are interchangeable.
119  */
120 
121 #define	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \
122 	((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames]))
123 
124 /*
125  * umem_bufctl_audits must be allocated from a UMC_NOHASH cache, so we
126  * require that 2 of them, plus 2 buftags, plus a umem_slab_t, all fit on
127  * a single page.
128  *
129  * For ILP32, this is about 1000 frames.
130  * For LP64, this is about 490 frames.
131  */
132 
133 #define	UMEM_BUFCTL_AUDIT_ALIGN	32
134 
135 #define	UMEM_BUFCTL_AUDIT_MAX_SIZE					\
136 	(P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 -			\
137 	    sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN))
138 
139 #define	UMEM_MAX_STACK_DEPTH						\
140 	((UMEM_BUFCTL_AUDIT_MAX_SIZE -					\
141 	    UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t))
142 
143 typedef struct umem_bufctl_audit {
144 	struct umem_bufctl	*bc_next;	/* next bufctl struct */
145 	void			*bc_addr;	/* address of buffer */
146 	struct umem_slab	*bc_slab;	/* controlling slab */
147 	umem_cache_t		*bc_cache;	/* controlling cache */
148 	hrtime_t		bc_timestamp;	/* transaction time */
149 	thread_t		bc_thread;	/* thread doing transaction */
150 	struct umem_bufctl	*bc_lastlog;	/* last log entry */
151 	void			*bc_contents;	/* contents at last free */
152 	int			bc_depth;	/* stack depth */
153 	uintptr_t		bc_stack[1];	/* pc stack */
154 } umem_bufctl_audit_t;
155 
156 #define	UMEM_LOCAL_BUFCTL_AUDIT(bcpp)					\
157 		*(bcpp) = (umem_bufctl_audit_t *)			\
158 		    alloca(UMEM_BUFCTL_AUDIT_SIZE)
159 
160 #define	UMEM_BUFCTL_AUDIT_SIZE						\
161 	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH)
162 
163 /*
164  * A umem_buftag structure is appended to each buffer whenever any of the
165  * UMF_BUFTAG flags (UMF_DEADBEEF, UMF_REDZONE, UMF_VERIFY) are set.
166  */
167 typedef struct umem_buftag {
168 	uint64_t		bt_redzone;	/* 64-bit redzone pattern */
169 	umem_bufctl_t		*bt_bufctl;	/* bufctl */
170 	intptr_t		bt_bxstat;	/* bufctl ^ (alloc/free) */
171 } umem_buftag_t;
172 
173 #define	UMEM_BUFTAG(cp, buf)		\
174 	((umem_buftag_t *)((char *)(buf) + (cp)->cache_buftag))
175 
176 #define	UMEM_BUFCTL(cp, buf)		\
177 	((umem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl))
178 
179 #define	UMEM_BUF(cp, bcp)		\
180 	((void *)((char *)(bcp) - (cp)->cache_bufctl))
181 
182 #define	UMEM_SLAB(cp, buf)		\
183 	((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1)
184 
185 #define	UMEM_CPU_CACHE(cp, cpu)		\
186 	(umem_cpu_cache_t *)((char *)cp + cpu->cpu_cache_offset)
187 
188 #define	UMEM_MAGAZINE_VALID(cp, mp)	\
189 	(((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \
190 	    (cp)->cache_magtype->mt_cache)
191 
192 #define	UMEM_SLAB_MEMBER(sp, buf)	\
193 	((size_t)(buf) - (size_t)(sp)->slab_base < \
194 	    (sp)->slab_cache->cache_slabsize)
195 
196 #define	UMEM_BUFTAG_ALLOC	0xa110c8edUL
197 #define	UMEM_BUFTAG_FREE	0xf4eef4eeUL
198 
199 typedef struct umem_slab {
200 	struct umem_cache	*slab_cache;	/* controlling cache */
201 	void			*slab_base;	/* base of allocated memory */
202 	struct umem_slab	*slab_next;	/* next slab on freelist */
203 	struct umem_slab	*slab_prev;	/* prev slab on freelist */
204 	struct umem_bufctl	*slab_head;	/* first free buffer */
205 	long			slab_refcnt;	/* outstanding allocations */
206 	long			slab_chunks;	/* chunks (bufs) in this slab */
207 } umem_slab_t;
208 
209 #define	UMEM_HASH_INITIAL	64
210 
211 #define	UMEM_HASH(cp, buf)	\
212 	((cp)->cache_hash_table +	\
213 	(((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask))
214 
215 typedef struct umem_magazine {
216 	void	*mag_next;
217 	void	*mag_round[1];		/* one or more rounds */
218 } umem_magazine_t;
219 
220 /*
221  * The magazine types for fast per-cpu allocation
222  */
223 typedef struct umem_magtype {
224 	int		mt_magsize;	/* magazine size (number of rounds) */
225 	int		mt_align;	/* magazine alignment */
226 	size_t		mt_minbuf;	/* all smaller buffers qualify */
227 	size_t		mt_maxbuf;	/* no larger buffers qualify */
228 	umem_cache_t	*mt_cache;	/* magazine cache */
229 } umem_magtype_t;
230 
231 #define	UMEM_CPU_CACHE_SIZE	64	/* must be power of 2 */
232 #define	UMEM_CPU_PAD		(UMEM_CPU_CACHE_SIZE - sizeof (mutex_t) - \
233 	2 * sizeof (uint_t) - 2 * sizeof (void *) - 4 * sizeof (int))
234 #define	UMEM_CACHE_SIZE(ncpus)	\
235 	((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus]))
236 
237 typedef struct umem_cpu_cache {
238 	mutex_t		cc_lock;	/* protects this cpu's local cache */
239 	uint_t		cc_alloc;	/* allocations from this cpu */
240 	uint_t		cc_free;	/* frees to this cpu */
241 	umem_magazine_t	*cc_loaded;	/* the currently loaded magazine */
242 	umem_magazine_t	*cc_ploaded;	/* the previously loaded magazine */
243 	int		cc_rounds;	/* number of objects in loaded mag */
244 	int		cc_prounds;	/* number of objects in previous mag */
245 	int		cc_magsize;	/* number of rounds in a full mag */
246 	int		cc_flags;	/* CPU-local copy of cache_flags */
247 #ifndef _LP64
248 	char		cc_pad[UMEM_CPU_PAD]; /* for nice alignment (32-bit) */
249 #endif
250 } umem_cpu_cache_t;
251 
252 /*
253  * The magazine lists used in the depot.
254  */
255 typedef struct umem_maglist {
256 	umem_magazine_t	*ml_list;	/* magazine list */
257 	long		ml_total;	/* number of magazines */
258 	long		ml_min;		/* min since last update */
259 	long		ml_reaplimit;	/* max reapable magazines */
260 	uint64_t	ml_alloc;	/* allocations from this list */
261 } umem_maglist_t;
262 
263 #define	UMEM_CACHE_NAMELEN	31
264 
265 struct umem_cache {
266 	/*
267 	 * Statistics
268 	 */
269 	uint64_t	cache_slab_create;	/* slab creates */
270 	uint64_t	cache_slab_destroy;	/* slab destroys */
271 	uint64_t	cache_slab_alloc;	/* slab layer allocations */
272 	uint64_t	cache_slab_free;	/* slab layer frees */
273 	uint64_t	cache_alloc_fail;	/* total failed allocations */
274 	uint64_t	cache_buftotal;		/* total buffers */
275 	uint64_t	cache_bufmax;		/* max buffers ever */
276 	uint64_t	cache_rescale;		/* # of hash table rescales */
277 	uint64_t	cache_lookup_depth;	/* hash lookup depth */
278 	uint64_t	cache_depot_contention;	/* mutex contention count */
279 	uint64_t	cache_depot_contention_prev; /* previous snapshot */
280 
281 	/*
282 	 * Cache properties
283 	 */
284 	char		cache_name[UMEM_CACHE_NAMELEN + 1];
285 	size_t		cache_bufsize;		/* object size */
286 	size_t		cache_align;		/* object alignment */
287 	umem_constructor_t *cache_constructor;
288 	umem_destructor_t *cache_destructor;
289 	umem_reclaim_t	*cache_reclaim;
290 	void		*cache_private;		/* opaque arg to callbacks */
291 	vmem_t		*cache_arena;		/* vmem source for slabs */
292 	int		cache_cflags;		/* cache creation flags */
293 	int		cache_flags;		/* various cache state info */
294 	int		cache_uflags;		/* UMU_* flags */
295 	uint32_t	cache_mtbf;		/* induced alloc failure rate */
296 	umem_cache_t	*cache_next;		/* forward cache linkage */
297 	umem_cache_t	*cache_prev;		/* backward cache linkage */
298 	umem_cache_t	*cache_unext;		/* next in update list */
299 	umem_cache_t	*cache_uprev;		/* prev in update list */
300 	uint32_t	cache_cpu_mask;		/* mask for cpu offset */
301 
302 	/*
303 	 * Slab layer
304 	 */
305 	mutex_t		cache_lock;		/* protects slab layer */
306 	size_t		cache_chunksize;	/* buf + alignment [+ debug] */
307 	size_t		cache_slabsize;		/* size of a slab */
308 	size_t		cache_bufctl;		/* buf-to-bufctl distance */
309 	size_t		cache_buftag;		/* buf-to-buftag distance */
310 	size_t		cache_verify;		/* bytes to verify */
311 	size_t		cache_contents;		/* bytes of saved content */
312 	size_t		cache_color;		/* next slab color */
313 	size_t		cache_mincolor;		/* maximum slab color */
314 	size_t		cache_maxcolor;		/* maximum slab color */
315 	size_t		cache_hash_shift;	/* get to interesting bits */
316 	size_t		cache_hash_mask;	/* hash table mask */
317 	umem_slab_t	*cache_freelist;	/* slab free list */
318 	umem_slab_t	cache_nullslab;		/* end of freelist marker */
319 	umem_cache_t	*cache_bufctl_cache;	/* source of bufctls */
320 	umem_bufctl_t	**cache_hash_table;	/* hash table base */
321 	/*
322 	 * Depot layer
323 	 */
324 	mutex_t		cache_depot_lock;	/* protects depot */
325 	umem_magtype_t	*cache_magtype;		/* magazine type */
326 	umem_maglist_t	cache_full;		/* full magazines */
327 	umem_maglist_t	cache_empty;		/* empty magazines */
328 
329 	/*
330 	 * Per-CPU layer
331 	 */
332 	umem_cpu_cache_t cache_cpu[1];		/* cache_cpu_mask + 1 entries */
333 };
334 
335 typedef struct umem_cpu_log_header {
336 	mutex_t		clh_lock;
337 	char		*clh_current;
338 	size_t		clh_avail;
339 	int		clh_chunk;
340 	int		clh_hits;
341 	char		clh_pad[64 - sizeof (mutex_t) - sizeof (char *) -
342 				sizeof (size_t) - 2 * sizeof (int)];
343 } umem_cpu_log_header_t;
344 
345 typedef struct umem_log_header {
346 	mutex_t		lh_lock;
347 	char		*lh_base;
348 	int		*lh_free;
349 	size_t		lh_chunksize;
350 	int		lh_nchunks;
351 	int		lh_head;
352 	int		lh_tail;
353 	int		lh_hits;
354 	umem_cpu_log_header_t lh_cpu[1];	/* actually umem_max_ncpus */
355 } umem_log_header_t;
356 
357 typedef struct umem_cpu {
358 	uint32_t cpu_cache_offset;
359 	uint32_t cpu_number;
360 } umem_cpu_t;
361 
362 #define	UMEM_MAXBUF	131072
363 
364 #define	UMEM_ALIGN		8	/* min guaranteed alignment */
365 #define	UMEM_ALIGN_SHIFT	3	/* log2(UMEM_ALIGN) */
366 #define	UMEM_VOID_FRACTION	8	/* never waste more than 1/8 of slab */
367 
368 /*
369  * For 64 bits, buffers >= 16 bytes must be 16-byte aligned
370  */
371 #ifdef _LP64
372 #define	UMEM_SECOND_ALIGN 16
373 #else
374 #define	UMEM_SECOND_ALIGN UMEM_ALIGN
375 #endif
376 
377 #define	MALLOC_MAGIC			0x3a10c000 /* 8-byte tag */
378 #define	MEMALIGN_MAGIC			0x3e3a1000
379 
380 #ifdef _LP64
381 #define	MALLOC_SECOND_MAGIC		0x16ba7000 /* 8-byte tag, 16-aligned */
382 #define	MALLOC_OVERSIZE_MAGIC		0x06e47000 /* 16-byte tag, _LP64 */
383 #endif
384 
385 #define	UMEM_MALLOC_ENCODE(type, sz)	(uint32_t)((type) - (sz))
386 #define	UMEM_MALLOC_DECODE(stat, sz)	(uint32_t)((stat) + (sz))
387 #define	UMEM_FREE_PATTERN_32		(uint32_t)(UMEM_FREE_PATTERN)
388 
389 #define	UMU_MAGAZINE_RESIZE	0x00000001
390 #define	UMU_HASH_RESCALE	0x00000002
391 #define	UMU_REAP		0x00000004
392 #define	UMU_NOTIFY		0x08000000
393 #define	UMU_ACTIVE		0x80000000
394 
395 #define	UMEM_READY_INIT_FAILED		-1
396 #define	UMEM_READY_STARTUP		1
397 #define	UMEM_READY_INITING		2
398 #define	UMEM_READY			3
399 
400 #ifdef UMEM_STANDALONE
401 extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
402 extern int umem_add(caddr_t, size_t);
403 #endif
404 
405 /*
406  * Private interface with libc for tcumem.
407  */
408 extern uintptr_t _tmem_get_base(void);
409 extern int _tmem_get_nentries(void);
410 extern void _tmem_set_cleanup(void(*)(void *, int));
411 
412 #ifdef	__cplusplus
413 }
414 #endif
415 
416 #endif	/* _UMEM_IMPL_H */
417