xref: /titanic_41/usr/src/lib/libumem/common/umem_impl.h (revision 3e3d658f8af590806331539f141881e82c204c61)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Copyright (c) 2012 Joyent, Inc.  All rights reserved.
29  */
30 
31 #ifndef _UMEM_IMPL_H
32 #define	_UMEM_IMPL_H
33 
34 #include <umem.h>
35 
36 #include <sys/sysmacros.h>
37 #include <sys/time.h>
38 #include <sys/vmem.h>
39 #include <thread.h>
40 
41 #ifdef	__cplusplus
42 extern "C" {
43 #endif
44 
45 /*
46  * umem memory allocator: implementation-private data structures
47  */
48 
49 /*
50  * Internal flags for umem_cache_create
51  */
52 #define	UMC_QCACHE	0x00100000
53 #define	UMC_INTERNAL	0x80000000
54 
55 /*
56  * Cache flags
57  */
58 #define	UMF_AUDIT	0x00000001	/* transaction auditing */
59 #define	UMF_DEADBEEF	0x00000002	/* deadbeef checking */
60 #define	UMF_REDZONE	0x00000004	/* redzone checking */
61 #define	UMF_CONTENTS	0x00000008	/* freed-buffer content logging */
62 #define	UMF_CHECKSIGNAL	0x00000010	/* abort when in signal context */
63 #define	UMF_NOMAGAZINE	0x00000020	/* disable per-cpu magazines */
64 #define	UMF_FIREWALL	0x00000040	/* put all bufs before unmapped pages */
65 #define	UMF_LITE	0x00000100	/* lightweight debugging */
66 
67 #define	UMF_HASH	0x00000200	/* cache has hash table */
68 #define	UMF_RANDOMIZE	0x00000400	/* randomize other umem_flags */
69 #define	UMF_PTC		0x00000800	/* cache has per-thread caching */
70 
71 #define	UMF_BUFTAG	(UMF_DEADBEEF | UMF_REDZONE)
72 #define	UMF_TOUCH	(UMF_BUFTAG | UMF_LITE | UMF_CONTENTS)
73 #define	UMF_RANDOM	(UMF_TOUCH | UMF_AUDIT | UMF_NOMAGAZINE)
74 #define	UMF_DEBUG	(UMF_RANDOM | UMF_FIREWALL)
75 
76 #define	UMEM_STACK_DEPTH	umem_stack_depth
77 
78 #define	UMEM_FREE_PATTERN		0xdeadbeefdeadbeefULL
79 #define	UMEM_UNINITIALIZED_PATTERN	0xbaddcafebaddcafeULL
80 #define	UMEM_REDZONE_PATTERN		0xfeedfacefeedfaceULL
81 #define	UMEM_REDZONE_BYTE		0xbb
82 
83 #define	UMEM_FATAL_FLAGS	(UMEM_NOFAIL)
84 #define	UMEM_SLEEP_FLAGS	(0)
85 
86 /*
87  * Redzone size encodings for umem_alloc() / umem_free().  We encode the
88  * allocation size, rather than storing it directly, so that umem_free()
89  * can distinguish frees of the wrong size from redzone violations.
90  */
91 #define	UMEM_SIZE_ENCODE(x)	(251 * (x) + 1)
92 #define	UMEM_SIZE_DECODE(x)	((x) / 251)
93 #define	UMEM_SIZE_VALID(x)	((x) % 251 == 1)
94 
95 /*
96  * The bufctl (buffer control) structure keeps some minimal information
97  * about each buffer: its address, its slab, and its current linkage,
98  * which is either on the slab's freelist (if the buffer is free), or
99  * on the cache's buf-to-bufctl hash table (if the buffer is allocated).
100  * In the case of non-hashed, or "raw", caches (the common case), only
101  * the freelist linkage is necessary: the buffer address is at a fixed
102  * offset from the bufctl address, and the slab is at the end of the page.
103  *
104  * NOTE: bc_next must be the first field; raw buffers have linkage only.
105  */
106 typedef struct umem_bufctl {
107 	struct umem_bufctl	*bc_next;	/* next bufctl struct */
108 	void			*bc_addr;	/* address of buffer */
109 	struct umem_slab	*bc_slab;	/* controlling slab */
110 } umem_bufctl_t;
111 
112 /*
113  * The UMF_AUDIT version of the bufctl structure.  The beginning of this
114  * structure must be identical to the normal bufctl structure so that
115  * pointers are interchangeable.
116  */
117 
118 #define	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \
119 	((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames]))
120 
121 /*
122  * umem_bufctl_audits must be allocated from a UMC_NOHASH cache, so we
123  * require that 2 of them, plus 2 buftags, plus a umem_slab_t, all fit on
124  * a single page.
125  *
126  * For ILP32, this is about 1000 frames.
127  * For LP64, this is about 490 frames.
128  */
129 
130 #define	UMEM_BUFCTL_AUDIT_ALIGN	32
131 
132 #define	UMEM_BUFCTL_AUDIT_MAX_SIZE					\
133 	(P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 -			\
134 	    sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN))
135 
136 #define	UMEM_MAX_STACK_DEPTH						\
137 	((UMEM_BUFCTL_AUDIT_MAX_SIZE -					\
138 	    UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t))
139 
140 typedef struct umem_bufctl_audit {
141 	struct umem_bufctl	*bc_next;	/* next bufctl struct */
142 	void			*bc_addr;	/* address of buffer */
143 	struct umem_slab	*bc_slab;	/* controlling slab */
144 	umem_cache_t		*bc_cache;	/* controlling cache */
145 	hrtime_t		bc_timestamp;	/* transaction time */
146 	thread_t		bc_thread;	/* thread doing transaction */
147 	struct umem_bufctl	*bc_lastlog;	/* last log entry */
148 	void			*bc_contents;	/* contents at last free */
149 	int			bc_depth;	/* stack depth */
150 	uintptr_t		bc_stack[1];	/* pc stack */
151 } umem_bufctl_audit_t;
152 
153 #define	UMEM_LOCAL_BUFCTL_AUDIT(bcpp)					\
154 		*(bcpp) = (umem_bufctl_audit_t *)			\
155 		    alloca(UMEM_BUFCTL_AUDIT_SIZE)
156 
157 #define	UMEM_BUFCTL_AUDIT_SIZE						\
158 	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH)
159 
160 /*
161  * A umem_buftag structure is appended to each buffer whenever any of the
162  * UMF_BUFTAG flags (UMF_DEADBEEF, UMF_REDZONE, UMF_VERIFY) are set.
163  */
164 typedef struct umem_buftag {
165 	uint64_t		bt_redzone;	/* 64-bit redzone pattern */
166 	umem_bufctl_t		*bt_bufctl;	/* bufctl */
167 	intptr_t		bt_bxstat;	/* bufctl ^ (alloc/free) */
168 } umem_buftag_t;
169 
170 #define	UMEM_BUFTAG(cp, buf)		\
171 	((umem_buftag_t *)((char *)(buf) + (cp)->cache_buftag))
172 
173 #define	UMEM_BUFCTL(cp, buf)		\
174 	((umem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl))
175 
176 #define	UMEM_BUF(cp, bcp)		\
177 	((void *)((char *)(bcp) - (cp)->cache_bufctl))
178 
179 #define	UMEM_SLAB(cp, buf)		\
180 	((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1)
181 
182 #define	UMEM_CPU_CACHE(cp, cpu)		\
183 	(umem_cpu_cache_t *)((char *)cp + cpu->cpu_cache_offset)
184 
185 #define	UMEM_MAGAZINE_VALID(cp, mp)	\
186 	(((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \
187 	    (cp)->cache_magtype->mt_cache)
188 
189 #define	UMEM_SLAB_MEMBER(sp, buf)	\
190 	((size_t)(buf) - (size_t)(sp)->slab_base < \
191 	    (sp)->slab_cache->cache_slabsize)
192 
193 #define	UMEM_BUFTAG_ALLOC	0xa110c8edUL
194 #define	UMEM_BUFTAG_FREE	0xf4eef4eeUL
195 
196 typedef struct umem_slab {
197 	struct umem_cache	*slab_cache;	/* controlling cache */
198 	void			*slab_base;	/* base of allocated memory */
199 	struct umem_slab	*slab_next;	/* next slab on freelist */
200 	struct umem_slab	*slab_prev;	/* prev slab on freelist */
201 	struct umem_bufctl	*slab_head;	/* first free buffer */
202 	long			slab_refcnt;	/* outstanding allocations */
203 	long			slab_chunks;	/* chunks (bufs) in this slab */
204 } umem_slab_t;
205 
206 #define	UMEM_HASH_INITIAL	64
207 
208 #define	UMEM_HASH(cp, buf)	\
209 	((cp)->cache_hash_table +	\
210 	(((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask))
211 
212 typedef struct umem_magazine {
213 	void	*mag_next;
214 	void	*mag_round[1];		/* one or more rounds */
215 } umem_magazine_t;
216 
217 /*
218  * The magazine types for fast per-cpu allocation
219  */
220 typedef struct umem_magtype {
221 	int		mt_magsize;	/* magazine size (number of rounds) */
222 	int		mt_align;	/* magazine alignment */
223 	size_t		mt_minbuf;	/* all smaller buffers qualify */
224 	size_t		mt_maxbuf;	/* no larger buffers qualify */
225 	umem_cache_t	*mt_cache;	/* magazine cache */
226 } umem_magtype_t;
227 
228 #define	UMEM_CPU_CACHE_SIZE	64	/* must be power of 2 */
229 #define	UMEM_CPU_PAD		(UMEM_CPU_CACHE_SIZE - sizeof (mutex_t) - \
230 	2 * sizeof (uint_t) - 2 * sizeof (void *) - 4 * sizeof (int))
231 #define	UMEM_CACHE_SIZE(ncpus)	\
232 	((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus]))
233 
234 typedef struct umem_cpu_cache {
235 	mutex_t		cc_lock;	/* protects this cpu's local cache */
236 	uint_t		cc_alloc;	/* allocations from this cpu */
237 	uint_t		cc_free;	/* frees to this cpu */
238 	umem_magazine_t	*cc_loaded;	/* the currently loaded magazine */
239 	umem_magazine_t	*cc_ploaded;	/* the previously loaded magazine */
240 	int		cc_rounds;	/* number of objects in loaded mag */
241 	int		cc_prounds;	/* number of objects in previous mag */
242 	int		cc_magsize;	/* number of rounds in a full mag */
243 	int		cc_flags;	/* CPU-local copy of cache_flags */
244 #ifndef _LP64
245 	char		cc_pad[UMEM_CPU_PAD]; /* for nice alignment (32-bit) */
246 #endif
247 } umem_cpu_cache_t;
248 
249 /*
250  * The magazine lists used in the depot.
251  */
252 typedef struct umem_maglist {
253 	umem_magazine_t	*ml_list;	/* magazine list */
254 	long		ml_total;	/* number of magazines */
255 	long		ml_min;		/* min since last update */
256 	long		ml_reaplimit;	/* max reapable magazines */
257 	uint64_t	ml_alloc;	/* allocations from this list */
258 } umem_maglist_t;
259 
260 #define	UMEM_CACHE_NAMELEN	31
261 
262 struct umem_cache {
263 	/*
264 	 * Statistics
265 	 */
266 	uint64_t	cache_slab_create;	/* slab creates */
267 	uint64_t	cache_slab_destroy;	/* slab destroys */
268 	uint64_t	cache_slab_alloc;	/* slab layer allocations */
269 	uint64_t	cache_slab_free;	/* slab layer frees */
270 	uint64_t	cache_alloc_fail;	/* total failed allocations */
271 	uint64_t	cache_buftotal;		/* total buffers */
272 	uint64_t	cache_bufmax;		/* max buffers ever */
273 	uint64_t	cache_rescale;		/* # of hash table rescales */
274 	uint64_t	cache_lookup_depth;	/* hash lookup depth */
275 	uint64_t	cache_depot_contention;	/* mutex contention count */
276 	uint64_t	cache_depot_contention_prev; /* previous snapshot */
277 
278 	/*
279 	 * Cache properties
280 	 */
281 	char		cache_name[UMEM_CACHE_NAMELEN + 1];
282 	size_t		cache_bufsize;		/* object size */
283 	size_t		cache_align;		/* object alignment */
284 	umem_constructor_t *cache_constructor;
285 	umem_destructor_t *cache_destructor;
286 	umem_reclaim_t	*cache_reclaim;
287 	void		*cache_private;		/* opaque arg to callbacks */
288 	vmem_t		*cache_arena;		/* vmem source for slabs */
289 	int		cache_cflags;		/* cache creation flags */
290 	int		cache_flags;		/* various cache state info */
291 	int		cache_uflags;		/* UMU_* flags */
292 	uint32_t	cache_mtbf;		/* induced alloc failure rate */
293 	umem_cache_t	*cache_next;		/* forward cache linkage */
294 	umem_cache_t	*cache_prev;		/* backward cache linkage */
295 	umem_cache_t	*cache_unext;		/* next in update list */
296 	umem_cache_t	*cache_uprev;		/* prev in update list */
297 	uint32_t	cache_cpu_mask;		/* mask for cpu offset */
298 
299 	/*
300 	 * Slab layer
301 	 */
302 	mutex_t		cache_lock;		/* protects slab layer */
303 	size_t		cache_chunksize;	/* buf + alignment [+ debug] */
304 	size_t		cache_slabsize;		/* size of a slab */
305 	size_t		cache_bufctl;		/* buf-to-bufctl distance */
306 	size_t		cache_buftag;		/* buf-to-buftag distance */
307 	size_t		cache_verify;		/* bytes to verify */
308 	size_t		cache_contents;		/* bytes of saved content */
309 	size_t		cache_color;		/* next slab color */
310 	size_t		cache_mincolor;		/* maximum slab color */
311 	size_t		cache_maxcolor;		/* maximum slab color */
312 	size_t		cache_hash_shift;	/* get to interesting bits */
313 	size_t		cache_hash_mask;	/* hash table mask */
314 	umem_slab_t	*cache_freelist;	/* slab free list */
315 	umem_slab_t	cache_nullslab;		/* end of freelist marker */
316 	umem_cache_t	*cache_bufctl_cache;	/* source of bufctls */
317 	umem_bufctl_t	**cache_hash_table;	/* hash table base */
318 	/*
319 	 * Depot layer
320 	 */
321 	mutex_t		cache_depot_lock;	/* protects depot */
322 	umem_magtype_t	*cache_magtype;		/* magazine type */
323 	umem_maglist_t	cache_full;		/* full magazines */
324 	umem_maglist_t	cache_empty;		/* empty magazines */
325 
326 	/*
327 	 * Per-CPU layer
328 	 */
329 	umem_cpu_cache_t cache_cpu[1];		/* cache_cpu_mask + 1 entries */
330 };
331 
332 typedef struct umem_cpu_log_header {
333 	mutex_t		clh_lock;
334 	char		*clh_current;
335 	size_t		clh_avail;
336 	int		clh_chunk;
337 	int		clh_hits;
338 	char		clh_pad[64 - sizeof (mutex_t) - sizeof (char *) -
339 				sizeof (size_t) - 2 * sizeof (int)];
340 } umem_cpu_log_header_t;
341 
342 typedef struct umem_log_header {
343 	mutex_t		lh_lock;
344 	char		*lh_base;
345 	int		*lh_free;
346 	size_t		lh_chunksize;
347 	int		lh_nchunks;
348 	int		lh_head;
349 	int		lh_tail;
350 	int		lh_hits;
351 	umem_cpu_log_header_t lh_cpu[1];	/* actually umem_max_ncpus */
352 } umem_log_header_t;
353 
354 typedef struct umem_cpu {
355 	uint32_t cpu_cache_offset;
356 	uint32_t cpu_number;
357 } umem_cpu_t;
358 
359 #define	UMEM_MAXBUF	131072
360 
361 #define	UMEM_ALIGN		8	/* min guaranteed alignment */
362 #define	UMEM_ALIGN_SHIFT	3	/* log2(UMEM_ALIGN) */
363 #define	UMEM_VOID_FRACTION	8	/* never waste more than 1/8 of slab */
364 
365 /*
366  * For 64 bits, buffers >= 16 bytes must be 16-byte aligned
367  */
368 #ifdef _LP64
369 #define	UMEM_SECOND_ALIGN 16
370 #else
371 #define	UMEM_SECOND_ALIGN UMEM_ALIGN
372 #endif
373 
374 #define	MALLOC_MAGIC			0x3a10c000 /* 8-byte tag */
375 #define	MEMALIGN_MAGIC			0x3e3a1000
376 
377 #ifdef _LP64
378 #define	MALLOC_SECOND_MAGIC		0x16ba7000 /* 8-byte tag, 16-aligned */
379 #define	MALLOC_OVERSIZE_MAGIC		0x06e47000 /* 16-byte tag, _LP64 */
380 #endif
381 
382 #define	UMEM_MALLOC_ENCODE(type, sz)	(uint32_t)((type) - (sz))
383 #define	UMEM_MALLOC_DECODE(stat, sz)	(uint32_t)((stat) + (sz))
384 #define	UMEM_FREE_PATTERN_32		(uint32_t)(UMEM_FREE_PATTERN)
385 
386 #define	UMU_MAGAZINE_RESIZE	0x00000001
387 #define	UMU_HASH_RESCALE	0x00000002
388 #define	UMU_REAP		0x00000004
389 #define	UMU_NOTIFY		0x08000000
390 #define	UMU_ACTIVE		0x80000000
391 
392 #define	UMEM_READY_INIT_FAILED		-1
393 #define	UMEM_READY_STARTUP		1
394 #define	UMEM_READY_INITING		2
395 #define	UMEM_READY			3
396 
397 #ifdef UMEM_STANDALONE
398 extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
399 extern int umem_add(caddr_t, size_t);
400 #endif
401 
402 /*
403  * Private interface with libc for tcumem.
404  */
405 extern uintptr_t _tmem_get_base(void);
406 extern int _tmem_get_nentries(void);
407 extern void _tmem_set_cleanup(void(*)(void *, int));
408 
409 #ifdef	__cplusplus
410 }
411 #endif
412 
413 #endif	/* _UMEM_IMPL_H */
414