1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _UMEM_IMPL_H 28 #define _UMEM_IMPL_H 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 #include <umem.h> 33 34 #include <sys/sysmacros.h> 35 #include <sys/time.h> 36 #include <sys/vmem.h> 37 #include <thread.h> 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 /* 44 * umem memory allocator: implementation-private data structures 45 */ 46 47 /* 48 * Internal flags for umem_cache_create 49 */ 50 #define UMC_QCACHE 0x00100000 51 #define UMC_INTERNAL 0x80000000 52 53 /* 54 * Cache flags 55 */ 56 #define UMF_AUDIT 0x00000001 /* transaction auditing */ 57 #define UMF_DEADBEEF 0x00000002 /* deadbeef checking */ 58 #define UMF_REDZONE 0x00000004 /* redzone checking */ 59 #define UMF_CONTENTS 0x00000008 /* freed-buffer content logging */ 60 #define UMF_CHECKSIGNAL 0x00000010 /* abort when in signal context */ 61 #define UMF_NOMAGAZINE 0x00000020 /* disable per-cpu magazines */ 62 #define UMF_FIREWALL 0x00000040 /* put all bufs before unmapped pages */ 63 #define UMF_LITE 0x00000100 /* lightweight debugging */ 64 65 #define UMF_HASH 0x00000200 /* cache has hash table */ 66 #define UMF_RANDOMIZE 0x00000400 /* randomize other umem_flags */ 67 68 #define UMF_BUFTAG (UMF_DEADBEEF | UMF_REDZONE) 69 #define UMF_TOUCH (UMF_BUFTAG | UMF_LITE | UMF_CONTENTS) 70 #define UMF_RANDOM (UMF_TOUCH | UMF_AUDIT | UMF_NOMAGAZINE) 71 #define UMF_DEBUG (UMF_RANDOM | UMF_FIREWALL) 72 73 #define UMEM_STACK_DEPTH umem_stack_depth 74 75 #define UMEM_FREE_PATTERN 0xdeadbeefdeadbeefULL 76 #define UMEM_UNINITIALIZED_PATTERN 0xbaddcafebaddcafeULL 77 #define UMEM_REDZONE_PATTERN 0xfeedfacefeedfaceULL 78 #define UMEM_REDZONE_BYTE 0xbb 79 80 #define UMEM_FATAL_FLAGS (UMEM_NOFAIL) 81 #define UMEM_SLEEP_FLAGS (0) 82 83 /* 84 * Redzone size encodings for umem_alloc() / umem_free(). We encode the 85 * allocation size, rather than storing it directly, so that umem_free() 86 * can distinguish frees of the wrong size from redzone violations. 87 */ 88 #define UMEM_SIZE_ENCODE(x) (251 * (x) + 1) 89 #define UMEM_SIZE_DECODE(x) ((x) / 251) 90 #define UMEM_SIZE_VALID(x) ((x) % 251 == 1) 91 92 /* 93 * The bufctl (buffer control) structure keeps some minimal information 94 * about each buffer: its address, its slab, and its current linkage, 95 * which is either on the slab's freelist (if the buffer is free), or 96 * on the cache's buf-to-bufctl hash table (if the buffer is allocated). 97 * In the case of non-hashed, or "raw", caches (the common case), only 98 * the freelist linkage is necessary: the buffer address is at a fixed 99 * offset from the bufctl address, and the slab is at the end of the page. 100 * 101 * NOTE: bc_next must be the first field; raw buffers have linkage only. 102 */ 103 typedef struct umem_bufctl { 104 struct umem_bufctl *bc_next; /* next bufctl struct */ 105 void *bc_addr; /* address of buffer */ 106 struct umem_slab *bc_slab; /* controlling slab */ 107 } umem_bufctl_t; 108 109 /* 110 * The UMF_AUDIT version of the bufctl structure. The beginning of this 111 * structure must be identical to the normal bufctl structure so that 112 * pointers are interchangeable. 113 */ 114 115 #define UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \ 116 ((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames])) 117 118 /* 119 * umem_bufctl_audits must be allocated from a UMC_NOHASH cache, so we 120 * require that 2 of them, plus 2 buftags, plus a umem_slab_t, all fit on 121 * a single page. 122 * 123 * For ILP32, this is about 1000 frames. 124 * For LP64, this is about 490 frames. 125 */ 126 127 #define UMEM_BUFCTL_AUDIT_ALIGN 32 128 129 #define UMEM_BUFCTL_AUDIT_MAX_SIZE \ 130 (P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 - \ 131 sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN)) 132 133 #define UMEM_MAX_STACK_DEPTH \ 134 ((UMEM_BUFCTL_AUDIT_MAX_SIZE - \ 135 UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t)) 136 137 typedef struct umem_bufctl_audit { 138 struct umem_bufctl *bc_next; /* next bufctl struct */ 139 void *bc_addr; /* address of buffer */ 140 struct umem_slab *bc_slab; /* controlling slab */ 141 umem_cache_t *bc_cache; /* controlling cache */ 142 hrtime_t bc_timestamp; /* transaction time */ 143 thread_t bc_thread; /* thread doing transaction */ 144 struct umem_bufctl *bc_lastlog; /* last log entry */ 145 void *bc_contents; /* contents at last free */ 146 int bc_depth; /* stack depth */ 147 uintptr_t bc_stack[1]; /* pc stack */ 148 } umem_bufctl_audit_t; 149 150 #define UMEM_LOCAL_BUFCTL_AUDIT(bcpp) \ 151 *(bcpp) = (umem_bufctl_audit_t *) \ 152 alloca(UMEM_BUFCTL_AUDIT_SIZE) 153 154 #define UMEM_BUFCTL_AUDIT_SIZE \ 155 UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH) 156 157 /* 158 * A umem_buftag structure is appended to each buffer whenever any of the 159 * UMF_BUFTAG flags (UMF_DEADBEEF, UMF_REDZONE, UMF_VERIFY) are set. 160 */ 161 typedef struct umem_buftag { 162 uint64_t bt_redzone; /* 64-bit redzone pattern */ 163 umem_bufctl_t *bt_bufctl; /* bufctl */ 164 intptr_t bt_bxstat; /* bufctl ^ (alloc/free) */ 165 } umem_buftag_t; 166 167 #define UMEM_BUFTAG(cp, buf) \ 168 ((umem_buftag_t *)((char *)(buf) + (cp)->cache_buftag)) 169 170 #define UMEM_BUFCTL(cp, buf) \ 171 ((umem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl)) 172 173 #define UMEM_BUF(cp, bcp) \ 174 ((void *)((char *)(bcp) - (cp)->cache_bufctl)) 175 176 #define UMEM_SLAB(cp, buf) \ 177 ((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1) 178 179 #define UMEM_CPU_CACHE(cp, cpu) \ 180 (umem_cpu_cache_t *)((char *)cp + cpu->cpu_cache_offset) 181 182 #define UMEM_MAGAZINE_VALID(cp, mp) \ 183 (((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \ 184 (cp)->cache_magtype->mt_cache) 185 186 #define UMEM_SLAB_MEMBER(sp, buf) \ 187 ((size_t)(buf) - (size_t)(sp)->slab_base < \ 188 (sp)->slab_cache->cache_slabsize) 189 190 #define UMEM_BUFTAG_ALLOC 0xa110c8edUL 191 #define UMEM_BUFTAG_FREE 0xf4eef4eeUL 192 193 typedef struct umem_slab { 194 struct umem_cache *slab_cache; /* controlling cache */ 195 void *slab_base; /* base of allocated memory */ 196 struct umem_slab *slab_next; /* next slab on freelist */ 197 struct umem_slab *slab_prev; /* prev slab on freelist */ 198 struct umem_bufctl *slab_head; /* first free buffer */ 199 long slab_refcnt; /* outstanding allocations */ 200 long slab_chunks; /* chunks (bufs) in this slab */ 201 } umem_slab_t; 202 203 #define UMEM_HASH_INITIAL 64 204 205 #define UMEM_HASH(cp, buf) \ 206 ((cp)->cache_hash_table + \ 207 (((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask)) 208 209 typedef struct umem_magazine { 210 void *mag_next; 211 void *mag_round[1]; /* one or more rounds */ 212 } umem_magazine_t; 213 214 /* 215 * The magazine types for fast per-cpu allocation 216 */ 217 typedef struct umem_magtype { 218 int mt_magsize; /* magazine size (number of rounds) */ 219 int mt_align; /* magazine alignment */ 220 size_t mt_minbuf; /* all smaller buffers qualify */ 221 size_t mt_maxbuf; /* no larger buffers qualify */ 222 umem_cache_t *mt_cache; /* magazine cache */ 223 } umem_magtype_t; 224 225 #define UMEM_CPU_CACHE_SIZE 64 /* must be power of 2 */ 226 #define UMEM_CPU_PAD (UMEM_CPU_CACHE_SIZE - sizeof (mutex_t) - \ 227 2 * sizeof (uint_t) - 2 * sizeof (void *) - 4 * sizeof (int)) 228 #define UMEM_CACHE_SIZE(ncpus) \ 229 ((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus])) 230 231 typedef struct umem_cpu_cache { 232 mutex_t cc_lock; /* protects this cpu's local cache */ 233 uint_t cc_alloc; /* allocations from this cpu */ 234 uint_t cc_free; /* frees to this cpu */ 235 umem_magazine_t *cc_loaded; /* the currently loaded magazine */ 236 umem_magazine_t *cc_ploaded; /* the previously loaded magazine */ 237 int cc_rounds; /* number of objects in loaded mag */ 238 int cc_prounds; /* number of objects in previous mag */ 239 int cc_magsize; /* number of rounds in a full mag */ 240 int cc_flags; /* CPU-local copy of cache_flags */ 241 #ifndef _LP64 242 char cc_pad[UMEM_CPU_PAD]; /* for nice alignment (32-bit) */ 243 #endif 244 } umem_cpu_cache_t; 245 246 /* 247 * The magazine lists used in the depot. 248 */ 249 typedef struct umem_maglist { 250 umem_magazine_t *ml_list; /* magazine list */ 251 long ml_total; /* number of magazines */ 252 long ml_min; /* min since last update */ 253 long ml_reaplimit; /* max reapable magazines */ 254 uint64_t ml_alloc; /* allocations from this list */ 255 } umem_maglist_t; 256 257 #define UMEM_CACHE_NAMELEN 31 258 259 struct umem_cache { 260 /* 261 * Statistics 262 */ 263 uint64_t cache_slab_create; /* slab creates */ 264 uint64_t cache_slab_destroy; /* slab destroys */ 265 uint64_t cache_slab_alloc; /* slab layer allocations */ 266 uint64_t cache_slab_free; /* slab layer frees */ 267 uint64_t cache_alloc_fail; /* total failed allocations */ 268 uint64_t cache_buftotal; /* total buffers */ 269 uint64_t cache_bufmax; /* max buffers ever */ 270 uint64_t cache_rescale; /* # of hash table rescales */ 271 uint64_t cache_lookup_depth; /* hash lookup depth */ 272 uint64_t cache_depot_contention; /* mutex contention count */ 273 uint64_t cache_depot_contention_prev; /* previous snapshot */ 274 275 /* 276 * Cache properties 277 */ 278 char cache_name[UMEM_CACHE_NAMELEN + 1]; 279 size_t cache_bufsize; /* object size */ 280 size_t cache_align; /* object alignment */ 281 umem_constructor_t *cache_constructor; 282 umem_destructor_t *cache_destructor; 283 umem_reclaim_t *cache_reclaim; 284 void *cache_private; /* opaque arg to callbacks */ 285 vmem_t *cache_arena; /* vmem source for slabs */ 286 int cache_cflags; /* cache creation flags */ 287 int cache_flags; /* various cache state info */ 288 int cache_uflags; /* UMU_* flags */ 289 uint32_t cache_mtbf; /* induced alloc failure rate */ 290 umem_cache_t *cache_next; /* forward cache linkage */ 291 umem_cache_t *cache_prev; /* backward cache linkage */ 292 umem_cache_t *cache_unext; /* next in update list */ 293 umem_cache_t *cache_uprev; /* prev in update list */ 294 uint32_t cache_cpu_mask; /* mask for cpu offset */ 295 296 /* 297 * Slab layer 298 */ 299 mutex_t cache_lock; /* protects slab layer */ 300 size_t cache_chunksize; /* buf + alignment [+ debug] */ 301 size_t cache_slabsize; /* size of a slab */ 302 size_t cache_bufctl; /* buf-to-bufctl distance */ 303 size_t cache_buftag; /* buf-to-buftag distance */ 304 size_t cache_verify; /* bytes to verify */ 305 size_t cache_contents; /* bytes of saved content */ 306 size_t cache_color; /* next slab color */ 307 size_t cache_mincolor; /* maximum slab color */ 308 size_t cache_maxcolor; /* maximum slab color */ 309 size_t cache_hash_shift; /* get to interesting bits */ 310 size_t cache_hash_mask; /* hash table mask */ 311 umem_slab_t *cache_freelist; /* slab free list */ 312 umem_slab_t cache_nullslab; /* end of freelist marker */ 313 umem_cache_t *cache_bufctl_cache; /* source of bufctls */ 314 umem_bufctl_t **cache_hash_table; /* hash table base */ 315 /* 316 * Depot layer 317 */ 318 mutex_t cache_depot_lock; /* protects depot */ 319 umem_magtype_t *cache_magtype; /* magazine type */ 320 umem_maglist_t cache_full; /* full magazines */ 321 umem_maglist_t cache_empty; /* empty magazines */ 322 323 /* 324 * Per-CPU layer 325 */ 326 umem_cpu_cache_t cache_cpu[1]; /* cache_cpu_mask + 1 entries */ 327 }; 328 329 typedef struct umem_cpu_log_header { 330 mutex_t clh_lock; 331 char *clh_current; 332 size_t clh_avail; 333 int clh_chunk; 334 int clh_hits; 335 char clh_pad[64 - sizeof (mutex_t) - sizeof (char *) - 336 sizeof (size_t) - 2 * sizeof (int)]; 337 } umem_cpu_log_header_t; 338 339 typedef struct umem_log_header { 340 mutex_t lh_lock; 341 char *lh_base; 342 int *lh_free; 343 size_t lh_chunksize; 344 int lh_nchunks; 345 int lh_head; 346 int lh_tail; 347 int lh_hits; 348 umem_cpu_log_header_t lh_cpu[1]; /* actually umem_max_ncpus */ 349 } umem_log_header_t; 350 351 typedef struct umem_cpu { 352 uint32_t cpu_cache_offset; 353 uint32_t cpu_number; 354 } umem_cpu_t; 355 356 #define UMEM_MAXBUF 16384 357 358 #define UMEM_ALIGN 8 /* min guaranteed alignment */ 359 #define UMEM_ALIGN_SHIFT 3 /* log2(UMEM_ALIGN) */ 360 #define UMEM_VOID_FRACTION 8 /* never waste more than 1/8 of slab */ 361 362 /* 363 * For 64 bits, buffers >= 16 bytes must be 16-byte aligned 364 */ 365 #ifdef _LP64 366 #define UMEM_SECOND_ALIGN 16 367 #else 368 #define UMEM_SECOND_ALIGN UMEM_ALIGN 369 #endif 370 371 #define MALLOC_MAGIC 0x3a10c000 /* 8-byte tag */ 372 #define MEMALIGN_MAGIC 0x3e3a1000 373 374 #ifdef _LP64 375 #define MALLOC_SECOND_MAGIC 0x16ba7000 /* 8-byte tag, 16-aligned */ 376 #define MALLOC_OVERSIZE_MAGIC 0x06e47000 /* 16-byte tag, _LP64 */ 377 #endif 378 379 #define UMEM_MALLOC_ENCODE(type, sz) (uint32_t)((type) - (sz)) 380 #define UMEM_MALLOC_DECODE(stat, sz) (uint32_t)((stat) + (sz)) 381 #define UMEM_FREE_PATTERN_32 (uint32_t)(UMEM_FREE_PATTERN) 382 383 #define UMU_MAGAZINE_RESIZE 0x00000001 384 #define UMU_HASH_RESCALE 0x00000002 385 #define UMU_REAP 0x00000004 386 #define UMU_NOTIFY 0x08000000 387 #define UMU_ACTIVE 0x80000000 388 389 #define UMEM_READY_INIT_FAILED -1 390 #define UMEM_READY_STARTUP 1 391 #define UMEM_READY_INITING 2 392 #define UMEM_READY 3 393 394 #ifdef UMEM_STANDALONE 395 extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t); 396 extern int umem_add(caddr_t, size_t); 397 #endif 398 399 #ifdef __cplusplus 400 } 401 #endif 402 403 #endif /* _UMEM_IMPL_H */ 404