xref: /titanic_50/usr/src/uts/common/os/vmem.c (revision 903a11ebdc8df157c4700150f41f1f262f4a8ae8)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5aaa10e67Sha137994  * Common Development and Distribution License (the "License").
6aaa10e67Sha137994  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2210b95589Sbonwick  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate /*
297c478bd9Sstevel@tonic-gate  * Big Theory Statement for the virtual memory allocator.
307c478bd9Sstevel@tonic-gate  *
317c478bd9Sstevel@tonic-gate  * For a more complete description of the main ideas, see:
327c478bd9Sstevel@tonic-gate  *
337c478bd9Sstevel@tonic-gate  *	Jeff Bonwick and Jonathan Adams,
347c478bd9Sstevel@tonic-gate  *
357c478bd9Sstevel@tonic-gate  *	Magazines and vmem: Extending the Slab Allocator to Many CPUs and
367c478bd9Sstevel@tonic-gate  *	Arbitrary Resources.
377c478bd9Sstevel@tonic-gate  *
387c478bd9Sstevel@tonic-gate  *	Proceedings of the 2001 Usenix Conference.
397c478bd9Sstevel@tonic-gate  *	Available as http://www.usenix.org/event/usenix01/bonwick.html
407c478bd9Sstevel@tonic-gate  *
417c478bd9Sstevel@tonic-gate  *
427c478bd9Sstevel@tonic-gate  * 1. General Concepts
437c478bd9Sstevel@tonic-gate  * -------------------
447c478bd9Sstevel@tonic-gate  *
457c478bd9Sstevel@tonic-gate  * 1.1 Overview
467c478bd9Sstevel@tonic-gate  * ------------
477c478bd9Sstevel@tonic-gate  * We divide the kernel address space into a number of logically distinct
487c478bd9Sstevel@tonic-gate  * pieces, or *arenas*: text, data, heap, stack, and so on.  Within these
497c478bd9Sstevel@tonic-gate  * arenas we often subdivide further; for example, we use heap addresses
507c478bd9Sstevel@tonic-gate  * not only for the kernel heap (kmem_alloc() space), but also for DVMA,
517c478bd9Sstevel@tonic-gate  * bp_mapin(), /dev/kmem, and even some device mappings like the TOD chip.
527c478bd9Sstevel@tonic-gate  * The kernel address space, therefore, is most accurately described as
537c478bd9Sstevel@tonic-gate  * a tree of arenas in which each node of the tree *imports* some subset
547c478bd9Sstevel@tonic-gate  * of its parent.  The virtual memory allocator manages these arenas and
557c478bd9Sstevel@tonic-gate  * supports their natural hierarchical structure.
567c478bd9Sstevel@tonic-gate  *
577c478bd9Sstevel@tonic-gate  * 1.2 Arenas
587c478bd9Sstevel@tonic-gate  * ----------
597c478bd9Sstevel@tonic-gate  * An arena is nothing more than a set of integers.  These integers most
607c478bd9Sstevel@tonic-gate  * commonly represent virtual addresses, but in fact they can represent
617c478bd9Sstevel@tonic-gate  * anything at all.  For example, we could use an arena containing the
627c478bd9Sstevel@tonic-gate  * integers minpid through maxpid to allocate process IDs.  vmem_create()
637c478bd9Sstevel@tonic-gate  * and vmem_destroy() create and destroy vmem arenas.  In order to
647c478bd9Sstevel@tonic-gate  * differentiate between arenas used for adresses and arenas used for
657c478bd9Sstevel@tonic-gate  * identifiers, the VMC_IDENTIFIER flag is passed to vmem_create().  This
667c478bd9Sstevel@tonic-gate  * prevents identifier exhaustion from being diagnosed as general memory
677c478bd9Sstevel@tonic-gate  * failure.
687c478bd9Sstevel@tonic-gate  *
697c478bd9Sstevel@tonic-gate  * 1.3 Spans
707c478bd9Sstevel@tonic-gate  * ---------
717c478bd9Sstevel@tonic-gate  * We represent the integers in an arena as a collection of *spans*, or
727c478bd9Sstevel@tonic-gate  * contiguous ranges of integers.  For example, the kernel heap consists
737c478bd9Sstevel@tonic-gate  * of just one span: [kernelheap, ekernelheap).  Spans can be added to an
747c478bd9Sstevel@tonic-gate  * arena in two ways: explicitly, by vmem_add(), or implicitly, by
757c478bd9Sstevel@tonic-gate  * importing, as described in Section 1.5 below.
767c478bd9Sstevel@tonic-gate  *
777c478bd9Sstevel@tonic-gate  * 1.4 Segments
787c478bd9Sstevel@tonic-gate  * ------------
797c478bd9Sstevel@tonic-gate  * Spans are subdivided into *segments*, each of which is either allocated
807c478bd9Sstevel@tonic-gate  * or free.  A segment, like a span, is a contiguous range of integers.
817c478bd9Sstevel@tonic-gate  * Each allocated segment [addr, addr + size) represents exactly one
827c478bd9Sstevel@tonic-gate  * vmem_alloc(size) that returned addr.  Free segments represent the space
837c478bd9Sstevel@tonic-gate  * between allocated segments.  If two free segments are adjacent, we
847c478bd9Sstevel@tonic-gate  * coalesce them into one larger segment; that is, if segments [a, b) and
857c478bd9Sstevel@tonic-gate  * [b, c) are both free, we merge them into a single segment [a, c).
867c478bd9Sstevel@tonic-gate  * The segments within a span are linked together in increasing-address order
877c478bd9Sstevel@tonic-gate  * so we can easily determine whether coalescing is possible.
887c478bd9Sstevel@tonic-gate  *
897c478bd9Sstevel@tonic-gate  * Segments never cross span boundaries.  When all segments within
907c478bd9Sstevel@tonic-gate  * an imported span become free, we return the span to its source.
917c478bd9Sstevel@tonic-gate  *
927c478bd9Sstevel@tonic-gate  * 1.5 Imported Memory
937c478bd9Sstevel@tonic-gate  * -------------------
947c478bd9Sstevel@tonic-gate  * As mentioned in the overview, some arenas are logical subsets of
957c478bd9Sstevel@tonic-gate  * other arenas.  For example, kmem_va_arena (a virtual address cache
967c478bd9Sstevel@tonic-gate  * that satisfies most kmem_slab_create() requests) is just a subset
977c478bd9Sstevel@tonic-gate  * of heap_arena (the kernel heap) that provides caching for the most
987c478bd9Sstevel@tonic-gate  * common slab sizes.  When kmem_va_arena runs out of virtual memory,
997c478bd9Sstevel@tonic-gate  * it *imports* more from the heap; we say that heap_arena is the
1007c478bd9Sstevel@tonic-gate  * *vmem source* for kmem_va_arena.  vmem_create() allows you to
1017c478bd9Sstevel@tonic-gate  * specify any existing vmem arena as the source for your new arena.
1027c478bd9Sstevel@tonic-gate  * Topologically, since every arena is a child of at most one source,
1037c478bd9Sstevel@tonic-gate  * the set of all arenas forms a collection of trees.
1047c478bd9Sstevel@tonic-gate  *
1057c478bd9Sstevel@tonic-gate  * 1.6 Constrained Allocations
1067c478bd9Sstevel@tonic-gate  * ---------------------------
1077c478bd9Sstevel@tonic-gate  * Some vmem clients are quite picky about the kind of address they want.
1087c478bd9Sstevel@tonic-gate  * For example, the DVMA code may need an address that is at a particular
1097c478bd9Sstevel@tonic-gate  * phase with respect to some alignment (to get good cache coloring), or
1107c478bd9Sstevel@tonic-gate  * that lies within certain limits (the addressable range of a device),
1117c478bd9Sstevel@tonic-gate  * or that doesn't cross some boundary (a DMA counter restriction) --
1127c478bd9Sstevel@tonic-gate  * or all of the above.  vmem_xalloc() allows the client to specify any
1137c478bd9Sstevel@tonic-gate  * or all of these constraints.
1147c478bd9Sstevel@tonic-gate  *
1157c478bd9Sstevel@tonic-gate  * 1.7 The Vmem Quantum
1167c478bd9Sstevel@tonic-gate  * --------------------
1177c478bd9Sstevel@tonic-gate  * Every arena has a notion of 'quantum', specified at vmem_create() time,
1187c478bd9Sstevel@tonic-gate  * that defines the arena's minimum unit of currency.  Most commonly the
1197c478bd9Sstevel@tonic-gate  * quantum is either 1 or PAGESIZE, but any power of 2 is legal.
1207c478bd9Sstevel@tonic-gate  * All vmem allocations are guaranteed to be quantum-aligned.
1217c478bd9Sstevel@tonic-gate  *
1227c478bd9Sstevel@tonic-gate  * 1.8 Quantum Caching
1237c478bd9Sstevel@tonic-gate  * -------------------
1247c478bd9Sstevel@tonic-gate  * A vmem arena may be so hot (frequently used) that the scalability of vmem
1257c478bd9Sstevel@tonic-gate  * allocation is a significant concern.  We address this by allowing the most
1267c478bd9Sstevel@tonic-gate  * common allocation sizes to be serviced by the kernel memory allocator,
1277c478bd9Sstevel@tonic-gate  * which provides low-latency per-cpu caching.  The qcache_max argument to
1287c478bd9Sstevel@tonic-gate  * vmem_create() specifies the largest allocation size to cache.
1297c478bd9Sstevel@tonic-gate  *
1307c478bd9Sstevel@tonic-gate  * 1.9 Relationship to Kernel Memory Allocator
1317c478bd9Sstevel@tonic-gate  * -------------------------------------------
1327c478bd9Sstevel@tonic-gate  * Every kmem cache has a vmem arena as its slab supplier.  The kernel memory
1337c478bd9Sstevel@tonic-gate  * allocator uses vmem_alloc() and vmem_free() to create and destroy slabs.
1347c478bd9Sstevel@tonic-gate  *
1357c478bd9Sstevel@tonic-gate  *
1367c478bd9Sstevel@tonic-gate  * 2. Implementation
1377c478bd9Sstevel@tonic-gate  * -----------------
1387c478bd9Sstevel@tonic-gate  *
1397c478bd9Sstevel@tonic-gate  * 2.1 Segment lists and markers
1407c478bd9Sstevel@tonic-gate  * -----------------------------
1417c478bd9Sstevel@tonic-gate  * The segment structure (vmem_seg_t) contains two doubly-linked lists.
1427c478bd9Sstevel@tonic-gate  *
1437c478bd9Sstevel@tonic-gate  * The arena list (vs_anext/vs_aprev) links all segments in the arena.
1447c478bd9Sstevel@tonic-gate  * In addition to the allocated and free segments, the arena contains
1457c478bd9Sstevel@tonic-gate  * special marker segments at span boundaries.  Span markers simplify
1467c478bd9Sstevel@tonic-gate  * coalescing and importing logic by making it easy to tell both when
1477c478bd9Sstevel@tonic-gate  * we're at a span boundary (so we don't coalesce across it), and when
1487c478bd9Sstevel@tonic-gate  * a span is completely free (its neighbors will both be span markers).
1497c478bd9Sstevel@tonic-gate  *
1507c478bd9Sstevel@tonic-gate  * Imported spans will have vs_import set.
1517c478bd9Sstevel@tonic-gate  *
1527c478bd9Sstevel@tonic-gate  * The next-of-kin list (vs_knext/vs_kprev) links segments of the same type:
1537c478bd9Sstevel@tonic-gate  * (1) for allocated segments, vs_knext is the hash chain linkage;
1547c478bd9Sstevel@tonic-gate  * (2) for free segments, vs_knext is the freelist linkage;
1557c478bd9Sstevel@tonic-gate  * (3) for span marker segments, vs_knext is the next span marker.
1567c478bd9Sstevel@tonic-gate  *
1577c478bd9Sstevel@tonic-gate  * 2.2 Allocation hashing
1587c478bd9Sstevel@tonic-gate  * ----------------------
1597c478bd9Sstevel@tonic-gate  * We maintain a hash table of all allocated segments, hashed by address.
1607c478bd9Sstevel@tonic-gate  * This allows vmem_free() to discover the target segment in constant time.
1617c478bd9Sstevel@tonic-gate  * vmem_update() periodically resizes hash tables to keep hash chains short.
1627c478bd9Sstevel@tonic-gate  *
1637c478bd9Sstevel@tonic-gate  * 2.3 Freelist management
1647c478bd9Sstevel@tonic-gate  * -----------------------
1657c478bd9Sstevel@tonic-gate  * We maintain power-of-2 freelists for free segments, i.e. free segments
1667c478bd9Sstevel@tonic-gate  * of size >= 2^n reside in vmp->vm_freelist[n].  To ensure constant-time
1677c478bd9Sstevel@tonic-gate  * allocation, vmem_xalloc() looks not in the first freelist that *might*
1687c478bd9Sstevel@tonic-gate  * satisfy the allocation, but in the first freelist that *definitely*
1697c478bd9Sstevel@tonic-gate  * satisfies the allocation (unless VM_BESTFIT is specified, or all larger
1707c478bd9Sstevel@tonic-gate  * freelists are empty).  For example, a 1000-byte allocation will be
1717c478bd9Sstevel@tonic-gate  * satisfied not from the 512..1023-byte freelist, whose members *might*
1727c478bd9Sstevel@tonic-gate  * contains a 1000-byte segment, but from a 1024-byte or larger freelist,
1737c478bd9Sstevel@tonic-gate  * the first member of which will *definitely* satisfy the allocation.
1747c478bd9Sstevel@tonic-gate  * This ensures that vmem_xalloc() works in constant time.
1757c478bd9Sstevel@tonic-gate  *
1767c478bd9Sstevel@tonic-gate  * We maintain a bit map to determine quickly which freelists are non-empty.
1777c478bd9Sstevel@tonic-gate  * vmp->vm_freemap & (1 << n) is non-zero iff vmp->vm_freelist[n] is non-empty.
1787c478bd9Sstevel@tonic-gate  *
1797c478bd9Sstevel@tonic-gate  * The different freelists are linked together into one large freelist,
1807c478bd9Sstevel@tonic-gate  * with the freelist heads serving as markers.  Freelist markers simplify
1817c478bd9Sstevel@tonic-gate  * the maintenance of vm_freemap by making it easy to tell when we're taking
1827c478bd9Sstevel@tonic-gate  * the last member of a freelist (both of its neighbors will be markers).
1837c478bd9Sstevel@tonic-gate  *
1847c478bd9Sstevel@tonic-gate  * 2.4 Vmem Locking
1857c478bd9Sstevel@tonic-gate  * ----------------
1867c478bd9Sstevel@tonic-gate  * For simplicity, all arena state is protected by a per-arena lock.
1877c478bd9Sstevel@tonic-gate  * For very hot arenas, use quantum caching for scalability.
1887c478bd9Sstevel@tonic-gate  *
1897c478bd9Sstevel@tonic-gate  * 2.5 Vmem Population
1907c478bd9Sstevel@tonic-gate  * -------------------
1917c478bd9Sstevel@tonic-gate  * Any internal vmem routine that might need to allocate new segment
1927c478bd9Sstevel@tonic-gate  * structures must prepare in advance by calling vmem_populate(), which
1937c478bd9Sstevel@tonic-gate  * will preallocate enough vmem_seg_t's to get is through the entire
1947c478bd9Sstevel@tonic-gate  * operation without dropping the arena lock.
1957c478bd9Sstevel@tonic-gate  *
1967c478bd9Sstevel@tonic-gate  * 2.6 Auditing
1977c478bd9Sstevel@tonic-gate  * ------------
1987c478bd9Sstevel@tonic-gate  * If KMF_AUDIT is set in kmem_flags, we audit vmem allocations as well.
1997c478bd9Sstevel@tonic-gate  * Since virtual addresses cannot be scribbled on, there is no equivalent
2007c478bd9Sstevel@tonic-gate  * in vmem to redzone checking, deadbeef, or other kmem debugging features.
2017c478bd9Sstevel@tonic-gate  * Moreover, we do not audit frees because segment coalescing destroys the
2027c478bd9Sstevel@tonic-gate  * association between an address and its segment structure.  Auditing is
2037c478bd9Sstevel@tonic-gate  * thus intended primarily to keep track of who's consuming the arena.
2047c478bd9Sstevel@tonic-gate  * Debugging support could certainly be extended in the future if it proves
2057c478bd9Sstevel@tonic-gate  * necessary, but we do so much live checking via the allocation hash table
2067c478bd9Sstevel@tonic-gate  * that even non-DEBUG systems get quite a bit of sanity checking already.
2077c478bd9Sstevel@tonic-gate  */
2087c478bd9Sstevel@tonic-gate 
2097c478bd9Sstevel@tonic-gate #include <sys/vmem_impl.h>
2107c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
2117c478bd9Sstevel@tonic-gate #include <sys/kstat.h>
2127c478bd9Sstevel@tonic-gate #include <sys/param.h>
2137c478bd9Sstevel@tonic-gate #include <sys/systm.h>
2147c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
2157c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
2167c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
2177c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
2187c478bd9Sstevel@tonic-gate #include <sys/debug.h>
2197c478bd9Sstevel@tonic-gate #include <sys/panic.h>
2207c478bd9Sstevel@tonic-gate 
2217c478bd9Sstevel@tonic-gate #define	VMEM_INITIAL		10	/* early vmem arenas */
2227c478bd9Sstevel@tonic-gate #define	VMEM_SEG_INITIAL	200	/* early segments */
2237c478bd9Sstevel@tonic-gate 
2247c478bd9Sstevel@tonic-gate /*
2257c478bd9Sstevel@tonic-gate  * Adding a new span to an arena requires two segment structures: one to
2267c478bd9Sstevel@tonic-gate  * represent the span, and one to represent the free segment it contains.
2277c478bd9Sstevel@tonic-gate  */
2287c478bd9Sstevel@tonic-gate #define	VMEM_SEGS_PER_SPAN_CREATE	2
2297c478bd9Sstevel@tonic-gate 
2307c478bd9Sstevel@tonic-gate /*
2317c478bd9Sstevel@tonic-gate  * Allocating a piece of an existing segment requires 0-2 segment structures
2327c478bd9Sstevel@tonic-gate  * depending on how much of the segment we're allocating.
2337c478bd9Sstevel@tonic-gate  *
2347c478bd9Sstevel@tonic-gate  * To allocate the entire segment, no new segment structures are needed; we
2357c478bd9Sstevel@tonic-gate  * simply move the existing segment structure from the freelist to the
2367c478bd9Sstevel@tonic-gate  * allocation hash table.
2377c478bd9Sstevel@tonic-gate  *
2387c478bd9Sstevel@tonic-gate  * To allocate a piece from the left or right end of the segment, we must
2397c478bd9Sstevel@tonic-gate  * split the segment into two pieces (allocated part and remainder), so we
2407c478bd9Sstevel@tonic-gate  * need one new segment structure to represent the remainder.
2417c478bd9Sstevel@tonic-gate  *
2427c478bd9Sstevel@tonic-gate  * To allocate from the middle of a segment, we need two new segment strucures
2437c478bd9Sstevel@tonic-gate  * to represent the remainders on either side of the allocated part.
2447c478bd9Sstevel@tonic-gate  */
2457c478bd9Sstevel@tonic-gate #define	VMEM_SEGS_PER_EXACT_ALLOC	0
2467c478bd9Sstevel@tonic-gate #define	VMEM_SEGS_PER_LEFT_ALLOC	1
2477c478bd9Sstevel@tonic-gate #define	VMEM_SEGS_PER_RIGHT_ALLOC	1
2487c478bd9Sstevel@tonic-gate #define	VMEM_SEGS_PER_MIDDLE_ALLOC	2
2497c478bd9Sstevel@tonic-gate 
2507c478bd9Sstevel@tonic-gate /*
2517c478bd9Sstevel@tonic-gate  * vmem_populate() preallocates segment structures for vmem to do its work.
2527c478bd9Sstevel@tonic-gate  * It must preallocate enough for the worst case, which is when we must import
2537c478bd9Sstevel@tonic-gate  * a new span and then allocate from the middle of it.
2547c478bd9Sstevel@tonic-gate  */
2557c478bd9Sstevel@tonic-gate #define	VMEM_SEGS_PER_ALLOC_MAX		\
2567c478bd9Sstevel@tonic-gate 	(VMEM_SEGS_PER_SPAN_CREATE + VMEM_SEGS_PER_MIDDLE_ALLOC)
2577c478bd9Sstevel@tonic-gate 
2587c478bd9Sstevel@tonic-gate /*
2597c478bd9Sstevel@tonic-gate  * The segment structures themselves are allocated from vmem_seg_arena, so
2607c478bd9Sstevel@tonic-gate  * we have a recursion problem when vmem_seg_arena needs to populate itself.
2617c478bd9Sstevel@tonic-gate  * We address this by working out the maximum number of segment structures
2627c478bd9Sstevel@tonic-gate  * this act will require, and multiplying by the maximum number of threads
2637c478bd9Sstevel@tonic-gate  * that we'll allow to do it simultaneously.
2647c478bd9Sstevel@tonic-gate  *
2657c478bd9Sstevel@tonic-gate  * The worst-case segment consumption to populate vmem_seg_arena is as
2667c478bd9Sstevel@tonic-gate  * follows (depicted as a stack trace to indicate why events are occurring):
2677c478bd9Sstevel@tonic-gate  *
2687c478bd9Sstevel@tonic-gate  * (In order to lower the fragmentation in the heap_arena, we specify a
2697c478bd9Sstevel@tonic-gate  * minimum import size for the vmem_metadata_arena which is the same size
2707c478bd9Sstevel@tonic-gate  * as the kmem_va quantum cache allocations.  This causes the worst-case
2717c478bd9Sstevel@tonic-gate  * allocation from the vmem_metadata_arena to be 3 segments.)
2727c478bd9Sstevel@tonic-gate  *
2737c478bd9Sstevel@tonic-gate  * vmem_alloc(vmem_seg_arena)		-> 2 segs (span create + exact alloc)
2747c478bd9Sstevel@tonic-gate  *  segkmem_alloc(vmem_metadata_arena)
2757c478bd9Sstevel@tonic-gate  *   vmem_alloc(vmem_metadata_arena)	-> 3 segs (span create + left alloc)
2767c478bd9Sstevel@tonic-gate  *    vmem_alloc(heap_arena)		-> 1 seg (left alloc)
2777c478bd9Sstevel@tonic-gate  *   page_create()
2787c478bd9Sstevel@tonic-gate  *   hat_memload()
2797c478bd9Sstevel@tonic-gate  *    kmem_cache_alloc()
2807c478bd9Sstevel@tonic-gate  *     kmem_slab_create()
2817c478bd9Sstevel@tonic-gate  *	vmem_alloc(hat_memload_arena)	-> 2 segs (span create + exact alloc)
2827c478bd9Sstevel@tonic-gate  *	 segkmem_alloc(heap_arena)
2837c478bd9Sstevel@tonic-gate  *	  vmem_alloc(heap_arena)	-> 1 seg (left alloc)
2847c478bd9Sstevel@tonic-gate  *	  page_create()
2857c478bd9Sstevel@tonic-gate  *	  hat_memload()		-> (hat layer won't recurse further)
2867c478bd9Sstevel@tonic-gate  *
2877c478bd9Sstevel@tonic-gate  * The worst-case consumption for each arena is 3 segment structures.
2887c478bd9Sstevel@tonic-gate  * Of course, a 3-seg reserve could easily be blown by multiple threads.
2897c478bd9Sstevel@tonic-gate  * Therefore, we serialize all allocations from vmem_seg_arena (which is OK
2907c478bd9Sstevel@tonic-gate  * because they're rare).  We cannot allow a non-blocking allocation to get
2917c478bd9Sstevel@tonic-gate  * tied up behind a blocking allocation, however, so we use separate locks
29210b95589Sbonwick  * for VM_SLEEP and VM_NOSLEEP allocations.  Similarly, VM_PUSHPAGE allocations
29310b95589Sbonwick  * must not block behind ordinary VM_SLEEPs.  In addition, if the system is
2947c478bd9Sstevel@tonic-gate  * panicking then we must keep enough resources for panic_thread to do its
29510b95589Sbonwick  * work.  Thus we have at most four threads trying to allocate from
2967c478bd9Sstevel@tonic-gate  * vmem_seg_arena, and each thread consumes at most three segment structures,
29710b95589Sbonwick  * so we must maintain a 12-seg reserve.
2987c478bd9Sstevel@tonic-gate  */
29910b95589Sbonwick #define	VMEM_POPULATE_RESERVE	12
3007c478bd9Sstevel@tonic-gate 
3017c478bd9Sstevel@tonic-gate /*
3027c478bd9Sstevel@tonic-gate  * vmem_populate() ensures that each arena has VMEM_MINFREE seg structures
3037c478bd9Sstevel@tonic-gate  * so that it can satisfy the worst-case allocation *and* participate in
3047c478bd9Sstevel@tonic-gate  * worst-case allocation from vmem_seg_arena.
3057c478bd9Sstevel@tonic-gate  */
3067c478bd9Sstevel@tonic-gate #define	VMEM_MINFREE	(VMEM_POPULATE_RESERVE + VMEM_SEGS_PER_ALLOC_MAX)
3077c478bd9Sstevel@tonic-gate 
3087c478bd9Sstevel@tonic-gate static vmem_t vmem0[VMEM_INITIAL];
3097c478bd9Sstevel@tonic-gate static vmem_t *vmem_populator[VMEM_INITIAL];
3107c478bd9Sstevel@tonic-gate static uint32_t vmem_id;
3117c478bd9Sstevel@tonic-gate static uint32_t vmem_populators;
3127c478bd9Sstevel@tonic-gate static vmem_seg_t vmem_seg0[VMEM_SEG_INITIAL];
3137c478bd9Sstevel@tonic-gate static vmem_seg_t *vmem_segfree;
3147c478bd9Sstevel@tonic-gate static kmutex_t vmem_list_lock;
3157c478bd9Sstevel@tonic-gate static kmutex_t vmem_segfree_lock;
3167c478bd9Sstevel@tonic-gate static kmutex_t vmem_sleep_lock;
3177c478bd9Sstevel@tonic-gate static kmutex_t vmem_nosleep_lock;
31810b95589Sbonwick static kmutex_t vmem_pushpage_lock;
3197c478bd9Sstevel@tonic-gate static kmutex_t vmem_panic_lock;
3207c478bd9Sstevel@tonic-gate static vmem_t *vmem_list;
3217c478bd9Sstevel@tonic-gate static vmem_t *vmem_metadata_arena;
3227c478bd9Sstevel@tonic-gate static vmem_t *vmem_seg_arena;
3237c478bd9Sstevel@tonic-gate static vmem_t *vmem_hash_arena;
3247c478bd9Sstevel@tonic-gate static vmem_t *vmem_vmem_arena;
3257c478bd9Sstevel@tonic-gate static long vmem_update_interval = 15;	/* vmem_update() every 15 seconds */
3267c478bd9Sstevel@tonic-gate uint32_t vmem_mtbf;		/* mean time between failures [default: off] */
3277c478bd9Sstevel@tonic-gate size_t vmem_seg_size = sizeof (vmem_seg_t);
3287c478bd9Sstevel@tonic-gate 
3297c478bd9Sstevel@tonic-gate static vmem_kstat_t vmem_kstat_template = {
3307c478bd9Sstevel@tonic-gate 	{ "mem_inuse",		KSTAT_DATA_UINT64 },
3317c478bd9Sstevel@tonic-gate 	{ "mem_import",		KSTAT_DATA_UINT64 },
3327c478bd9Sstevel@tonic-gate 	{ "mem_total",		KSTAT_DATA_UINT64 },
3337c478bd9Sstevel@tonic-gate 	{ "vmem_source",	KSTAT_DATA_UINT32 },
3347c478bd9Sstevel@tonic-gate 	{ "alloc",		KSTAT_DATA_UINT64 },
3357c478bd9Sstevel@tonic-gate 	{ "free",		KSTAT_DATA_UINT64 },
3367c478bd9Sstevel@tonic-gate 	{ "wait",		KSTAT_DATA_UINT64 },
3377c478bd9Sstevel@tonic-gate 	{ "fail",		KSTAT_DATA_UINT64 },
3387c478bd9Sstevel@tonic-gate 	{ "lookup",		KSTAT_DATA_UINT64 },
3397c478bd9Sstevel@tonic-gate 	{ "search",		KSTAT_DATA_UINT64 },
3407c478bd9Sstevel@tonic-gate 	{ "populate_wait",	KSTAT_DATA_UINT64 },
3417c478bd9Sstevel@tonic-gate 	{ "populate_fail",	KSTAT_DATA_UINT64 },
3427c478bd9Sstevel@tonic-gate 	{ "contains",		KSTAT_DATA_UINT64 },
3437c478bd9Sstevel@tonic-gate 	{ "contains_search",	KSTAT_DATA_UINT64 },
3447c478bd9Sstevel@tonic-gate };
3457c478bd9Sstevel@tonic-gate 
3467c478bd9Sstevel@tonic-gate /*
3477c478bd9Sstevel@tonic-gate  * Insert/delete from arena list (type 'a') or next-of-kin list (type 'k').
3487c478bd9Sstevel@tonic-gate  */
3497c478bd9Sstevel@tonic-gate #define	VMEM_INSERT(vprev, vsp, type)					\
3507c478bd9Sstevel@tonic-gate {									\
3517c478bd9Sstevel@tonic-gate 	vmem_seg_t *vnext = (vprev)->vs_##type##next;			\
3527c478bd9Sstevel@tonic-gate 	(vsp)->vs_##type##next = (vnext);				\
3537c478bd9Sstevel@tonic-gate 	(vsp)->vs_##type##prev = (vprev);				\
3547c478bd9Sstevel@tonic-gate 	(vprev)->vs_##type##next = (vsp);				\
3557c478bd9Sstevel@tonic-gate 	(vnext)->vs_##type##prev = (vsp);				\
3567c478bd9Sstevel@tonic-gate }
3577c478bd9Sstevel@tonic-gate 
3587c478bd9Sstevel@tonic-gate #define	VMEM_DELETE(vsp, type)						\
3597c478bd9Sstevel@tonic-gate {									\
3607c478bd9Sstevel@tonic-gate 	vmem_seg_t *vprev = (vsp)->vs_##type##prev;			\
3617c478bd9Sstevel@tonic-gate 	vmem_seg_t *vnext = (vsp)->vs_##type##next;			\
3627c478bd9Sstevel@tonic-gate 	(vprev)->vs_##type##next = (vnext);				\
3637c478bd9Sstevel@tonic-gate 	(vnext)->vs_##type##prev = (vprev);				\
3647c478bd9Sstevel@tonic-gate }
3657c478bd9Sstevel@tonic-gate 
3667c478bd9Sstevel@tonic-gate /*
3677c478bd9Sstevel@tonic-gate  * Get a vmem_seg_t from the global segfree list.
3687c478bd9Sstevel@tonic-gate  */
3697c478bd9Sstevel@tonic-gate static vmem_seg_t *
3707c478bd9Sstevel@tonic-gate vmem_getseg_global(void)
3717c478bd9Sstevel@tonic-gate {
3727c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp;
3737c478bd9Sstevel@tonic-gate 
3747c478bd9Sstevel@tonic-gate 	mutex_enter(&vmem_segfree_lock);
3757c478bd9Sstevel@tonic-gate 	if ((vsp = vmem_segfree) != NULL)
3767c478bd9Sstevel@tonic-gate 		vmem_segfree = vsp->vs_knext;
3777c478bd9Sstevel@tonic-gate 	mutex_exit(&vmem_segfree_lock);
3787c478bd9Sstevel@tonic-gate 
3797c478bd9Sstevel@tonic-gate 	return (vsp);
3807c478bd9Sstevel@tonic-gate }
3817c478bd9Sstevel@tonic-gate 
3827c478bd9Sstevel@tonic-gate /*
3837c478bd9Sstevel@tonic-gate  * Put a vmem_seg_t on the global segfree list.
3847c478bd9Sstevel@tonic-gate  */
3857c478bd9Sstevel@tonic-gate static void
3867c478bd9Sstevel@tonic-gate vmem_putseg_global(vmem_seg_t *vsp)
3877c478bd9Sstevel@tonic-gate {
3887c478bd9Sstevel@tonic-gate 	mutex_enter(&vmem_segfree_lock);
3897c478bd9Sstevel@tonic-gate 	vsp->vs_knext = vmem_segfree;
3907c478bd9Sstevel@tonic-gate 	vmem_segfree = vsp;
3917c478bd9Sstevel@tonic-gate 	mutex_exit(&vmem_segfree_lock);
3927c478bd9Sstevel@tonic-gate }
3937c478bd9Sstevel@tonic-gate 
3947c478bd9Sstevel@tonic-gate /*
3957c478bd9Sstevel@tonic-gate  * Get a vmem_seg_t from vmp's segfree list.
3967c478bd9Sstevel@tonic-gate  */
3977c478bd9Sstevel@tonic-gate static vmem_seg_t *
3987c478bd9Sstevel@tonic-gate vmem_getseg(vmem_t *vmp)
3997c478bd9Sstevel@tonic-gate {
4007c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp;
4017c478bd9Sstevel@tonic-gate 
4027c478bd9Sstevel@tonic-gate 	ASSERT(vmp->vm_nsegfree > 0);
4037c478bd9Sstevel@tonic-gate 
4047c478bd9Sstevel@tonic-gate 	vsp = vmp->vm_segfree;
4057c478bd9Sstevel@tonic-gate 	vmp->vm_segfree = vsp->vs_knext;
4067c478bd9Sstevel@tonic-gate 	vmp->vm_nsegfree--;
4077c478bd9Sstevel@tonic-gate 
4087c478bd9Sstevel@tonic-gate 	return (vsp);
4097c478bd9Sstevel@tonic-gate }
4107c478bd9Sstevel@tonic-gate 
4117c478bd9Sstevel@tonic-gate /*
4127c478bd9Sstevel@tonic-gate  * Put a vmem_seg_t on vmp's segfree list.
4137c478bd9Sstevel@tonic-gate  */
4147c478bd9Sstevel@tonic-gate static void
4157c478bd9Sstevel@tonic-gate vmem_putseg(vmem_t *vmp, vmem_seg_t *vsp)
4167c478bd9Sstevel@tonic-gate {
4177c478bd9Sstevel@tonic-gate 	vsp->vs_knext = vmp->vm_segfree;
4187c478bd9Sstevel@tonic-gate 	vmp->vm_segfree = vsp;
4197c478bd9Sstevel@tonic-gate 	vmp->vm_nsegfree++;
4207c478bd9Sstevel@tonic-gate }
4217c478bd9Sstevel@tonic-gate 
4227c478bd9Sstevel@tonic-gate /*
4237c478bd9Sstevel@tonic-gate  * Add vsp to the appropriate freelist.
4247c478bd9Sstevel@tonic-gate  */
4257c478bd9Sstevel@tonic-gate static void
4267c478bd9Sstevel@tonic-gate vmem_freelist_insert(vmem_t *vmp, vmem_seg_t *vsp)
4277c478bd9Sstevel@tonic-gate {
4287c478bd9Sstevel@tonic-gate 	vmem_seg_t *vprev;
4297c478bd9Sstevel@tonic-gate 
4307c478bd9Sstevel@tonic-gate 	ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp);
4317c478bd9Sstevel@tonic-gate 
4327c478bd9Sstevel@tonic-gate 	vprev = (vmem_seg_t *)&vmp->vm_freelist[highbit(VS_SIZE(vsp)) - 1];
4337c478bd9Sstevel@tonic-gate 	vsp->vs_type = VMEM_FREE;
4347c478bd9Sstevel@tonic-gate 	vmp->vm_freemap |= VS_SIZE(vprev);
4357c478bd9Sstevel@tonic-gate 	VMEM_INSERT(vprev, vsp, k);
4367c478bd9Sstevel@tonic-gate 
4377c478bd9Sstevel@tonic-gate 	cv_broadcast(&vmp->vm_cv);
4387c478bd9Sstevel@tonic-gate }
4397c478bd9Sstevel@tonic-gate 
4407c478bd9Sstevel@tonic-gate /*
4417c478bd9Sstevel@tonic-gate  * Take vsp from the freelist.
4427c478bd9Sstevel@tonic-gate  */
4437c478bd9Sstevel@tonic-gate static void
4447c478bd9Sstevel@tonic-gate vmem_freelist_delete(vmem_t *vmp, vmem_seg_t *vsp)
4457c478bd9Sstevel@tonic-gate {
4467c478bd9Sstevel@tonic-gate 	ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp);
4477c478bd9Sstevel@tonic-gate 	ASSERT(vsp->vs_type == VMEM_FREE);
4487c478bd9Sstevel@tonic-gate 
4497c478bd9Sstevel@tonic-gate 	if (vsp->vs_knext->vs_start == 0 && vsp->vs_kprev->vs_start == 0) {
4507c478bd9Sstevel@tonic-gate 		/*
4517c478bd9Sstevel@tonic-gate 		 * The segments on both sides of 'vsp' are freelist heads,
4527c478bd9Sstevel@tonic-gate 		 * so taking vsp leaves the freelist at vsp->vs_kprev empty.
4537c478bd9Sstevel@tonic-gate 		 */
4547c478bd9Sstevel@tonic-gate 		ASSERT(vmp->vm_freemap & VS_SIZE(vsp->vs_kprev));
4557c478bd9Sstevel@tonic-gate 		vmp->vm_freemap ^= VS_SIZE(vsp->vs_kprev);
4567c478bd9Sstevel@tonic-gate 	}
4577c478bd9Sstevel@tonic-gate 	VMEM_DELETE(vsp, k);
4587c478bd9Sstevel@tonic-gate }
4597c478bd9Sstevel@tonic-gate 
4607c478bd9Sstevel@tonic-gate /*
4617c478bd9Sstevel@tonic-gate  * Add vsp to the allocated-segment hash table and update kstats.
4627c478bd9Sstevel@tonic-gate  */
4637c478bd9Sstevel@tonic-gate static void
4647c478bd9Sstevel@tonic-gate vmem_hash_insert(vmem_t *vmp, vmem_seg_t *vsp)
4657c478bd9Sstevel@tonic-gate {
4667c478bd9Sstevel@tonic-gate 	vmem_seg_t **bucket;
4677c478bd9Sstevel@tonic-gate 
4687c478bd9Sstevel@tonic-gate 	vsp->vs_type = VMEM_ALLOC;
4697c478bd9Sstevel@tonic-gate 	bucket = VMEM_HASH(vmp, vsp->vs_start);
4707c478bd9Sstevel@tonic-gate 	vsp->vs_knext = *bucket;
4717c478bd9Sstevel@tonic-gate 	*bucket = vsp;
4727c478bd9Sstevel@tonic-gate 
4737c478bd9Sstevel@tonic-gate 	if (vmem_seg_size == sizeof (vmem_seg_t)) {
4747c478bd9Sstevel@tonic-gate 		vsp->vs_depth = (uint8_t)getpcstack(vsp->vs_stack,
4757c478bd9Sstevel@tonic-gate 		    VMEM_STACK_DEPTH);
4767c478bd9Sstevel@tonic-gate 		vsp->vs_thread = curthread;
4777c478bd9Sstevel@tonic-gate 		vsp->vs_timestamp = gethrtime();
4787c478bd9Sstevel@tonic-gate 	} else {
4797c478bd9Sstevel@tonic-gate 		vsp->vs_depth = 0;
4807c478bd9Sstevel@tonic-gate 	}
4817c478bd9Sstevel@tonic-gate 
4827c478bd9Sstevel@tonic-gate 	vmp->vm_kstat.vk_alloc.value.ui64++;
4837c478bd9Sstevel@tonic-gate 	vmp->vm_kstat.vk_mem_inuse.value.ui64 += VS_SIZE(vsp);
4847c478bd9Sstevel@tonic-gate }
4857c478bd9Sstevel@tonic-gate 
4867c478bd9Sstevel@tonic-gate /*
4877c478bd9Sstevel@tonic-gate  * Remove vsp from the allocated-segment hash table and update kstats.
4887c478bd9Sstevel@tonic-gate  */
4897c478bd9Sstevel@tonic-gate static vmem_seg_t *
4907c478bd9Sstevel@tonic-gate vmem_hash_delete(vmem_t *vmp, uintptr_t addr, size_t size)
4917c478bd9Sstevel@tonic-gate {
4927c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp, **prev_vspp;
4937c478bd9Sstevel@tonic-gate 
4947c478bd9Sstevel@tonic-gate 	prev_vspp = VMEM_HASH(vmp, addr);
4957c478bd9Sstevel@tonic-gate 	while ((vsp = *prev_vspp) != NULL) {
4967c478bd9Sstevel@tonic-gate 		if (vsp->vs_start == addr) {
4977c478bd9Sstevel@tonic-gate 			*prev_vspp = vsp->vs_knext;
4987c478bd9Sstevel@tonic-gate 			break;
4997c478bd9Sstevel@tonic-gate 		}
5007c478bd9Sstevel@tonic-gate 		vmp->vm_kstat.vk_lookup.value.ui64++;
5017c478bd9Sstevel@tonic-gate 		prev_vspp = &vsp->vs_knext;
5027c478bd9Sstevel@tonic-gate 	}
5037c478bd9Sstevel@tonic-gate 
5047c478bd9Sstevel@tonic-gate 	if (vsp == NULL)
5057c478bd9Sstevel@tonic-gate 		panic("vmem_hash_delete(%p, %lx, %lu): bad free",
506*903a11ebSrh87107 		    (void *)vmp, addr, size);
5077c478bd9Sstevel@tonic-gate 	if (VS_SIZE(vsp) != size)
5087c478bd9Sstevel@tonic-gate 		panic("vmem_hash_delete(%p, %lx, %lu): wrong size (expect %lu)",
509*903a11ebSrh87107 		    (void *)vmp, addr, size, VS_SIZE(vsp));
5107c478bd9Sstevel@tonic-gate 
5117c478bd9Sstevel@tonic-gate 	vmp->vm_kstat.vk_free.value.ui64++;
5127c478bd9Sstevel@tonic-gate 	vmp->vm_kstat.vk_mem_inuse.value.ui64 -= size;
5137c478bd9Sstevel@tonic-gate 
5147c478bd9Sstevel@tonic-gate 	return (vsp);
5157c478bd9Sstevel@tonic-gate }
5167c478bd9Sstevel@tonic-gate 
5177c478bd9Sstevel@tonic-gate /*
5187c478bd9Sstevel@tonic-gate  * Create a segment spanning the range [start, end) and add it to the arena.
5197c478bd9Sstevel@tonic-gate  */
5207c478bd9Sstevel@tonic-gate static vmem_seg_t *
5217c478bd9Sstevel@tonic-gate vmem_seg_create(vmem_t *vmp, vmem_seg_t *vprev, uintptr_t start, uintptr_t end)
5227c478bd9Sstevel@tonic-gate {
5237c478bd9Sstevel@tonic-gate 	vmem_seg_t *newseg = vmem_getseg(vmp);
5247c478bd9Sstevel@tonic-gate 
5257c478bd9Sstevel@tonic-gate 	newseg->vs_start = start;
5267c478bd9Sstevel@tonic-gate 	newseg->vs_end = end;
5277c478bd9Sstevel@tonic-gate 	newseg->vs_type = 0;
5287c478bd9Sstevel@tonic-gate 	newseg->vs_import = 0;
5297c478bd9Sstevel@tonic-gate 
5307c478bd9Sstevel@tonic-gate 	VMEM_INSERT(vprev, newseg, a);
5317c478bd9Sstevel@tonic-gate 
5327c478bd9Sstevel@tonic-gate 	return (newseg);
5337c478bd9Sstevel@tonic-gate }
5347c478bd9Sstevel@tonic-gate 
5357c478bd9Sstevel@tonic-gate /*
5367c478bd9Sstevel@tonic-gate  * Remove segment vsp from the arena.
5377c478bd9Sstevel@tonic-gate  */
5387c478bd9Sstevel@tonic-gate static void
5397c478bd9Sstevel@tonic-gate vmem_seg_destroy(vmem_t *vmp, vmem_seg_t *vsp)
5407c478bd9Sstevel@tonic-gate {
5417c478bd9Sstevel@tonic-gate 	ASSERT(vsp->vs_type != VMEM_ROTOR);
5427c478bd9Sstevel@tonic-gate 	VMEM_DELETE(vsp, a);
5437c478bd9Sstevel@tonic-gate 
5447c478bd9Sstevel@tonic-gate 	vmem_putseg(vmp, vsp);
5457c478bd9Sstevel@tonic-gate }
5467c478bd9Sstevel@tonic-gate 
5477c478bd9Sstevel@tonic-gate /*
5487c478bd9Sstevel@tonic-gate  * Add the span [vaddr, vaddr + size) to vmp and update kstats.
5497c478bd9Sstevel@tonic-gate  */
5507c478bd9Sstevel@tonic-gate static vmem_seg_t *
5517c478bd9Sstevel@tonic-gate vmem_span_create(vmem_t *vmp, void *vaddr, size_t size, uint8_t import)
5527c478bd9Sstevel@tonic-gate {
5537c478bd9Sstevel@tonic-gate 	vmem_seg_t *newseg, *span;
5547c478bd9Sstevel@tonic-gate 	uintptr_t start = (uintptr_t)vaddr;
5557c478bd9Sstevel@tonic-gate 	uintptr_t end = start + size;
5567c478bd9Sstevel@tonic-gate 
5577c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&vmp->vm_lock));
5587c478bd9Sstevel@tonic-gate 
5597c478bd9Sstevel@tonic-gate 	if ((start | end) & (vmp->vm_quantum - 1))
5607c478bd9Sstevel@tonic-gate 		panic("vmem_span_create(%p, %p, %lu): misaligned",
561*903a11ebSrh87107 		    (void *)vmp, vaddr, size);
5627c478bd9Sstevel@tonic-gate 
5637c478bd9Sstevel@tonic-gate 	span = vmem_seg_create(vmp, vmp->vm_seg0.vs_aprev, start, end);
5647c478bd9Sstevel@tonic-gate 	span->vs_type = VMEM_SPAN;
5657c478bd9Sstevel@tonic-gate 	span->vs_import = import;
5667c478bd9Sstevel@tonic-gate 	VMEM_INSERT(vmp->vm_seg0.vs_kprev, span, k);
5677c478bd9Sstevel@tonic-gate 
5687c478bd9Sstevel@tonic-gate 	newseg = vmem_seg_create(vmp, span, start, end);
5697c478bd9Sstevel@tonic-gate 	vmem_freelist_insert(vmp, newseg);
5707c478bd9Sstevel@tonic-gate 
5717c478bd9Sstevel@tonic-gate 	if (import)
5727c478bd9Sstevel@tonic-gate 		vmp->vm_kstat.vk_mem_import.value.ui64 += size;
5737c478bd9Sstevel@tonic-gate 	vmp->vm_kstat.vk_mem_total.value.ui64 += size;
5747c478bd9Sstevel@tonic-gate 
5757c478bd9Sstevel@tonic-gate 	return (newseg);
5767c478bd9Sstevel@tonic-gate }
5777c478bd9Sstevel@tonic-gate 
5787c478bd9Sstevel@tonic-gate /*
5797c478bd9Sstevel@tonic-gate  * Remove span vsp from vmp and update kstats.
5807c478bd9Sstevel@tonic-gate  */
5817c478bd9Sstevel@tonic-gate static void
5827c478bd9Sstevel@tonic-gate vmem_span_destroy(vmem_t *vmp, vmem_seg_t *vsp)
5837c478bd9Sstevel@tonic-gate {
5847c478bd9Sstevel@tonic-gate 	vmem_seg_t *span = vsp->vs_aprev;
5857c478bd9Sstevel@tonic-gate 	size_t size = VS_SIZE(vsp);
5867c478bd9Sstevel@tonic-gate 
5877c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&vmp->vm_lock));
5887c478bd9Sstevel@tonic-gate 	ASSERT(span->vs_type == VMEM_SPAN);
5897c478bd9Sstevel@tonic-gate 
5907c478bd9Sstevel@tonic-gate 	if (span->vs_import)
5917c478bd9Sstevel@tonic-gate 		vmp->vm_kstat.vk_mem_import.value.ui64 -= size;
5927c478bd9Sstevel@tonic-gate 	vmp->vm_kstat.vk_mem_total.value.ui64 -= size;
5937c478bd9Sstevel@tonic-gate 
5947c478bd9Sstevel@tonic-gate 	VMEM_DELETE(span, k);
5957c478bd9Sstevel@tonic-gate 
5967c478bd9Sstevel@tonic-gate 	vmem_seg_destroy(vmp, vsp);
5977c478bd9Sstevel@tonic-gate 	vmem_seg_destroy(vmp, span);
5987c478bd9Sstevel@tonic-gate }
5997c478bd9Sstevel@tonic-gate 
6007c478bd9Sstevel@tonic-gate /*
6017c478bd9Sstevel@tonic-gate  * Allocate the subrange [addr, addr + size) from segment vsp.
6027c478bd9Sstevel@tonic-gate  * If there are leftovers on either side, place them on the freelist.
6037c478bd9Sstevel@tonic-gate  * Returns a pointer to the segment representing [addr, addr + size).
6047c478bd9Sstevel@tonic-gate  */
6057c478bd9Sstevel@tonic-gate static vmem_seg_t *
6067c478bd9Sstevel@tonic-gate vmem_seg_alloc(vmem_t *vmp, vmem_seg_t *vsp, uintptr_t addr, size_t size)
6077c478bd9Sstevel@tonic-gate {
6087c478bd9Sstevel@tonic-gate 	uintptr_t vs_start = vsp->vs_start;
6097c478bd9Sstevel@tonic-gate 	uintptr_t vs_end = vsp->vs_end;
6107c478bd9Sstevel@tonic-gate 	size_t vs_size = vs_end - vs_start;
6117c478bd9Sstevel@tonic-gate 	size_t realsize = P2ROUNDUP(size, vmp->vm_quantum);
6127c478bd9Sstevel@tonic-gate 	uintptr_t addr_end = addr + realsize;
6137c478bd9Sstevel@tonic-gate 
6147c478bd9Sstevel@tonic-gate 	ASSERT(P2PHASE(vs_start, vmp->vm_quantum) == 0);
6157c478bd9Sstevel@tonic-gate 	ASSERT(P2PHASE(addr, vmp->vm_quantum) == 0);
6167c478bd9Sstevel@tonic-gate 	ASSERT(vsp->vs_type == VMEM_FREE);
6177c478bd9Sstevel@tonic-gate 	ASSERT(addr >= vs_start && addr_end - 1 <= vs_end - 1);
6187c478bd9Sstevel@tonic-gate 	ASSERT(addr - 1 <= addr_end - 1);
6197c478bd9Sstevel@tonic-gate 
6207c478bd9Sstevel@tonic-gate 	/*
6217c478bd9Sstevel@tonic-gate 	 * If we're allocating from the start of the segment, and the
6227c478bd9Sstevel@tonic-gate 	 * remainder will be on the same freelist, we can save quite
6237c478bd9Sstevel@tonic-gate 	 * a bit of work.
6247c478bd9Sstevel@tonic-gate 	 */
6257c478bd9Sstevel@tonic-gate 	if (P2SAMEHIGHBIT(vs_size, vs_size - realsize) && addr == vs_start) {
6267c478bd9Sstevel@tonic-gate 		ASSERT(highbit(vs_size) == highbit(vs_size - realsize));
6277c478bd9Sstevel@tonic-gate 		vsp->vs_start = addr_end;
6287c478bd9Sstevel@tonic-gate 		vsp = vmem_seg_create(vmp, vsp->vs_aprev, addr, addr + size);
6297c478bd9Sstevel@tonic-gate 		vmem_hash_insert(vmp, vsp);
6307c478bd9Sstevel@tonic-gate 		return (vsp);
6317c478bd9Sstevel@tonic-gate 	}
6327c478bd9Sstevel@tonic-gate 
6337c478bd9Sstevel@tonic-gate 	vmem_freelist_delete(vmp, vsp);
6347c478bd9Sstevel@tonic-gate 
6357c478bd9Sstevel@tonic-gate 	if (vs_end != addr_end)
6367c478bd9Sstevel@tonic-gate 		vmem_freelist_insert(vmp,
6377c478bd9Sstevel@tonic-gate 		    vmem_seg_create(vmp, vsp, addr_end, vs_end));
6387c478bd9Sstevel@tonic-gate 
6397c478bd9Sstevel@tonic-gate 	if (vs_start != addr)
6407c478bd9Sstevel@tonic-gate 		vmem_freelist_insert(vmp,
6417c478bd9Sstevel@tonic-gate 		    vmem_seg_create(vmp, vsp->vs_aprev, vs_start, addr));
6427c478bd9Sstevel@tonic-gate 
6437c478bd9Sstevel@tonic-gate 	vsp->vs_start = addr;
6447c478bd9Sstevel@tonic-gate 	vsp->vs_end = addr + size;
6457c478bd9Sstevel@tonic-gate 
6467c478bd9Sstevel@tonic-gate 	vmem_hash_insert(vmp, vsp);
6477c478bd9Sstevel@tonic-gate 	return (vsp);
6487c478bd9Sstevel@tonic-gate }
6497c478bd9Sstevel@tonic-gate 
6507c478bd9Sstevel@tonic-gate /*
6517c478bd9Sstevel@tonic-gate  * Returns 1 if we are populating, 0 otherwise.
6527c478bd9Sstevel@tonic-gate  * Call it if we want to prevent recursion from HAT.
6537c478bd9Sstevel@tonic-gate  */
6547c478bd9Sstevel@tonic-gate int
6557c478bd9Sstevel@tonic-gate vmem_is_populator()
6567c478bd9Sstevel@tonic-gate {
6577c478bd9Sstevel@tonic-gate 	return (mutex_owner(&vmem_sleep_lock) == curthread ||
6587c478bd9Sstevel@tonic-gate 	    mutex_owner(&vmem_nosleep_lock) == curthread ||
65910b95589Sbonwick 	    mutex_owner(&vmem_pushpage_lock) == curthread ||
6607c478bd9Sstevel@tonic-gate 	    mutex_owner(&vmem_panic_lock) == curthread);
6617c478bd9Sstevel@tonic-gate }
6627c478bd9Sstevel@tonic-gate 
6637c478bd9Sstevel@tonic-gate /*
6647c478bd9Sstevel@tonic-gate  * Populate vmp's segfree list with VMEM_MINFREE vmem_seg_t structures.
6657c478bd9Sstevel@tonic-gate  */
6667c478bd9Sstevel@tonic-gate static int
6677c478bd9Sstevel@tonic-gate vmem_populate(vmem_t *vmp, int vmflag)
6687c478bd9Sstevel@tonic-gate {
6697c478bd9Sstevel@tonic-gate 	char *p;
6707c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp;
6717c478bd9Sstevel@tonic-gate 	ssize_t nseg;
6727c478bd9Sstevel@tonic-gate 	size_t size;
6737c478bd9Sstevel@tonic-gate 	kmutex_t *lp;
6747c478bd9Sstevel@tonic-gate 	int i;
6757c478bd9Sstevel@tonic-gate 
6767c478bd9Sstevel@tonic-gate 	while (vmp->vm_nsegfree < VMEM_MINFREE &&
6777c478bd9Sstevel@tonic-gate 	    (vsp = vmem_getseg_global()) != NULL)
6787c478bd9Sstevel@tonic-gate 		vmem_putseg(vmp, vsp);
6797c478bd9Sstevel@tonic-gate 
6807c478bd9Sstevel@tonic-gate 	if (vmp->vm_nsegfree >= VMEM_MINFREE)
6817c478bd9Sstevel@tonic-gate 		return (1);
6827c478bd9Sstevel@tonic-gate 
6837c478bd9Sstevel@tonic-gate 	/*
6847c478bd9Sstevel@tonic-gate 	 * If we're already populating, tap the reserve.
6857c478bd9Sstevel@tonic-gate 	 */
6867c478bd9Sstevel@tonic-gate 	if (vmem_is_populator()) {
6877c478bd9Sstevel@tonic-gate 		ASSERT(vmp->vm_cflags & VMC_POPULATOR);
6887c478bd9Sstevel@tonic-gate 		return (1);
6897c478bd9Sstevel@tonic-gate 	}
6907c478bd9Sstevel@tonic-gate 
6917c478bd9Sstevel@tonic-gate 	mutex_exit(&vmp->vm_lock);
6927c478bd9Sstevel@tonic-gate 
6937c478bd9Sstevel@tonic-gate 	if (panic_thread == curthread)
6947c478bd9Sstevel@tonic-gate 		lp = &vmem_panic_lock;
6957c478bd9Sstevel@tonic-gate 	else if (vmflag & VM_NOSLEEP)
6967c478bd9Sstevel@tonic-gate 		lp = &vmem_nosleep_lock;
69710b95589Sbonwick 	else if (vmflag & VM_PUSHPAGE)
69810b95589Sbonwick 		lp = &vmem_pushpage_lock;
6997c478bd9Sstevel@tonic-gate 	else
7007c478bd9Sstevel@tonic-gate 		lp = &vmem_sleep_lock;
7017c478bd9Sstevel@tonic-gate 
7027c478bd9Sstevel@tonic-gate 	mutex_enter(lp);
7037c478bd9Sstevel@tonic-gate 
7047c478bd9Sstevel@tonic-gate 	nseg = VMEM_MINFREE + vmem_populators * VMEM_POPULATE_RESERVE;
7057c478bd9Sstevel@tonic-gate 	size = P2ROUNDUP(nseg * vmem_seg_size, vmem_seg_arena->vm_quantum);
7067c478bd9Sstevel@tonic-gate 	nseg = size / vmem_seg_size;
7077c478bd9Sstevel@tonic-gate 
7087c478bd9Sstevel@tonic-gate 	/*
7097c478bd9Sstevel@tonic-gate 	 * The following vmem_alloc() may need to populate vmem_seg_arena
7107c478bd9Sstevel@tonic-gate 	 * and all the things it imports from.  When doing so, it will tap
7117c478bd9Sstevel@tonic-gate 	 * each arena's reserve to prevent recursion (see the block comment
7127c478bd9Sstevel@tonic-gate 	 * above the definition of VMEM_POPULATE_RESERVE).
7137c478bd9Sstevel@tonic-gate 	 */
7147c478bd9Sstevel@tonic-gate 	p = vmem_alloc(vmem_seg_arena, size, vmflag & VM_KMFLAGS);
7157c478bd9Sstevel@tonic-gate 	if (p == NULL) {
7167c478bd9Sstevel@tonic-gate 		mutex_exit(lp);
7177c478bd9Sstevel@tonic-gate 		mutex_enter(&vmp->vm_lock);
7187c478bd9Sstevel@tonic-gate 		vmp->vm_kstat.vk_populate_fail.value.ui64++;
7197c478bd9Sstevel@tonic-gate 		return (0);
7207c478bd9Sstevel@tonic-gate 	}
7217c478bd9Sstevel@tonic-gate 
7227c478bd9Sstevel@tonic-gate 	/*
7237c478bd9Sstevel@tonic-gate 	 * Restock the arenas that may have been depleted during population.
7247c478bd9Sstevel@tonic-gate 	 */
7257c478bd9Sstevel@tonic-gate 	for (i = 0; i < vmem_populators; i++) {
7267c478bd9Sstevel@tonic-gate 		mutex_enter(&vmem_populator[i]->vm_lock);
7277c478bd9Sstevel@tonic-gate 		while (vmem_populator[i]->vm_nsegfree < VMEM_POPULATE_RESERVE)
7287c478bd9Sstevel@tonic-gate 			vmem_putseg(vmem_populator[i],
7297c478bd9Sstevel@tonic-gate 			    (vmem_seg_t *)(p + --nseg * vmem_seg_size));
7307c478bd9Sstevel@tonic-gate 		mutex_exit(&vmem_populator[i]->vm_lock);
7317c478bd9Sstevel@tonic-gate 	}
7327c478bd9Sstevel@tonic-gate 
7337c478bd9Sstevel@tonic-gate 	mutex_exit(lp);
7347c478bd9Sstevel@tonic-gate 	mutex_enter(&vmp->vm_lock);
7357c478bd9Sstevel@tonic-gate 
7367c478bd9Sstevel@tonic-gate 	/*
7377c478bd9Sstevel@tonic-gate 	 * Now take our own segments.
7387c478bd9Sstevel@tonic-gate 	 */
7397c478bd9Sstevel@tonic-gate 	ASSERT(nseg >= VMEM_MINFREE);
7407c478bd9Sstevel@tonic-gate 	while (vmp->vm_nsegfree < VMEM_MINFREE)
7417c478bd9Sstevel@tonic-gate 		vmem_putseg(vmp, (vmem_seg_t *)(p + --nseg * vmem_seg_size));
7427c478bd9Sstevel@tonic-gate 
7437c478bd9Sstevel@tonic-gate 	/*
7447c478bd9Sstevel@tonic-gate 	 * Give the remainder to charity.
7457c478bd9Sstevel@tonic-gate 	 */
7467c478bd9Sstevel@tonic-gate 	while (nseg > 0)
7477c478bd9Sstevel@tonic-gate 		vmem_putseg_global((vmem_seg_t *)(p + --nseg * vmem_seg_size));
7487c478bd9Sstevel@tonic-gate 
7497c478bd9Sstevel@tonic-gate 	return (1);
7507c478bd9Sstevel@tonic-gate }
7517c478bd9Sstevel@tonic-gate 
7527c478bd9Sstevel@tonic-gate /*
7537c478bd9Sstevel@tonic-gate  * Advance a walker from its previous position to 'afterme'.
7547c478bd9Sstevel@tonic-gate  * Note: may drop and reacquire vmp->vm_lock.
7557c478bd9Sstevel@tonic-gate  */
7567c478bd9Sstevel@tonic-gate static void
7577c478bd9Sstevel@tonic-gate vmem_advance(vmem_t *vmp, vmem_seg_t *walker, vmem_seg_t *afterme)
7587c478bd9Sstevel@tonic-gate {
7597c478bd9Sstevel@tonic-gate 	vmem_seg_t *vprev = walker->vs_aprev;
7607c478bd9Sstevel@tonic-gate 	vmem_seg_t *vnext = walker->vs_anext;
7617c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp = NULL;
7627c478bd9Sstevel@tonic-gate 
7637c478bd9Sstevel@tonic-gate 	VMEM_DELETE(walker, a);
7647c478bd9Sstevel@tonic-gate 
7657c478bd9Sstevel@tonic-gate 	if (afterme != NULL)
7667c478bd9Sstevel@tonic-gate 		VMEM_INSERT(afterme, walker, a);
7677c478bd9Sstevel@tonic-gate 
7687c478bd9Sstevel@tonic-gate 	/*
7697c478bd9Sstevel@tonic-gate 	 * The walker segment's presence may have prevented its neighbors
7707c478bd9Sstevel@tonic-gate 	 * from coalescing.  If so, coalesce them now.
7717c478bd9Sstevel@tonic-gate 	 */
7727c478bd9Sstevel@tonic-gate 	if (vprev->vs_type == VMEM_FREE) {
7737c478bd9Sstevel@tonic-gate 		if (vnext->vs_type == VMEM_FREE) {
7747c478bd9Sstevel@tonic-gate 			ASSERT(vprev->vs_end == vnext->vs_start);
7757c478bd9Sstevel@tonic-gate 			vmem_freelist_delete(vmp, vnext);
7767c478bd9Sstevel@tonic-gate 			vmem_freelist_delete(vmp, vprev);
7777c478bd9Sstevel@tonic-gate 			vprev->vs_end = vnext->vs_end;
7787c478bd9Sstevel@tonic-gate 			vmem_freelist_insert(vmp, vprev);
7797c478bd9Sstevel@tonic-gate 			vmem_seg_destroy(vmp, vnext);
7807c478bd9Sstevel@tonic-gate 		}
7817c478bd9Sstevel@tonic-gate 		vsp = vprev;
7827c478bd9Sstevel@tonic-gate 	} else if (vnext->vs_type == VMEM_FREE) {
7837c478bd9Sstevel@tonic-gate 		vsp = vnext;
7847c478bd9Sstevel@tonic-gate 	}
7857c478bd9Sstevel@tonic-gate 
7867c478bd9Sstevel@tonic-gate 	/*
7877c478bd9Sstevel@tonic-gate 	 * vsp could represent a complete imported span,
7887c478bd9Sstevel@tonic-gate 	 * in which case we must return it to the source.
7897c478bd9Sstevel@tonic-gate 	 */
7907c478bd9Sstevel@tonic-gate 	if (vsp != NULL && vsp->vs_aprev->vs_import &&
7917c478bd9Sstevel@tonic-gate 	    vmp->vm_source_free != NULL &&
7927c478bd9Sstevel@tonic-gate 	    vsp->vs_aprev->vs_type == VMEM_SPAN &&
7937c478bd9Sstevel@tonic-gate 	    vsp->vs_anext->vs_type == VMEM_SPAN) {
7947c478bd9Sstevel@tonic-gate 		void *vaddr = (void *)vsp->vs_start;
7957c478bd9Sstevel@tonic-gate 		size_t size = VS_SIZE(vsp);
7967c478bd9Sstevel@tonic-gate 		ASSERT(size == VS_SIZE(vsp->vs_aprev));
7977c478bd9Sstevel@tonic-gate 		vmem_freelist_delete(vmp, vsp);
7987c478bd9Sstevel@tonic-gate 		vmem_span_destroy(vmp, vsp);
7997c478bd9Sstevel@tonic-gate 		mutex_exit(&vmp->vm_lock);
8007c478bd9Sstevel@tonic-gate 		vmp->vm_source_free(vmp->vm_source, vaddr, size);
8017c478bd9Sstevel@tonic-gate 		mutex_enter(&vmp->vm_lock);
8027c478bd9Sstevel@tonic-gate 	}
8037c478bd9Sstevel@tonic-gate }
8047c478bd9Sstevel@tonic-gate 
8057c478bd9Sstevel@tonic-gate /*
8067c478bd9Sstevel@tonic-gate  * VM_NEXTFIT allocations deliberately cycle through all virtual addresses
8077c478bd9Sstevel@tonic-gate  * in an arena, so that we avoid reusing addresses for as long as possible.
8087c478bd9Sstevel@tonic-gate  * This helps to catch used-after-freed bugs.  It's also the perfect policy
8097c478bd9Sstevel@tonic-gate  * for allocating things like process IDs, where we want to cycle through
8107c478bd9Sstevel@tonic-gate  * all values in order.
8117c478bd9Sstevel@tonic-gate  */
8127c478bd9Sstevel@tonic-gate static void *
8137c478bd9Sstevel@tonic-gate vmem_nextfit_alloc(vmem_t *vmp, size_t size, int vmflag)
8147c478bd9Sstevel@tonic-gate {
8157c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp, *rotor;
8167c478bd9Sstevel@tonic-gate 	uintptr_t addr;
8177c478bd9Sstevel@tonic-gate 	size_t realsize = P2ROUNDUP(size, vmp->vm_quantum);
8187c478bd9Sstevel@tonic-gate 	size_t vs_size;
8197c478bd9Sstevel@tonic-gate 
8207c478bd9Sstevel@tonic-gate 	mutex_enter(&vmp->vm_lock);
8217c478bd9Sstevel@tonic-gate 
8227c478bd9Sstevel@tonic-gate 	if (vmp->vm_nsegfree < VMEM_MINFREE && !vmem_populate(vmp, vmflag)) {
8237c478bd9Sstevel@tonic-gate 		mutex_exit(&vmp->vm_lock);
8247c478bd9Sstevel@tonic-gate 		return (NULL);
8257c478bd9Sstevel@tonic-gate 	}
8267c478bd9Sstevel@tonic-gate 
8277c478bd9Sstevel@tonic-gate 	/*
8287c478bd9Sstevel@tonic-gate 	 * The common case is that the segment right after the rotor is free,
8297c478bd9Sstevel@tonic-gate 	 * and large enough that extracting 'size' bytes won't change which
8307c478bd9Sstevel@tonic-gate 	 * freelist it's on.  In this case we can avoid a *lot* of work.
8317c478bd9Sstevel@tonic-gate 	 * Instead of the normal vmem_seg_alloc(), we just advance the start
8327c478bd9Sstevel@tonic-gate 	 * address of the victim segment.  Instead of moving the rotor, we
8337c478bd9Sstevel@tonic-gate 	 * create the new segment structure *behind the rotor*, which has
8347c478bd9Sstevel@tonic-gate 	 * the same effect.  And finally, we know we don't have to coalesce
8357c478bd9Sstevel@tonic-gate 	 * the rotor's neighbors because the new segment lies between them.
8367c478bd9Sstevel@tonic-gate 	 */
8377c478bd9Sstevel@tonic-gate 	rotor = &vmp->vm_rotor;
8387c478bd9Sstevel@tonic-gate 	vsp = rotor->vs_anext;
8397c478bd9Sstevel@tonic-gate 	if (vsp->vs_type == VMEM_FREE && (vs_size = VS_SIZE(vsp)) > realsize &&
8407c478bd9Sstevel@tonic-gate 	    P2SAMEHIGHBIT(vs_size, vs_size - realsize)) {
8417c478bd9Sstevel@tonic-gate 		ASSERT(highbit(vs_size) == highbit(vs_size - realsize));
8427c478bd9Sstevel@tonic-gate 		addr = vsp->vs_start;
8437c478bd9Sstevel@tonic-gate 		vsp->vs_start = addr + realsize;
8447c478bd9Sstevel@tonic-gate 		vmem_hash_insert(vmp,
8457c478bd9Sstevel@tonic-gate 		    vmem_seg_create(vmp, rotor->vs_aprev, addr, addr + size));
8467c478bd9Sstevel@tonic-gate 		mutex_exit(&vmp->vm_lock);
8477c478bd9Sstevel@tonic-gate 		return ((void *)addr);
8487c478bd9Sstevel@tonic-gate 	}
8497c478bd9Sstevel@tonic-gate 
8507c478bd9Sstevel@tonic-gate 	/*
8517c478bd9Sstevel@tonic-gate 	 * Starting at the rotor, look for a segment large enough to
8527c478bd9Sstevel@tonic-gate 	 * satisfy the allocation.
8537c478bd9Sstevel@tonic-gate 	 */
8547c478bd9Sstevel@tonic-gate 	for (;;) {
8557c478bd9Sstevel@tonic-gate 		vmp->vm_kstat.vk_search.value.ui64++;
8567c478bd9Sstevel@tonic-gate 		if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size)
8577c478bd9Sstevel@tonic-gate 			break;
8587c478bd9Sstevel@tonic-gate 		vsp = vsp->vs_anext;
8597c478bd9Sstevel@tonic-gate 		if (vsp == rotor) {
8607c478bd9Sstevel@tonic-gate 			/*
8617c478bd9Sstevel@tonic-gate 			 * We've come full circle.  One possibility is that the
8627c478bd9Sstevel@tonic-gate 			 * there's actually enough space, but the rotor itself
8637c478bd9Sstevel@tonic-gate 			 * is preventing the allocation from succeeding because
8647c478bd9Sstevel@tonic-gate 			 * it's sitting between two free segments.  Therefore,
8657c478bd9Sstevel@tonic-gate 			 * we advance the rotor and see if that liberates a
8667c478bd9Sstevel@tonic-gate 			 * suitable segment.
8677c478bd9Sstevel@tonic-gate 			 */
8687c478bd9Sstevel@tonic-gate 			vmem_advance(vmp, rotor, rotor->vs_anext);
8697c478bd9Sstevel@tonic-gate 			vsp = rotor->vs_aprev;
8707c478bd9Sstevel@tonic-gate 			if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size)
8717c478bd9Sstevel@tonic-gate 				break;
8727c478bd9Sstevel@tonic-gate 			/*
8737c478bd9Sstevel@tonic-gate 			 * If there's a lower arena we can import from, or it's
8747c478bd9Sstevel@tonic-gate 			 * a VM_NOSLEEP allocation, let vmem_xalloc() handle it.
8757c478bd9Sstevel@tonic-gate 			 * Otherwise, wait until another thread frees something.
8767c478bd9Sstevel@tonic-gate 			 */
8777c478bd9Sstevel@tonic-gate 			if (vmp->vm_source_alloc != NULL ||
8787c478bd9Sstevel@tonic-gate 			    (vmflag & VM_NOSLEEP)) {
8797c478bd9Sstevel@tonic-gate 				mutex_exit(&vmp->vm_lock);
8807c478bd9Sstevel@tonic-gate 				return (vmem_xalloc(vmp, size, vmp->vm_quantum,
8817c478bd9Sstevel@tonic-gate 				    0, 0, NULL, NULL, vmflag & VM_KMFLAGS));
8827c478bd9Sstevel@tonic-gate 			}
8837c478bd9Sstevel@tonic-gate 			vmp->vm_kstat.vk_wait.value.ui64++;
8847c478bd9Sstevel@tonic-gate 			cv_wait(&vmp->vm_cv, &vmp->vm_lock);
8857c478bd9Sstevel@tonic-gate 			vsp = rotor->vs_anext;
8867c478bd9Sstevel@tonic-gate 		}
8877c478bd9Sstevel@tonic-gate 	}
8887c478bd9Sstevel@tonic-gate 
8897c478bd9Sstevel@tonic-gate 	/*
8907c478bd9Sstevel@tonic-gate 	 * We found a segment.  Extract enough space to satisfy the allocation.
8917c478bd9Sstevel@tonic-gate 	 */
8927c478bd9Sstevel@tonic-gate 	addr = vsp->vs_start;
8937c478bd9Sstevel@tonic-gate 	vsp = vmem_seg_alloc(vmp, vsp, addr, size);
8947c478bd9Sstevel@tonic-gate 	ASSERT(vsp->vs_type == VMEM_ALLOC &&
8957c478bd9Sstevel@tonic-gate 	    vsp->vs_start == addr && vsp->vs_end == addr + size);
8967c478bd9Sstevel@tonic-gate 
8977c478bd9Sstevel@tonic-gate 	/*
8987c478bd9Sstevel@tonic-gate 	 * Advance the rotor to right after the newly-allocated segment.
8997c478bd9Sstevel@tonic-gate 	 * That's where the next VM_NEXTFIT allocation will begin searching.
9007c478bd9Sstevel@tonic-gate 	 */
9017c478bd9Sstevel@tonic-gate 	vmem_advance(vmp, rotor, vsp);
9027c478bd9Sstevel@tonic-gate 	mutex_exit(&vmp->vm_lock);
9037c478bd9Sstevel@tonic-gate 	return ((void *)addr);
9047c478bd9Sstevel@tonic-gate }
9057c478bd9Sstevel@tonic-gate 
9067c478bd9Sstevel@tonic-gate /*
9077c478bd9Sstevel@tonic-gate  * Checks if vmp is guaranteed to have a size-byte buffer somewhere on its
9087c478bd9Sstevel@tonic-gate  * freelist.  If size is not a power-of-2, it can return a false-negative.
9097c478bd9Sstevel@tonic-gate  *
9107c478bd9Sstevel@tonic-gate  * Used to decide if a newly imported span is superfluous after re-acquiring
9117c478bd9Sstevel@tonic-gate  * the arena lock.
9127c478bd9Sstevel@tonic-gate  */
9137c478bd9Sstevel@tonic-gate static int
9147c478bd9Sstevel@tonic-gate vmem_canalloc(vmem_t *vmp, size_t size)
9157c478bd9Sstevel@tonic-gate {
9167c478bd9Sstevel@tonic-gate 	int hb;
9177c478bd9Sstevel@tonic-gate 	int flist = 0;
9187c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&vmp->vm_lock));
9197c478bd9Sstevel@tonic-gate 
9207c478bd9Sstevel@tonic-gate 	if ((size & (size - 1)) == 0)
9217c478bd9Sstevel@tonic-gate 		flist = lowbit(P2ALIGN(vmp->vm_freemap, size));
9227c478bd9Sstevel@tonic-gate 	else if ((hb = highbit(size)) < VMEM_FREELISTS)
9237c478bd9Sstevel@tonic-gate 		flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb));
9247c478bd9Sstevel@tonic-gate 
9257c478bd9Sstevel@tonic-gate 	return (flist);
9267c478bd9Sstevel@tonic-gate }
9277c478bd9Sstevel@tonic-gate 
9287c478bd9Sstevel@tonic-gate /*
9297c478bd9Sstevel@tonic-gate  * Allocate size bytes at offset phase from an align boundary such that the
9307c478bd9Sstevel@tonic-gate  * resulting segment [addr, addr + size) is a subset of [minaddr, maxaddr)
9317c478bd9Sstevel@tonic-gate  * that does not straddle a nocross-aligned boundary.
9327c478bd9Sstevel@tonic-gate  */
9337c478bd9Sstevel@tonic-gate void *
9347c478bd9Sstevel@tonic-gate vmem_xalloc(vmem_t *vmp, size_t size, size_t align_arg, size_t phase,
9357c478bd9Sstevel@tonic-gate 	size_t nocross, void *minaddr, void *maxaddr, int vmflag)
9367c478bd9Sstevel@tonic-gate {
9377c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp;
9387c478bd9Sstevel@tonic-gate 	vmem_seg_t *vbest = NULL;
9397c478bd9Sstevel@tonic-gate 	uintptr_t addr, taddr, start, end;
9407c478bd9Sstevel@tonic-gate 	uintptr_t align = (align_arg != 0) ? align_arg : vmp->vm_quantum;
9417c478bd9Sstevel@tonic-gate 	void *vaddr, *xvaddr = NULL;
9427c478bd9Sstevel@tonic-gate 	size_t xsize;
9437c478bd9Sstevel@tonic-gate 	int hb, flist, resv;
9447c478bd9Sstevel@tonic-gate 	uint32_t mtbf;
9457c478bd9Sstevel@tonic-gate 
9467c478bd9Sstevel@tonic-gate 	if ((align | phase | nocross) & (vmp->vm_quantum - 1))
9477c478bd9Sstevel@tonic-gate 		panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): "
9487c478bd9Sstevel@tonic-gate 		    "parameters not vm_quantum aligned",
9497c478bd9Sstevel@tonic-gate 		    (void *)vmp, size, align_arg, phase, nocross,
9507c478bd9Sstevel@tonic-gate 		    minaddr, maxaddr, vmflag);
9517c478bd9Sstevel@tonic-gate 
9527c478bd9Sstevel@tonic-gate 	if (nocross != 0 &&
9537c478bd9Sstevel@tonic-gate 	    (align > nocross || P2ROUNDUP(phase + size, align) > nocross))
9547c478bd9Sstevel@tonic-gate 		panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): "
9557c478bd9Sstevel@tonic-gate 		    "overconstrained allocation",
9567c478bd9Sstevel@tonic-gate 		    (void *)vmp, size, align_arg, phase, nocross,
9577c478bd9Sstevel@tonic-gate 		    minaddr, maxaddr, vmflag);
9587c478bd9Sstevel@tonic-gate 
9597c478bd9Sstevel@tonic-gate 	if (phase >= align || (align & (align - 1)) != 0 ||
9607c478bd9Sstevel@tonic-gate 	    (nocross & (nocross - 1)) != 0)
9617c478bd9Sstevel@tonic-gate 		panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): "
9627c478bd9Sstevel@tonic-gate 		    "parameters inconsistent or invalid",
9637c478bd9Sstevel@tonic-gate 		    (void *)vmp, size, align_arg, phase, nocross,
9647c478bd9Sstevel@tonic-gate 		    minaddr, maxaddr, vmflag);
9657c478bd9Sstevel@tonic-gate 
9667c478bd9Sstevel@tonic-gate 	if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 &&
9677c478bd9Sstevel@tonic-gate 	    (vmflag & (VM_NOSLEEP | VM_PANIC)) == VM_NOSLEEP)
9687c478bd9Sstevel@tonic-gate 		return (NULL);
9697c478bd9Sstevel@tonic-gate 
9707c478bd9Sstevel@tonic-gate 	mutex_enter(&vmp->vm_lock);
9717c478bd9Sstevel@tonic-gate 	for (;;) {
9727c478bd9Sstevel@tonic-gate 		if (vmp->vm_nsegfree < VMEM_MINFREE &&
9737c478bd9Sstevel@tonic-gate 		    !vmem_populate(vmp, vmflag))
9747c478bd9Sstevel@tonic-gate 			break;
9757c478bd9Sstevel@tonic-gate do_alloc:
9767c478bd9Sstevel@tonic-gate 		/*
9777c478bd9Sstevel@tonic-gate 		 * highbit() returns the highest bit + 1, which is exactly
9787c478bd9Sstevel@tonic-gate 		 * what we want: we want to search the first freelist whose
9797c478bd9Sstevel@tonic-gate 		 * members are *definitely* large enough to satisfy our
9807c478bd9Sstevel@tonic-gate 		 * allocation.  However, there are certain cases in which we
9817c478bd9Sstevel@tonic-gate 		 * want to look at the next-smallest freelist (which *might*
9827c478bd9Sstevel@tonic-gate 		 * be able to satisfy the allocation):
9837c478bd9Sstevel@tonic-gate 		 *
9847c478bd9Sstevel@tonic-gate 		 * (1)	The size is exactly a power of 2, in which case
9857c478bd9Sstevel@tonic-gate 		 *	the smaller freelist is always big enough;
9867c478bd9Sstevel@tonic-gate 		 *
9877c478bd9Sstevel@tonic-gate 		 * (2)	All other freelists are empty;
9887c478bd9Sstevel@tonic-gate 		 *
9897c478bd9Sstevel@tonic-gate 		 * (3)	We're in the highest possible freelist, which is
9907c478bd9Sstevel@tonic-gate 		 *	always empty (e.g. the 4GB freelist on 32-bit systems);
9917c478bd9Sstevel@tonic-gate 		 *
9927c478bd9Sstevel@tonic-gate 		 * (4)	We're doing a best-fit or first-fit allocation.
9937c478bd9Sstevel@tonic-gate 		 */
9947c478bd9Sstevel@tonic-gate 		if ((size & (size - 1)) == 0) {
9957c478bd9Sstevel@tonic-gate 			flist = lowbit(P2ALIGN(vmp->vm_freemap, size));
9967c478bd9Sstevel@tonic-gate 		} else {
9977c478bd9Sstevel@tonic-gate 			hb = highbit(size);
9987c478bd9Sstevel@tonic-gate 			if ((vmp->vm_freemap >> hb) == 0 ||
9997c478bd9Sstevel@tonic-gate 			    hb == VMEM_FREELISTS ||
10007c478bd9Sstevel@tonic-gate 			    (vmflag & (VM_BESTFIT | VM_FIRSTFIT)))
10017c478bd9Sstevel@tonic-gate 				hb--;
10027c478bd9Sstevel@tonic-gate 			flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb));
10037c478bd9Sstevel@tonic-gate 		}
10047c478bd9Sstevel@tonic-gate 
10057c478bd9Sstevel@tonic-gate 		for (vbest = NULL, vsp = (flist == 0) ? NULL :
10067c478bd9Sstevel@tonic-gate 		    vmp->vm_freelist[flist - 1].vs_knext;
10077c478bd9Sstevel@tonic-gate 		    vsp != NULL; vsp = vsp->vs_knext) {
10087c478bd9Sstevel@tonic-gate 			vmp->vm_kstat.vk_search.value.ui64++;
10097c478bd9Sstevel@tonic-gate 			if (vsp->vs_start == 0) {
10107c478bd9Sstevel@tonic-gate 				/*
10117c478bd9Sstevel@tonic-gate 				 * We're moving up to a larger freelist,
10127c478bd9Sstevel@tonic-gate 				 * so if we've already found a candidate,
10137c478bd9Sstevel@tonic-gate 				 * the fit can't possibly get any better.
10147c478bd9Sstevel@tonic-gate 				 */
10157c478bd9Sstevel@tonic-gate 				if (vbest != NULL)
10167c478bd9Sstevel@tonic-gate 					break;
10177c478bd9Sstevel@tonic-gate 				/*
10187c478bd9Sstevel@tonic-gate 				 * Find the next non-empty freelist.
10197c478bd9Sstevel@tonic-gate 				 */
10207c478bd9Sstevel@tonic-gate 				flist = lowbit(P2ALIGN(vmp->vm_freemap,
10217c478bd9Sstevel@tonic-gate 				    VS_SIZE(vsp)));
10227c478bd9Sstevel@tonic-gate 				if (flist-- == 0)
10237c478bd9Sstevel@tonic-gate 					break;
10247c478bd9Sstevel@tonic-gate 				vsp = (vmem_seg_t *)&vmp->vm_freelist[flist];
10257c478bd9Sstevel@tonic-gate 				ASSERT(vsp->vs_knext->vs_type == VMEM_FREE);
10267c478bd9Sstevel@tonic-gate 				continue;
10277c478bd9Sstevel@tonic-gate 			}
10287c478bd9Sstevel@tonic-gate 			if (vsp->vs_end - 1 < (uintptr_t)minaddr)
10297c478bd9Sstevel@tonic-gate 				continue;
10307c478bd9Sstevel@tonic-gate 			if (vsp->vs_start > (uintptr_t)maxaddr - 1)
10317c478bd9Sstevel@tonic-gate 				continue;
10327c478bd9Sstevel@tonic-gate 			start = MAX(vsp->vs_start, (uintptr_t)minaddr);
10337c478bd9Sstevel@tonic-gate 			end = MIN(vsp->vs_end - 1, (uintptr_t)maxaddr - 1) + 1;
10347c478bd9Sstevel@tonic-gate 			taddr = P2PHASEUP(start, align, phase);
10357c478bd9Sstevel@tonic-gate 			if (P2CROSS(taddr, taddr + size - 1, nocross))
10367c478bd9Sstevel@tonic-gate 				taddr +=
10377c478bd9Sstevel@tonic-gate 				    P2ROUNDUP(P2NPHASE(taddr, nocross), align);
10387c478bd9Sstevel@tonic-gate 			if ((taddr - start) + size > end - start ||
10397c478bd9Sstevel@tonic-gate 			    (vbest != NULL && VS_SIZE(vsp) >= VS_SIZE(vbest)))
10407c478bd9Sstevel@tonic-gate 				continue;
10417c478bd9Sstevel@tonic-gate 			vbest = vsp;
10427c478bd9Sstevel@tonic-gate 			addr = taddr;
10437c478bd9Sstevel@tonic-gate 			if (!(vmflag & VM_BESTFIT) || VS_SIZE(vbest) == size)
10447c478bd9Sstevel@tonic-gate 				break;
10457c478bd9Sstevel@tonic-gate 		}
10467c478bd9Sstevel@tonic-gate 		if (vbest != NULL)
10477c478bd9Sstevel@tonic-gate 			break;
10487c478bd9Sstevel@tonic-gate 		ASSERT(xvaddr == NULL);
10497c478bd9Sstevel@tonic-gate 		if (size == 0)
10507c478bd9Sstevel@tonic-gate 			panic("vmem_xalloc(): size == 0");
10517c478bd9Sstevel@tonic-gate 		if (vmp->vm_source_alloc != NULL && nocross == 0 &&
10527c478bd9Sstevel@tonic-gate 		    minaddr == NULL && maxaddr == NULL) {
10537c478bd9Sstevel@tonic-gate 			size_t aneeded, asize;
10547c478bd9Sstevel@tonic-gate 			size_t aquantum = MAX(vmp->vm_quantum,
10557c478bd9Sstevel@tonic-gate 			    vmp->vm_source->vm_quantum);
10567c478bd9Sstevel@tonic-gate 			size_t aphase = phase;
1057aaa10e67Sha137994 			if ((align > aquantum) &&
1058aaa10e67Sha137994 			    !(vmp->vm_cflags & VMC_XALIGN)) {
10597c478bd9Sstevel@tonic-gate 				aphase = (P2PHASE(phase, aquantum) != 0) ?
10607c478bd9Sstevel@tonic-gate 				    align - vmp->vm_quantum : align - aquantum;
10617c478bd9Sstevel@tonic-gate 				ASSERT(aphase >= phase);
10627c478bd9Sstevel@tonic-gate 			}
10637c478bd9Sstevel@tonic-gate 			aneeded = MAX(size + aphase, vmp->vm_min_import);
10647c478bd9Sstevel@tonic-gate 			asize = P2ROUNDUP(aneeded, aquantum);
10657c478bd9Sstevel@tonic-gate 
10667c478bd9Sstevel@tonic-gate 			/*
10677c478bd9Sstevel@tonic-gate 			 * Determine how many segment structures we'll consume.
10687c478bd9Sstevel@tonic-gate 			 * The calculation must be precise because if we're
10697c478bd9Sstevel@tonic-gate 			 * here on behalf of vmem_populate(), we are taking
10707c478bd9Sstevel@tonic-gate 			 * segments from a very limited reserve.
10717c478bd9Sstevel@tonic-gate 			 */
10727c478bd9Sstevel@tonic-gate 			if (size == asize && !(vmp->vm_cflags & VMC_XALLOC))
10737c478bd9Sstevel@tonic-gate 				resv = VMEM_SEGS_PER_SPAN_CREATE +
10747c478bd9Sstevel@tonic-gate 				    VMEM_SEGS_PER_EXACT_ALLOC;
10757c478bd9Sstevel@tonic-gate 			else if (phase == 0 &&
10767c478bd9Sstevel@tonic-gate 			    align <= vmp->vm_source->vm_quantum)
10777c478bd9Sstevel@tonic-gate 				resv = VMEM_SEGS_PER_SPAN_CREATE +
10787c478bd9Sstevel@tonic-gate 				    VMEM_SEGS_PER_LEFT_ALLOC;
10797c478bd9Sstevel@tonic-gate 			else
10807c478bd9Sstevel@tonic-gate 				resv = VMEM_SEGS_PER_ALLOC_MAX;
10817c478bd9Sstevel@tonic-gate 
10827c478bd9Sstevel@tonic-gate 			ASSERT(vmp->vm_nsegfree >= resv);
10837c478bd9Sstevel@tonic-gate 			vmp->vm_nsegfree -= resv;	/* reserve our segs */
10847c478bd9Sstevel@tonic-gate 			mutex_exit(&vmp->vm_lock);
10857c478bd9Sstevel@tonic-gate 			if (vmp->vm_cflags & VMC_XALLOC) {
10867c478bd9Sstevel@tonic-gate 				size_t oasize = asize;
10877c478bd9Sstevel@tonic-gate 				vaddr = ((vmem_ximport_t *)
10887c478bd9Sstevel@tonic-gate 				    vmp->vm_source_alloc)(vmp->vm_source,
1089aaa10e67Sha137994 				    &asize, align, vmflag & VM_KMFLAGS);
10907c478bd9Sstevel@tonic-gate 				ASSERT(asize >= oasize);
10917c478bd9Sstevel@tonic-gate 				ASSERT(P2PHASE(asize,
10927c478bd9Sstevel@tonic-gate 				    vmp->vm_source->vm_quantum) == 0);
1093aaa10e67Sha137994 				ASSERT(!(vmp->vm_cflags & VMC_XALIGN) ||
1094aaa10e67Sha137994 				    IS_P2ALIGNED(vaddr, align));
10957c478bd9Sstevel@tonic-gate 			} else {
10967c478bd9Sstevel@tonic-gate 				vaddr = vmp->vm_source_alloc(vmp->vm_source,
10977c478bd9Sstevel@tonic-gate 				    asize, vmflag & VM_KMFLAGS);
10987c478bd9Sstevel@tonic-gate 			}
10997c478bd9Sstevel@tonic-gate 			mutex_enter(&vmp->vm_lock);
11007c478bd9Sstevel@tonic-gate 			vmp->vm_nsegfree += resv;	/* claim reservation */
11017c478bd9Sstevel@tonic-gate 			aneeded = size + align - vmp->vm_quantum;
11027c478bd9Sstevel@tonic-gate 			aneeded = P2ROUNDUP(aneeded, vmp->vm_quantum);
11037c478bd9Sstevel@tonic-gate 			if (vaddr != NULL) {
11047c478bd9Sstevel@tonic-gate 				/*
11057c478bd9Sstevel@tonic-gate 				 * Since we dropped the vmem lock while
11067c478bd9Sstevel@tonic-gate 				 * calling the import function, other
11077c478bd9Sstevel@tonic-gate 				 * threads could have imported space
11087c478bd9Sstevel@tonic-gate 				 * and made our import unnecessary.  In
11097c478bd9Sstevel@tonic-gate 				 * order to save space, we return
11107c478bd9Sstevel@tonic-gate 				 * excess imports immediately.
11117c478bd9Sstevel@tonic-gate 				 */
11127c478bd9Sstevel@tonic-gate 				if (asize > aneeded &&
11137c478bd9Sstevel@tonic-gate 				    vmp->vm_source_free != NULL &&
11147c478bd9Sstevel@tonic-gate 				    vmem_canalloc(vmp, aneeded)) {
11157c478bd9Sstevel@tonic-gate 					ASSERT(resv >=
11167c478bd9Sstevel@tonic-gate 					    VMEM_SEGS_PER_MIDDLE_ALLOC);
11177c478bd9Sstevel@tonic-gate 					xvaddr = vaddr;
11187c478bd9Sstevel@tonic-gate 					xsize = asize;
11197c478bd9Sstevel@tonic-gate 					goto do_alloc;
11207c478bd9Sstevel@tonic-gate 				}
11217c478bd9Sstevel@tonic-gate 				vbest = vmem_span_create(vmp, vaddr, asize, 1);
11227c478bd9Sstevel@tonic-gate 				addr = P2PHASEUP(vbest->vs_start, align, phase);
11237c478bd9Sstevel@tonic-gate 				break;
11247c478bd9Sstevel@tonic-gate 			} else if (vmem_canalloc(vmp, aneeded)) {
11257c478bd9Sstevel@tonic-gate 				/*
11267c478bd9Sstevel@tonic-gate 				 * Our import failed, but another thread
11277c478bd9Sstevel@tonic-gate 				 * added sufficient free memory to the arena
11287c478bd9Sstevel@tonic-gate 				 * to satisfy our request.  Go back and
11297c478bd9Sstevel@tonic-gate 				 * grab it.
11307c478bd9Sstevel@tonic-gate 				 */
11317c478bd9Sstevel@tonic-gate 				ASSERT(resv >= VMEM_SEGS_PER_MIDDLE_ALLOC);
11327c478bd9Sstevel@tonic-gate 				goto do_alloc;
11337c478bd9Sstevel@tonic-gate 			}
11347c478bd9Sstevel@tonic-gate 		}
11357c478bd9Sstevel@tonic-gate 
11367c478bd9Sstevel@tonic-gate 		/*
11377c478bd9Sstevel@tonic-gate 		 * If the requestor chooses to fail the allocation attempt
11387c478bd9Sstevel@tonic-gate 		 * rather than reap wait and retry - get out of the loop.
11397c478bd9Sstevel@tonic-gate 		 */
11407c478bd9Sstevel@tonic-gate 		if (vmflag & VM_ABORT)
11417c478bd9Sstevel@tonic-gate 			break;
11427c478bd9Sstevel@tonic-gate 		mutex_exit(&vmp->vm_lock);
11437c478bd9Sstevel@tonic-gate 		if (vmp->vm_cflags & VMC_IDENTIFIER)
11447c478bd9Sstevel@tonic-gate 			kmem_reap_idspace();
11457c478bd9Sstevel@tonic-gate 		else
11467c478bd9Sstevel@tonic-gate 			kmem_reap();
11477c478bd9Sstevel@tonic-gate 		mutex_enter(&vmp->vm_lock);
11487c478bd9Sstevel@tonic-gate 		if (vmflag & VM_NOSLEEP)
11497c478bd9Sstevel@tonic-gate 			break;
11507c478bd9Sstevel@tonic-gate 		vmp->vm_kstat.vk_wait.value.ui64++;
11517c478bd9Sstevel@tonic-gate 		cv_wait(&vmp->vm_cv, &vmp->vm_lock);
11527c478bd9Sstevel@tonic-gate 	}
11537c478bd9Sstevel@tonic-gate 	if (vbest != NULL) {
11547c478bd9Sstevel@tonic-gate 		ASSERT(vbest->vs_type == VMEM_FREE);
11557c478bd9Sstevel@tonic-gate 		ASSERT(vbest->vs_knext != vbest);
11567c478bd9Sstevel@tonic-gate 		(void) vmem_seg_alloc(vmp, vbest, addr, size);
11577c478bd9Sstevel@tonic-gate 		mutex_exit(&vmp->vm_lock);
11587c478bd9Sstevel@tonic-gate 		if (xvaddr)
11597c478bd9Sstevel@tonic-gate 			vmp->vm_source_free(vmp->vm_source, xvaddr, xsize);
11607c478bd9Sstevel@tonic-gate 		ASSERT(P2PHASE(addr, align) == phase);
11617c478bd9Sstevel@tonic-gate 		ASSERT(!P2CROSS(addr, addr + size - 1, nocross));
11627c478bd9Sstevel@tonic-gate 		ASSERT(addr >= (uintptr_t)minaddr);
11637c478bd9Sstevel@tonic-gate 		ASSERT(addr + size - 1 <= (uintptr_t)maxaddr - 1);
11647c478bd9Sstevel@tonic-gate 		return ((void *)addr);
11657c478bd9Sstevel@tonic-gate 	}
11667c478bd9Sstevel@tonic-gate 	vmp->vm_kstat.vk_fail.value.ui64++;
11677c478bd9Sstevel@tonic-gate 	mutex_exit(&vmp->vm_lock);
11687c478bd9Sstevel@tonic-gate 	if (vmflag & VM_PANIC)
11697c478bd9Sstevel@tonic-gate 		panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): "
11707c478bd9Sstevel@tonic-gate 		    "cannot satisfy mandatory allocation",
11717c478bd9Sstevel@tonic-gate 		    (void *)vmp, size, align_arg, phase, nocross,
11727c478bd9Sstevel@tonic-gate 		    minaddr, maxaddr, vmflag);
11737c478bd9Sstevel@tonic-gate 	ASSERT(xvaddr == NULL);
11747c478bd9Sstevel@tonic-gate 	return (NULL);
11757c478bd9Sstevel@tonic-gate }
11767c478bd9Sstevel@tonic-gate 
11777c478bd9Sstevel@tonic-gate /*
11787c478bd9Sstevel@tonic-gate  * Free the segment [vaddr, vaddr + size), where vaddr was a constrained
11797c478bd9Sstevel@tonic-gate  * allocation.  vmem_xalloc() and vmem_xfree() must always be paired because
11807c478bd9Sstevel@tonic-gate  * both routines bypass the quantum caches.
11817c478bd9Sstevel@tonic-gate  */
11827c478bd9Sstevel@tonic-gate void
11837c478bd9Sstevel@tonic-gate vmem_xfree(vmem_t *vmp, void *vaddr, size_t size)
11847c478bd9Sstevel@tonic-gate {
11857c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp, *vnext, *vprev;
11867c478bd9Sstevel@tonic-gate 
11877c478bd9Sstevel@tonic-gate 	mutex_enter(&vmp->vm_lock);
11887c478bd9Sstevel@tonic-gate 
11897c478bd9Sstevel@tonic-gate 	vsp = vmem_hash_delete(vmp, (uintptr_t)vaddr, size);
11907c478bd9Sstevel@tonic-gate 	vsp->vs_end = P2ROUNDUP(vsp->vs_end, vmp->vm_quantum);
11917c478bd9Sstevel@tonic-gate 
11927c478bd9Sstevel@tonic-gate 	/*
11937c478bd9Sstevel@tonic-gate 	 * Attempt to coalesce with the next segment.
11947c478bd9Sstevel@tonic-gate 	 */
11957c478bd9Sstevel@tonic-gate 	vnext = vsp->vs_anext;
11967c478bd9Sstevel@tonic-gate 	if (vnext->vs_type == VMEM_FREE) {
11977c478bd9Sstevel@tonic-gate 		ASSERT(vsp->vs_end == vnext->vs_start);
11987c478bd9Sstevel@tonic-gate 		vmem_freelist_delete(vmp, vnext);
11997c478bd9Sstevel@tonic-gate 		vsp->vs_end = vnext->vs_end;
12007c478bd9Sstevel@tonic-gate 		vmem_seg_destroy(vmp, vnext);
12017c478bd9Sstevel@tonic-gate 	}
12027c478bd9Sstevel@tonic-gate 
12037c478bd9Sstevel@tonic-gate 	/*
12047c478bd9Sstevel@tonic-gate 	 * Attempt to coalesce with the previous segment.
12057c478bd9Sstevel@tonic-gate 	 */
12067c478bd9Sstevel@tonic-gate 	vprev = vsp->vs_aprev;
12077c478bd9Sstevel@tonic-gate 	if (vprev->vs_type == VMEM_FREE) {
12087c478bd9Sstevel@tonic-gate 		ASSERT(vprev->vs_end == vsp->vs_start);
12097c478bd9Sstevel@tonic-gate 		vmem_freelist_delete(vmp, vprev);
12107c478bd9Sstevel@tonic-gate 		vprev->vs_end = vsp->vs_end;
12117c478bd9Sstevel@tonic-gate 		vmem_seg_destroy(vmp, vsp);
12127c478bd9Sstevel@tonic-gate 		vsp = vprev;
12137c478bd9Sstevel@tonic-gate 	}
12147c478bd9Sstevel@tonic-gate 
12157c478bd9Sstevel@tonic-gate 	/*
12167c478bd9Sstevel@tonic-gate 	 * If the entire span is free, return it to the source.
12177c478bd9Sstevel@tonic-gate 	 */
12187c478bd9Sstevel@tonic-gate 	if (vsp->vs_aprev->vs_import && vmp->vm_source_free != NULL &&
12197c478bd9Sstevel@tonic-gate 	    vsp->vs_aprev->vs_type == VMEM_SPAN &&
12207c478bd9Sstevel@tonic-gate 	    vsp->vs_anext->vs_type == VMEM_SPAN) {
12217c478bd9Sstevel@tonic-gate 		vaddr = (void *)vsp->vs_start;
12227c478bd9Sstevel@tonic-gate 		size = VS_SIZE(vsp);
12237c478bd9Sstevel@tonic-gate 		ASSERT(size == VS_SIZE(vsp->vs_aprev));
12247c478bd9Sstevel@tonic-gate 		vmem_span_destroy(vmp, vsp);
12257c478bd9Sstevel@tonic-gate 		mutex_exit(&vmp->vm_lock);
12267c478bd9Sstevel@tonic-gate 		vmp->vm_source_free(vmp->vm_source, vaddr, size);
12277c478bd9Sstevel@tonic-gate 	} else {
12287c478bd9Sstevel@tonic-gate 		vmem_freelist_insert(vmp, vsp);
12297c478bd9Sstevel@tonic-gate 		mutex_exit(&vmp->vm_lock);
12307c478bd9Sstevel@tonic-gate 	}
12317c478bd9Sstevel@tonic-gate }
12327c478bd9Sstevel@tonic-gate 
12337c478bd9Sstevel@tonic-gate /*
12347c478bd9Sstevel@tonic-gate  * Allocate size bytes from arena vmp.  Returns the allocated address
12357c478bd9Sstevel@tonic-gate  * on success, NULL on failure.  vmflag specifies VM_SLEEP or VM_NOSLEEP,
12367c478bd9Sstevel@tonic-gate  * and may also specify best-fit, first-fit, or next-fit allocation policy
12377c478bd9Sstevel@tonic-gate  * instead of the default instant-fit policy.  VM_SLEEP allocations are
12387c478bd9Sstevel@tonic-gate  * guaranteed to succeed.
12397c478bd9Sstevel@tonic-gate  */
12407c478bd9Sstevel@tonic-gate void *
12417c478bd9Sstevel@tonic-gate vmem_alloc(vmem_t *vmp, size_t size, int vmflag)
12427c478bd9Sstevel@tonic-gate {
12437c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp;
12447c478bd9Sstevel@tonic-gate 	uintptr_t addr;
12457c478bd9Sstevel@tonic-gate 	int hb;
12467c478bd9Sstevel@tonic-gate 	int flist = 0;
12477c478bd9Sstevel@tonic-gate 	uint32_t mtbf;
12487c478bd9Sstevel@tonic-gate 
12497c478bd9Sstevel@tonic-gate 	if (size - 1 < vmp->vm_qcache_max)
12507c478bd9Sstevel@tonic-gate 		return (kmem_cache_alloc(vmp->vm_qcache[(size - 1) >>
12517c478bd9Sstevel@tonic-gate 		    vmp->vm_qshift], vmflag & VM_KMFLAGS));
12527c478bd9Sstevel@tonic-gate 
12537c478bd9Sstevel@tonic-gate 	if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 &&
12547c478bd9Sstevel@tonic-gate 	    (vmflag & (VM_NOSLEEP | VM_PANIC)) == VM_NOSLEEP)
12557c478bd9Sstevel@tonic-gate 		return (NULL);
12567c478bd9Sstevel@tonic-gate 
12577c478bd9Sstevel@tonic-gate 	if (vmflag & VM_NEXTFIT)
12587c478bd9Sstevel@tonic-gate 		return (vmem_nextfit_alloc(vmp, size, vmflag));
12597c478bd9Sstevel@tonic-gate 
12607c478bd9Sstevel@tonic-gate 	if (vmflag & (VM_BESTFIT | VM_FIRSTFIT))
12617c478bd9Sstevel@tonic-gate 		return (vmem_xalloc(vmp, size, vmp->vm_quantum, 0, 0,
12627c478bd9Sstevel@tonic-gate 		    NULL, NULL, vmflag));
12637c478bd9Sstevel@tonic-gate 
12647c478bd9Sstevel@tonic-gate 	/*
12657c478bd9Sstevel@tonic-gate 	 * Unconstrained instant-fit allocation from the segment list.
12667c478bd9Sstevel@tonic-gate 	 */
12677c478bd9Sstevel@tonic-gate 	mutex_enter(&vmp->vm_lock);
12687c478bd9Sstevel@tonic-gate 
12697c478bd9Sstevel@tonic-gate 	if (vmp->vm_nsegfree >= VMEM_MINFREE || vmem_populate(vmp, vmflag)) {
12707c478bd9Sstevel@tonic-gate 		if ((size & (size - 1)) == 0)
12717c478bd9Sstevel@tonic-gate 			flist = lowbit(P2ALIGN(vmp->vm_freemap, size));
12727c478bd9Sstevel@tonic-gate 		else if ((hb = highbit(size)) < VMEM_FREELISTS)
12737c478bd9Sstevel@tonic-gate 			flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb));
12747c478bd9Sstevel@tonic-gate 	}
12757c478bd9Sstevel@tonic-gate 
12767c478bd9Sstevel@tonic-gate 	if (flist-- == 0) {
12777c478bd9Sstevel@tonic-gate 		mutex_exit(&vmp->vm_lock);
12787c478bd9Sstevel@tonic-gate 		return (vmem_xalloc(vmp, size, vmp->vm_quantum,
12797c478bd9Sstevel@tonic-gate 		    0, 0, NULL, NULL, vmflag));
12807c478bd9Sstevel@tonic-gate 	}
12817c478bd9Sstevel@tonic-gate 
12827c478bd9Sstevel@tonic-gate 	ASSERT(size <= (1UL << flist));
12837c478bd9Sstevel@tonic-gate 	vsp = vmp->vm_freelist[flist].vs_knext;
12847c478bd9Sstevel@tonic-gate 	addr = vsp->vs_start;
12857c478bd9Sstevel@tonic-gate 	(void) vmem_seg_alloc(vmp, vsp, addr, size);
12867c478bd9Sstevel@tonic-gate 	mutex_exit(&vmp->vm_lock);
12877c478bd9Sstevel@tonic-gate 	return ((void *)addr);
12887c478bd9Sstevel@tonic-gate }
12897c478bd9Sstevel@tonic-gate 
12907c478bd9Sstevel@tonic-gate /*
12917c478bd9Sstevel@tonic-gate  * Free the segment [vaddr, vaddr + size).
12927c478bd9Sstevel@tonic-gate  */
12937c478bd9Sstevel@tonic-gate void
12947c478bd9Sstevel@tonic-gate vmem_free(vmem_t *vmp, void *vaddr, size_t size)
12957c478bd9Sstevel@tonic-gate {
12967c478bd9Sstevel@tonic-gate 	if (size - 1 < vmp->vm_qcache_max)
12977c478bd9Sstevel@tonic-gate 		kmem_cache_free(vmp->vm_qcache[(size - 1) >> vmp->vm_qshift],
12987c478bd9Sstevel@tonic-gate 		    vaddr);
12997c478bd9Sstevel@tonic-gate 	else
13007c478bd9Sstevel@tonic-gate 		vmem_xfree(vmp, vaddr, size);
13017c478bd9Sstevel@tonic-gate }
13027c478bd9Sstevel@tonic-gate 
13037c478bd9Sstevel@tonic-gate /*
13047c478bd9Sstevel@tonic-gate  * Determine whether arena vmp contains the segment [vaddr, vaddr + size).
13057c478bd9Sstevel@tonic-gate  */
13067c478bd9Sstevel@tonic-gate int
13077c478bd9Sstevel@tonic-gate vmem_contains(vmem_t *vmp, void *vaddr, size_t size)
13087c478bd9Sstevel@tonic-gate {
13097c478bd9Sstevel@tonic-gate 	uintptr_t start = (uintptr_t)vaddr;
13107c478bd9Sstevel@tonic-gate 	uintptr_t end = start + size;
13117c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp;
13127c478bd9Sstevel@tonic-gate 	vmem_seg_t *seg0 = &vmp->vm_seg0;
13137c478bd9Sstevel@tonic-gate 
13147c478bd9Sstevel@tonic-gate 	mutex_enter(&vmp->vm_lock);
13157c478bd9Sstevel@tonic-gate 	vmp->vm_kstat.vk_contains.value.ui64++;
13167c478bd9Sstevel@tonic-gate 	for (vsp = seg0->vs_knext; vsp != seg0; vsp = vsp->vs_knext) {
13177c478bd9Sstevel@tonic-gate 		vmp->vm_kstat.vk_contains_search.value.ui64++;
13187c478bd9Sstevel@tonic-gate 		ASSERT(vsp->vs_type == VMEM_SPAN);
13197c478bd9Sstevel@tonic-gate 		if (start >= vsp->vs_start && end - 1 <= vsp->vs_end - 1)
13207c478bd9Sstevel@tonic-gate 			break;
13217c478bd9Sstevel@tonic-gate 	}
13227c478bd9Sstevel@tonic-gate 	mutex_exit(&vmp->vm_lock);
13237c478bd9Sstevel@tonic-gate 	return (vsp != seg0);
13247c478bd9Sstevel@tonic-gate }
13257c478bd9Sstevel@tonic-gate 
13267c478bd9Sstevel@tonic-gate /*
13277c478bd9Sstevel@tonic-gate  * Add the span [vaddr, vaddr + size) to arena vmp.
13287c478bd9Sstevel@tonic-gate  */
13297c478bd9Sstevel@tonic-gate void *
13307c478bd9Sstevel@tonic-gate vmem_add(vmem_t *vmp, void *vaddr, size_t size, int vmflag)
13317c478bd9Sstevel@tonic-gate {
13327c478bd9Sstevel@tonic-gate 	if (vaddr == NULL || size == 0)
1333*903a11ebSrh87107 		panic("vmem_add(%p, %p, %lu): bad arguments",
1334*903a11ebSrh87107 		    (void *)vmp, vaddr, size);
13357c478bd9Sstevel@tonic-gate 
13367c478bd9Sstevel@tonic-gate 	ASSERT(!vmem_contains(vmp, vaddr, size));
13377c478bd9Sstevel@tonic-gate 
13387c478bd9Sstevel@tonic-gate 	mutex_enter(&vmp->vm_lock);
13397c478bd9Sstevel@tonic-gate 	if (vmem_populate(vmp, vmflag))
13407c478bd9Sstevel@tonic-gate 		(void) vmem_span_create(vmp, vaddr, size, 0);
13417c478bd9Sstevel@tonic-gate 	else
13427c478bd9Sstevel@tonic-gate 		vaddr = NULL;
13437c478bd9Sstevel@tonic-gate 	mutex_exit(&vmp->vm_lock);
13447c478bd9Sstevel@tonic-gate 	return (vaddr);
13457c478bd9Sstevel@tonic-gate }
13467c478bd9Sstevel@tonic-gate 
13477c478bd9Sstevel@tonic-gate /*
13487c478bd9Sstevel@tonic-gate  * Walk the vmp arena, applying func to each segment matching typemask.
13497c478bd9Sstevel@tonic-gate  * If VMEM_REENTRANT is specified, the arena lock is dropped across each
13507c478bd9Sstevel@tonic-gate  * call to func(); otherwise, it is held for the duration of vmem_walk()
13517c478bd9Sstevel@tonic-gate  * to ensure a consistent snapshot.  Note that VMEM_REENTRANT callbacks
13527c478bd9Sstevel@tonic-gate  * are *not* necessarily consistent, so they may only be used when a hint
13537c478bd9Sstevel@tonic-gate  * is adequate.
13547c478bd9Sstevel@tonic-gate  */
13557c478bd9Sstevel@tonic-gate void
13567c478bd9Sstevel@tonic-gate vmem_walk(vmem_t *vmp, int typemask,
13577c478bd9Sstevel@tonic-gate 	void (*func)(void *, void *, size_t), void *arg)
13587c478bd9Sstevel@tonic-gate {
13597c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp;
13607c478bd9Sstevel@tonic-gate 	vmem_seg_t *seg0 = &vmp->vm_seg0;
13617c478bd9Sstevel@tonic-gate 	vmem_seg_t walker;
13627c478bd9Sstevel@tonic-gate 
13637c478bd9Sstevel@tonic-gate 	if (typemask & VMEM_WALKER)
13647c478bd9Sstevel@tonic-gate 		return;
13657c478bd9Sstevel@tonic-gate 
13667c478bd9Sstevel@tonic-gate 	bzero(&walker, sizeof (walker));
13677c478bd9Sstevel@tonic-gate 	walker.vs_type = VMEM_WALKER;
13687c478bd9Sstevel@tonic-gate 
13697c478bd9Sstevel@tonic-gate 	mutex_enter(&vmp->vm_lock);
13707c478bd9Sstevel@tonic-gate 	VMEM_INSERT(seg0, &walker, a);
13717c478bd9Sstevel@tonic-gate 	for (vsp = seg0->vs_anext; vsp != seg0; vsp = vsp->vs_anext) {
13727c478bd9Sstevel@tonic-gate 		if (vsp->vs_type & typemask) {
13737c478bd9Sstevel@tonic-gate 			void *start = (void *)vsp->vs_start;
13747c478bd9Sstevel@tonic-gate 			size_t size = VS_SIZE(vsp);
13757c478bd9Sstevel@tonic-gate 			if (typemask & VMEM_REENTRANT) {
13767c478bd9Sstevel@tonic-gate 				vmem_advance(vmp, &walker, vsp);
13777c478bd9Sstevel@tonic-gate 				mutex_exit(&vmp->vm_lock);
13787c478bd9Sstevel@tonic-gate 				func(arg, start, size);
13797c478bd9Sstevel@tonic-gate 				mutex_enter(&vmp->vm_lock);
13807c478bd9Sstevel@tonic-gate 				vsp = &walker;
13817c478bd9Sstevel@tonic-gate 			} else {
13827c478bd9Sstevel@tonic-gate 				func(arg, start, size);
13837c478bd9Sstevel@tonic-gate 			}
13847c478bd9Sstevel@tonic-gate 		}
13857c478bd9Sstevel@tonic-gate 	}
13867c478bd9Sstevel@tonic-gate 	vmem_advance(vmp, &walker, NULL);
13877c478bd9Sstevel@tonic-gate 	mutex_exit(&vmp->vm_lock);
13887c478bd9Sstevel@tonic-gate }
13897c478bd9Sstevel@tonic-gate 
13907c478bd9Sstevel@tonic-gate /*
13917c478bd9Sstevel@tonic-gate  * Return the total amount of memory whose type matches typemask.  Thus:
13927c478bd9Sstevel@tonic-gate  *
13937c478bd9Sstevel@tonic-gate  *	typemask VMEM_ALLOC yields total memory allocated (in use).
13947c478bd9Sstevel@tonic-gate  *	typemask VMEM_FREE yields total memory free (available).
13957c478bd9Sstevel@tonic-gate  *	typemask (VMEM_ALLOC | VMEM_FREE) yields total arena size.
13967c478bd9Sstevel@tonic-gate  */
13977c478bd9Sstevel@tonic-gate size_t
13987c478bd9Sstevel@tonic-gate vmem_size(vmem_t *vmp, int typemask)
13997c478bd9Sstevel@tonic-gate {
14007c478bd9Sstevel@tonic-gate 	uint64_t size = 0;
14017c478bd9Sstevel@tonic-gate 
14027c478bd9Sstevel@tonic-gate 	if (typemask & VMEM_ALLOC)
14037c478bd9Sstevel@tonic-gate 		size += vmp->vm_kstat.vk_mem_inuse.value.ui64;
14047c478bd9Sstevel@tonic-gate 	if (typemask & VMEM_FREE)
14057c478bd9Sstevel@tonic-gate 		size += vmp->vm_kstat.vk_mem_total.value.ui64 -
14067c478bd9Sstevel@tonic-gate 		    vmp->vm_kstat.vk_mem_inuse.value.ui64;
14077c478bd9Sstevel@tonic-gate 	return ((size_t)size);
14087c478bd9Sstevel@tonic-gate }
14097c478bd9Sstevel@tonic-gate 
14107c478bd9Sstevel@tonic-gate /*
14117c478bd9Sstevel@tonic-gate  * Create an arena called name whose initial span is [base, base + size).
14127c478bd9Sstevel@tonic-gate  * The arena's natural unit of currency is quantum, so vmem_alloc()
14137c478bd9Sstevel@tonic-gate  * guarantees quantum-aligned results.  The arena may import new spans
14147c478bd9Sstevel@tonic-gate  * by invoking afunc() on source, and may return those spans by invoking
14157c478bd9Sstevel@tonic-gate  * ffunc() on source.  To make small allocations fast and scalable,
14167c478bd9Sstevel@tonic-gate  * the arena offers high-performance caching for each integer multiple
14177c478bd9Sstevel@tonic-gate  * of quantum up to qcache_max.
14187c478bd9Sstevel@tonic-gate  */
14197c478bd9Sstevel@tonic-gate static vmem_t *
14207c478bd9Sstevel@tonic-gate vmem_create_common(const char *name, void *base, size_t size, size_t quantum,
14217c478bd9Sstevel@tonic-gate 	void *(*afunc)(vmem_t *, size_t, int),
14227c478bd9Sstevel@tonic-gate 	void (*ffunc)(vmem_t *, void *, size_t),
14237c478bd9Sstevel@tonic-gate 	vmem_t *source, size_t qcache_max, int vmflag)
14247c478bd9Sstevel@tonic-gate {
14257c478bd9Sstevel@tonic-gate 	int i;
14267c478bd9Sstevel@tonic-gate 	size_t nqcache;
14277c478bd9Sstevel@tonic-gate 	vmem_t *vmp, *cur, **vmpp;
14287c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp;
14297c478bd9Sstevel@tonic-gate 	vmem_freelist_t *vfp;
14307c478bd9Sstevel@tonic-gate 	uint32_t id = atomic_add_32_nv(&vmem_id, 1);
14317c478bd9Sstevel@tonic-gate 
14327c478bd9Sstevel@tonic-gate 	if (vmem_vmem_arena != NULL) {
14337c478bd9Sstevel@tonic-gate 		vmp = vmem_alloc(vmem_vmem_arena, sizeof (vmem_t),
14347c478bd9Sstevel@tonic-gate 		    vmflag & VM_KMFLAGS);
14357c478bd9Sstevel@tonic-gate 	} else {
14367c478bd9Sstevel@tonic-gate 		ASSERT(id <= VMEM_INITIAL);
14377c478bd9Sstevel@tonic-gate 		vmp = &vmem0[id - 1];
14387c478bd9Sstevel@tonic-gate 	}
14397c478bd9Sstevel@tonic-gate 
14407c478bd9Sstevel@tonic-gate 	/* An identifier arena must inherit from another identifier arena */
14417c478bd9Sstevel@tonic-gate 	ASSERT(source == NULL || ((source->vm_cflags & VMC_IDENTIFIER) ==
14427c478bd9Sstevel@tonic-gate 	    (vmflag & VMC_IDENTIFIER)));
14437c478bd9Sstevel@tonic-gate 
14447c478bd9Sstevel@tonic-gate 	if (vmp == NULL)
14457c478bd9Sstevel@tonic-gate 		return (NULL);
14467c478bd9Sstevel@tonic-gate 	bzero(vmp, sizeof (vmem_t));
14477c478bd9Sstevel@tonic-gate 
14487c478bd9Sstevel@tonic-gate 	(void) snprintf(vmp->vm_name, VMEM_NAMELEN, "%s", name);
14497c478bd9Sstevel@tonic-gate 	mutex_init(&vmp->vm_lock, NULL, MUTEX_DEFAULT, NULL);
14507c478bd9Sstevel@tonic-gate 	cv_init(&vmp->vm_cv, NULL, CV_DEFAULT, NULL);
14517c478bd9Sstevel@tonic-gate 	vmp->vm_cflags = vmflag;
14527c478bd9Sstevel@tonic-gate 	vmflag &= VM_KMFLAGS;
14537c478bd9Sstevel@tonic-gate 
14547c478bd9Sstevel@tonic-gate 	vmp->vm_quantum = quantum;
14557c478bd9Sstevel@tonic-gate 	vmp->vm_qshift = highbit(quantum) - 1;
14567c478bd9Sstevel@tonic-gate 	nqcache = MIN(qcache_max >> vmp->vm_qshift, VMEM_NQCACHE_MAX);
14577c478bd9Sstevel@tonic-gate 
14587c478bd9Sstevel@tonic-gate 	for (i = 0; i <= VMEM_FREELISTS; i++) {
14597c478bd9Sstevel@tonic-gate 		vfp = &vmp->vm_freelist[i];
14607c478bd9Sstevel@tonic-gate 		vfp->vs_end = 1UL << i;
14617c478bd9Sstevel@tonic-gate 		vfp->vs_knext = (vmem_seg_t *)(vfp + 1);
14627c478bd9Sstevel@tonic-gate 		vfp->vs_kprev = (vmem_seg_t *)(vfp - 1);
14637c478bd9Sstevel@tonic-gate 	}
14647c478bd9Sstevel@tonic-gate 
14657c478bd9Sstevel@tonic-gate 	vmp->vm_freelist[0].vs_kprev = NULL;
14667c478bd9Sstevel@tonic-gate 	vmp->vm_freelist[VMEM_FREELISTS].vs_knext = NULL;
14677c478bd9Sstevel@tonic-gate 	vmp->vm_freelist[VMEM_FREELISTS].vs_end = 0;
14687c478bd9Sstevel@tonic-gate 	vmp->vm_hash_table = vmp->vm_hash0;
14697c478bd9Sstevel@tonic-gate 	vmp->vm_hash_mask = VMEM_HASH_INITIAL - 1;
14707c478bd9Sstevel@tonic-gate 	vmp->vm_hash_shift = highbit(vmp->vm_hash_mask);
14717c478bd9Sstevel@tonic-gate 
14727c478bd9Sstevel@tonic-gate 	vsp = &vmp->vm_seg0;
14737c478bd9Sstevel@tonic-gate 	vsp->vs_anext = vsp;
14747c478bd9Sstevel@tonic-gate 	vsp->vs_aprev = vsp;
14757c478bd9Sstevel@tonic-gate 	vsp->vs_knext = vsp;
14767c478bd9Sstevel@tonic-gate 	vsp->vs_kprev = vsp;
14777c478bd9Sstevel@tonic-gate 	vsp->vs_type = VMEM_SPAN;
14787c478bd9Sstevel@tonic-gate 
14797c478bd9Sstevel@tonic-gate 	vsp = &vmp->vm_rotor;
14807c478bd9Sstevel@tonic-gate 	vsp->vs_type = VMEM_ROTOR;
14817c478bd9Sstevel@tonic-gate 	VMEM_INSERT(&vmp->vm_seg0, vsp, a);
14827c478bd9Sstevel@tonic-gate 
14837c478bd9Sstevel@tonic-gate 	bcopy(&vmem_kstat_template, &vmp->vm_kstat, sizeof (vmem_kstat_t));
14847c478bd9Sstevel@tonic-gate 
14857c478bd9Sstevel@tonic-gate 	vmp->vm_id = id;
14867c478bd9Sstevel@tonic-gate 	if (source != NULL)
14877c478bd9Sstevel@tonic-gate 		vmp->vm_kstat.vk_source_id.value.ui32 = source->vm_id;
14887c478bd9Sstevel@tonic-gate 	vmp->vm_source = source;
14897c478bd9Sstevel@tonic-gate 	vmp->vm_source_alloc = afunc;
14907c478bd9Sstevel@tonic-gate 	vmp->vm_source_free = ffunc;
14917c478bd9Sstevel@tonic-gate 
14927c478bd9Sstevel@tonic-gate 	/*
14937c478bd9Sstevel@tonic-gate 	 * Some arenas (like vmem_metadata and kmem_metadata) cannot
14947c478bd9Sstevel@tonic-gate 	 * use quantum caching to lower fragmentation.  Instead, we
14957c478bd9Sstevel@tonic-gate 	 * increase their imports, giving a similar effect.
14967c478bd9Sstevel@tonic-gate 	 */
14977c478bd9Sstevel@tonic-gate 	if (vmp->vm_cflags & VMC_NO_QCACHE) {
14987c478bd9Sstevel@tonic-gate 		vmp->vm_min_import =
14997c478bd9Sstevel@tonic-gate 		    VMEM_QCACHE_SLABSIZE(nqcache << vmp->vm_qshift);
15007c478bd9Sstevel@tonic-gate 		nqcache = 0;
15017c478bd9Sstevel@tonic-gate 	}
15027c478bd9Sstevel@tonic-gate 
15037c478bd9Sstevel@tonic-gate 	if (nqcache != 0) {
15047c478bd9Sstevel@tonic-gate 		ASSERT(!(vmflag & VM_NOSLEEP));
15057c478bd9Sstevel@tonic-gate 		vmp->vm_qcache_max = nqcache << vmp->vm_qshift;
15067c478bd9Sstevel@tonic-gate 		for (i = 0; i < nqcache; i++) {
15077c478bd9Sstevel@tonic-gate 			char buf[VMEM_NAMELEN + 21];
15087c478bd9Sstevel@tonic-gate 			(void) sprintf(buf, "%s_%lu", vmp->vm_name,
15097c478bd9Sstevel@tonic-gate 			    (i + 1) * quantum);
15107c478bd9Sstevel@tonic-gate 			vmp->vm_qcache[i] = kmem_cache_create(buf,
15117c478bd9Sstevel@tonic-gate 			    (i + 1) * quantum, quantum, NULL, NULL, NULL,
15127c478bd9Sstevel@tonic-gate 			    NULL, vmp, KMC_QCACHE | KMC_NOTOUCH);
15137c478bd9Sstevel@tonic-gate 		}
15147c478bd9Sstevel@tonic-gate 	}
15157c478bd9Sstevel@tonic-gate 
15167c478bd9Sstevel@tonic-gate 	if ((vmp->vm_ksp = kstat_create("vmem", vmp->vm_id, vmp->vm_name,
15177c478bd9Sstevel@tonic-gate 	    "vmem", KSTAT_TYPE_NAMED, sizeof (vmem_kstat_t) /
15187c478bd9Sstevel@tonic-gate 	    sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL)) != NULL) {
15197c478bd9Sstevel@tonic-gate 		vmp->vm_ksp->ks_data = &vmp->vm_kstat;
15207c478bd9Sstevel@tonic-gate 		kstat_install(vmp->vm_ksp);
15217c478bd9Sstevel@tonic-gate 	}
15227c478bd9Sstevel@tonic-gate 
15237c478bd9Sstevel@tonic-gate 	mutex_enter(&vmem_list_lock);
15247c478bd9Sstevel@tonic-gate 	vmpp = &vmem_list;
15257c478bd9Sstevel@tonic-gate 	while ((cur = *vmpp) != NULL)
15267c478bd9Sstevel@tonic-gate 		vmpp = &cur->vm_next;
15277c478bd9Sstevel@tonic-gate 	*vmpp = vmp;
15287c478bd9Sstevel@tonic-gate 	mutex_exit(&vmem_list_lock);
15297c478bd9Sstevel@tonic-gate 
15307c478bd9Sstevel@tonic-gate 	if (vmp->vm_cflags & VMC_POPULATOR) {
15317c478bd9Sstevel@tonic-gate 		ASSERT(vmem_populators < VMEM_INITIAL);
15327c478bd9Sstevel@tonic-gate 		vmem_populator[atomic_add_32_nv(&vmem_populators, 1) - 1] = vmp;
15337c478bd9Sstevel@tonic-gate 		mutex_enter(&vmp->vm_lock);
15347c478bd9Sstevel@tonic-gate 		(void) vmem_populate(vmp, vmflag | VM_PANIC);
15357c478bd9Sstevel@tonic-gate 		mutex_exit(&vmp->vm_lock);
15367c478bd9Sstevel@tonic-gate 	}
15377c478bd9Sstevel@tonic-gate 
15387c478bd9Sstevel@tonic-gate 	if ((base || size) && vmem_add(vmp, base, size, vmflag) == NULL) {
15397c478bd9Sstevel@tonic-gate 		vmem_destroy(vmp);
15407c478bd9Sstevel@tonic-gate 		return (NULL);
15417c478bd9Sstevel@tonic-gate 	}
15427c478bd9Sstevel@tonic-gate 
15437c478bd9Sstevel@tonic-gate 	return (vmp);
15447c478bd9Sstevel@tonic-gate }
15457c478bd9Sstevel@tonic-gate 
15467c478bd9Sstevel@tonic-gate vmem_t *
15477c478bd9Sstevel@tonic-gate vmem_xcreate(const char *name, void *base, size_t size, size_t quantum,
15487c478bd9Sstevel@tonic-gate     vmem_ximport_t *afunc, vmem_free_t *ffunc, vmem_t *source,
15497c478bd9Sstevel@tonic-gate     size_t qcache_max, int vmflag)
15507c478bd9Sstevel@tonic-gate {
15517c478bd9Sstevel@tonic-gate 	ASSERT(!(vmflag & (VMC_POPULATOR | VMC_XALLOC)));
15527c478bd9Sstevel@tonic-gate 	vmflag &= ~(VMC_POPULATOR | VMC_XALLOC);
15537c478bd9Sstevel@tonic-gate 
15547c478bd9Sstevel@tonic-gate 	return (vmem_create_common(name, base, size, quantum,
15557c478bd9Sstevel@tonic-gate 	    (vmem_alloc_t *)afunc, ffunc, source, qcache_max,
15567c478bd9Sstevel@tonic-gate 	    vmflag | VMC_XALLOC));
15577c478bd9Sstevel@tonic-gate }
15587c478bd9Sstevel@tonic-gate 
15597c478bd9Sstevel@tonic-gate vmem_t *
15607c478bd9Sstevel@tonic-gate vmem_create(const char *name, void *base, size_t size, size_t quantum,
15617c478bd9Sstevel@tonic-gate     vmem_alloc_t *afunc, vmem_free_t *ffunc, vmem_t *source,
15627c478bd9Sstevel@tonic-gate     size_t qcache_max, int vmflag)
15637c478bd9Sstevel@tonic-gate {
1564aaa10e67Sha137994 	ASSERT(!(vmflag & (VMC_XALLOC | VMC_XALIGN)));
1565aaa10e67Sha137994 	vmflag &= ~(VMC_XALLOC | VMC_XALIGN);
15667c478bd9Sstevel@tonic-gate 
15677c478bd9Sstevel@tonic-gate 	return (vmem_create_common(name, base, size, quantum,
15687c478bd9Sstevel@tonic-gate 	    afunc, ffunc, source, qcache_max, vmflag));
15697c478bd9Sstevel@tonic-gate }
15707c478bd9Sstevel@tonic-gate 
15717c478bd9Sstevel@tonic-gate /*
15727c478bd9Sstevel@tonic-gate  * Destroy arena vmp.
15737c478bd9Sstevel@tonic-gate  */
15747c478bd9Sstevel@tonic-gate void
15757c478bd9Sstevel@tonic-gate vmem_destroy(vmem_t *vmp)
15767c478bd9Sstevel@tonic-gate {
15777c478bd9Sstevel@tonic-gate 	vmem_t *cur, **vmpp;
15787c478bd9Sstevel@tonic-gate 	vmem_seg_t *seg0 = &vmp->vm_seg0;
15797c478bd9Sstevel@tonic-gate 	vmem_seg_t *vsp;
15807c478bd9Sstevel@tonic-gate 	size_t leaked;
15817c478bd9Sstevel@tonic-gate 	int i;
15827c478bd9Sstevel@tonic-gate 
15837c478bd9Sstevel@tonic-gate 	mutex_enter(&vmem_list_lock);
15847c478bd9Sstevel@tonic-gate 	vmpp = &vmem_list;
15857c478bd9Sstevel@tonic-gate 	while ((cur = *vmpp) != vmp)
15867c478bd9Sstevel@tonic-gate 		vmpp = &cur->vm_next;
15877c478bd9Sstevel@tonic-gate 	*vmpp = vmp->vm_next;
15887c478bd9Sstevel@tonic-gate 	mutex_exit(&vmem_list_lock);
15897c478bd9Sstevel@tonic-gate 
15907c478bd9Sstevel@tonic-gate 	for (i = 0; i < VMEM_NQCACHE_MAX; i++)
15917c478bd9Sstevel@tonic-gate 		if (vmp->vm_qcache[i])
15927c478bd9Sstevel@tonic-gate 			kmem_cache_destroy(vmp->vm_qcache[i]);
15937c478bd9Sstevel@tonic-gate 
15947c478bd9Sstevel@tonic-gate 	leaked = vmem_size(vmp, VMEM_ALLOC);
15957c478bd9Sstevel@tonic-gate 	if (leaked != 0)
15967c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "vmem_destroy('%s'): leaked %lu %s",
15977c478bd9Sstevel@tonic-gate 		    vmp->vm_name, leaked, (vmp->vm_cflags & VMC_IDENTIFIER) ?
15987c478bd9Sstevel@tonic-gate 		    "identifiers" : "bytes");
15997c478bd9Sstevel@tonic-gate 
16007c478bd9Sstevel@tonic-gate 	if (vmp->vm_hash_table != vmp->vm_hash0)
16017c478bd9Sstevel@tonic-gate 		vmem_free(vmem_hash_arena, vmp->vm_hash_table,
16027c478bd9Sstevel@tonic-gate 		    (vmp->vm_hash_mask + 1) * sizeof (void *));
16037c478bd9Sstevel@tonic-gate 
16047c478bd9Sstevel@tonic-gate 	/*
16057c478bd9Sstevel@tonic-gate 	 * Give back the segment structures for anything that's left in the
16067c478bd9Sstevel@tonic-gate 	 * arena, e.g. the primary spans and their free segments.
16077c478bd9Sstevel@tonic-gate 	 */
16087c478bd9Sstevel@tonic-gate 	VMEM_DELETE(&vmp->vm_rotor, a);
16097c478bd9Sstevel@tonic-gate 	for (vsp = seg0->vs_anext; vsp != seg0; vsp = vsp->vs_anext)
16107c478bd9Sstevel@tonic-gate 		vmem_putseg_global(vsp);
16117c478bd9Sstevel@tonic-gate 
16127c478bd9Sstevel@tonic-gate 	while (vmp->vm_nsegfree > 0)
16137c478bd9Sstevel@tonic-gate 		vmem_putseg_global(vmem_getseg(vmp));
16147c478bd9Sstevel@tonic-gate 
16157c478bd9Sstevel@tonic-gate 	kstat_delete(vmp->vm_ksp);
16167c478bd9Sstevel@tonic-gate 
16177c478bd9Sstevel@tonic-gate 	mutex_destroy(&vmp->vm_lock);
16187c478bd9Sstevel@tonic-gate 	cv_destroy(&vmp->vm_cv);
16197c478bd9Sstevel@tonic-gate 	vmem_free(vmem_vmem_arena, vmp, sizeof (vmem_t));
16207c478bd9Sstevel@tonic-gate }
16217c478bd9Sstevel@tonic-gate 
16227c478bd9Sstevel@tonic-gate /*
16237c478bd9Sstevel@tonic-gate  * Resize vmp's hash table to keep the average lookup depth near 1.0.
16247c478bd9Sstevel@tonic-gate  */
16257c478bd9Sstevel@tonic-gate static void
16267c478bd9Sstevel@tonic-gate vmem_hash_rescale(vmem_t *vmp)
16277c478bd9Sstevel@tonic-gate {
16287c478bd9Sstevel@tonic-gate 	vmem_seg_t **old_table, **new_table, *vsp;
16297c478bd9Sstevel@tonic-gate 	size_t old_size, new_size, h, nseg;
16307c478bd9Sstevel@tonic-gate 
16317c478bd9Sstevel@tonic-gate 	nseg = (size_t)(vmp->vm_kstat.vk_alloc.value.ui64 -
16327c478bd9Sstevel@tonic-gate 	    vmp->vm_kstat.vk_free.value.ui64);
16337c478bd9Sstevel@tonic-gate 
16347c478bd9Sstevel@tonic-gate 	new_size = MAX(VMEM_HASH_INITIAL, 1 << (highbit(3 * nseg + 4) - 2));
16357c478bd9Sstevel@tonic-gate 	old_size = vmp->vm_hash_mask + 1;
16367c478bd9Sstevel@tonic-gate 
16377c478bd9Sstevel@tonic-gate 	if ((old_size >> 1) <= new_size && new_size <= (old_size << 1))
16387c478bd9Sstevel@tonic-gate 		return;
16397c478bd9Sstevel@tonic-gate 
16407c478bd9Sstevel@tonic-gate 	new_table = vmem_alloc(vmem_hash_arena, new_size * sizeof (void *),
16417c478bd9Sstevel@tonic-gate 	    VM_NOSLEEP);
16427c478bd9Sstevel@tonic-gate 	if (new_table == NULL)
16437c478bd9Sstevel@tonic-gate 		return;
16447c478bd9Sstevel@tonic-gate 	bzero(new_table, new_size * sizeof (void *));
16457c478bd9Sstevel@tonic-gate 
16467c478bd9Sstevel@tonic-gate 	mutex_enter(&vmp->vm_lock);
16477c478bd9Sstevel@tonic-gate 
16487c478bd9Sstevel@tonic-gate 	old_size = vmp->vm_hash_mask + 1;
16497c478bd9Sstevel@tonic-gate 	old_table = vmp->vm_hash_table;
16507c478bd9Sstevel@tonic-gate 
16517c478bd9Sstevel@tonic-gate 	vmp->vm_hash_mask = new_size - 1;
16527c478bd9Sstevel@tonic-gate 	vmp->vm_hash_table = new_table;
16537c478bd9Sstevel@tonic-gate 	vmp->vm_hash_shift = highbit(vmp->vm_hash_mask);
16547c478bd9Sstevel@tonic-gate 
16557c478bd9Sstevel@tonic-gate 	for (h = 0; h < old_size; h++) {
16567c478bd9Sstevel@tonic-gate 		vsp = old_table[h];
16577c478bd9Sstevel@tonic-gate 		while (vsp != NULL) {
16587c478bd9Sstevel@tonic-gate 			uintptr_t addr = vsp->vs_start;
16597c478bd9Sstevel@tonic-gate 			vmem_seg_t *next_vsp = vsp->vs_knext;
16607c478bd9Sstevel@tonic-gate 			vmem_seg_t **hash_bucket = VMEM_HASH(vmp, addr);
16617c478bd9Sstevel@tonic-gate 			vsp->vs_knext = *hash_bucket;
16627c478bd9Sstevel@tonic-gate 			*hash_bucket = vsp;
16637c478bd9Sstevel@tonic-gate 			vsp = next_vsp;
16647c478bd9Sstevel@tonic-gate 		}
16657c478bd9Sstevel@tonic-gate 	}
16667c478bd9Sstevel@tonic-gate 
16677c478bd9Sstevel@tonic-gate 	mutex_exit(&vmp->vm_lock);
16687c478bd9Sstevel@tonic-gate 
16697c478bd9Sstevel@tonic-gate 	if (old_table != vmp->vm_hash0)
16707c478bd9Sstevel@tonic-gate 		vmem_free(vmem_hash_arena, old_table,
16717c478bd9Sstevel@tonic-gate 		    old_size * sizeof (void *));
16727c478bd9Sstevel@tonic-gate }
16737c478bd9Sstevel@tonic-gate 
16747c478bd9Sstevel@tonic-gate /*
16757c478bd9Sstevel@tonic-gate  * Perform periodic maintenance on all vmem arenas.
16767c478bd9Sstevel@tonic-gate  */
16777c478bd9Sstevel@tonic-gate void
16787c478bd9Sstevel@tonic-gate vmem_update(void *dummy)
16797c478bd9Sstevel@tonic-gate {
16807c478bd9Sstevel@tonic-gate 	vmem_t *vmp;
16817c478bd9Sstevel@tonic-gate 
16827c478bd9Sstevel@tonic-gate 	mutex_enter(&vmem_list_lock);
16837c478bd9Sstevel@tonic-gate 	for (vmp = vmem_list; vmp != NULL; vmp = vmp->vm_next) {
16847c478bd9Sstevel@tonic-gate 		/*
16857c478bd9Sstevel@tonic-gate 		 * If threads are waiting for resources, wake them up
16867c478bd9Sstevel@tonic-gate 		 * periodically so they can issue another kmem_reap()
16877c478bd9Sstevel@tonic-gate 		 * to reclaim resources cached by the slab allocator.
16887c478bd9Sstevel@tonic-gate 		 */
16897c478bd9Sstevel@tonic-gate 		cv_broadcast(&vmp->vm_cv);
16907c478bd9Sstevel@tonic-gate 
16917c478bd9Sstevel@tonic-gate 		/*
16927c478bd9Sstevel@tonic-gate 		 * Rescale the hash table to keep the hash chains short.
16937c478bd9Sstevel@tonic-gate 		 */
16947c478bd9Sstevel@tonic-gate 		vmem_hash_rescale(vmp);
16957c478bd9Sstevel@tonic-gate 	}
16967c478bd9Sstevel@tonic-gate 	mutex_exit(&vmem_list_lock);
16977c478bd9Sstevel@tonic-gate 
16987c478bd9Sstevel@tonic-gate 	(void) timeout(vmem_update, dummy, vmem_update_interval * hz);
16997c478bd9Sstevel@tonic-gate }
17007c478bd9Sstevel@tonic-gate 
17017c478bd9Sstevel@tonic-gate /*
17027c478bd9Sstevel@tonic-gate  * Prepare vmem for use.
17037c478bd9Sstevel@tonic-gate  */
17047c478bd9Sstevel@tonic-gate vmem_t *
17057c478bd9Sstevel@tonic-gate vmem_init(const char *heap_name,
17067c478bd9Sstevel@tonic-gate 	void *heap_start, size_t heap_size, size_t heap_quantum,
17077c478bd9Sstevel@tonic-gate 	void *(*heap_alloc)(vmem_t *, size_t, int),
17087c478bd9Sstevel@tonic-gate 	void (*heap_free)(vmem_t *, void *, size_t))
17097c478bd9Sstevel@tonic-gate {
17107c478bd9Sstevel@tonic-gate 	uint32_t id;
17117c478bd9Sstevel@tonic-gate 	int nseg = VMEM_SEG_INITIAL;
17127c478bd9Sstevel@tonic-gate 	vmem_t *heap;
17137c478bd9Sstevel@tonic-gate 
17147c478bd9Sstevel@tonic-gate 	while (--nseg >= 0)
17157c478bd9Sstevel@tonic-gate 		vmem_putseg_global(&vmem_seg0[nseg]);
17167c478bd9Sstevel@tonic-gate 
17177c478bd9Sstevel@tonic-gate 	heap = vmem_create(heap_name,
17187c478bd9Sstevel@tonic-gate 	    heap_start, heap_size, heap_quantum,
17197c478bd9Sstevel@tonic-gate 	    NULL, NULL, NULL, 0,
17207c478bd9Sstevel@tonic-gate 	    VM_SLEEP | VMC_POPULATOR);
17217c478bd9Sstevel@tonic-gate 
17227c478bd9Sstevel@tonic-gate 	vmem_metadata_arena = vmem_create("vmem_metadata",
17237c478bd9Sstevel@tonic-gate 	    NULL, 0, heap_quantum,
17247c478bd9Sstevel@tonic-gate 	    vmem_alloc, vmem_free, heap, 8 * heap_quantum,
17257c478bd9Sstevel@tonic-gate 	    VM_SLEEP | VMC_POPULATOR | VMC_NO_QCACHE);
17267c478bd9Sstevel@tonic-gate 
17277c478bd9Sstevel@tonic-gate 	vmem_seg_arena = vmem_create("vmem_seg",
17287c478bd9Sstevel@tonic-gate 	    NULL, 0, heap_quantum,
17297c478bd9Sstevel@tonic-gate 	    heap_alloc, heap_free, vmem_metadata_arena, 0,
17307c478bd9Sstevel@tonic-gate 	    VM_SLEEP | VMC_POPULATOR);
17317c478bd9Sstevel@tonic-gate 
17327c478bd9Sstevel@tonic-gate 	vmem_hash_arena = vmem_create("vmem_hash",
17337c478bd9Sstevel@tonic-gate 	    NULL, 0, 8,
17347c478bd9Sstevel@tonic-gate 	    heap_alloc, heap_free, vmem_metadata_arena, 0,
17357c478bd9Sstevel@tonic-gate 	    VM_SLEEP);
17367c478bd9Sstevel@tonic-gate 
17377c478bd9Sstevel@tonic-gate 	vmem_vmem_arena = vmem_create("vmem_vmem",
17387c478bd9Sstevel@tonic-gate 	    vmem0, sizeof (vmem0), 1,
17397c478bd9Sstevel@tonic-gate 	    heap_alloc, heap_free, vmem_metadata_arena, 0,
17407c478bd9Sstevel@tonic-gate 	    VM_SLEEP);
17417c478bd9Sstevel@tonic-gate 
17427c478bd9Sstevel@tonic-gate 	for (id = 0; id < vmem_id; id++)
17437c478bd9Sstevel@tonic-gate 		(void) vmem_xalloc(vmem_vmem_arena, sizeof (vmem_t),
17447c478bd9Sstevel@tonic-gate 		    1, 0, 0, &vmem0[id], &vmem0[id + 1],
17457c478bd9Sstevel@tonic-gate 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
17467c478bd9Sstevel@tonic-gate 
17477c478bd9Sstevel@tonic-gate 	return (heap);
17487c478bd9Sstevel@tonic-gate }
1749