17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5aaa10e67Sha137994 * Common Development and Distribution License (the "License"). 6aaa10e67Sha137994 * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 2210b95589Sbonwick * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate /* 297c478bd9Sstevel@tonic-gate * Big Theory Statement for the virtual memory allocator. 307c478bd9Sstevel@tonic-gate * 317c478bd9Sstevel@tonic-gate * For a more complete description of the main ideas, see: 327c478bd9Sstevel@tonic-gate * 337c478bd9Sstevel@tonic-gate * Jeff Bonwick and Jonathan Adams, 347c478bd9Sstevel@tonic-gate * 357c478bd9Sstevel@tonic-gate * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 367c478bd9Sstevel@tonic-gate * Arbitrary Resources. 377c478bd9Sstevel@tonic-gate * 387c478bd9Sstevel@tonic-gate * Proceedings of the 2001 Usenix Conference. 397c478bd9Sstevel@tonic-gate * Available as http://www.usenix.org/event/usenix01/bonwick.html 407c478bd9Sstevel@tonic-gate * 417c478bd9Sstevel@tonic-gate * 427c478bd9Sstevel@tonic-gate * 1. General Concepts 437c478bd9Sstevel@tonic-gate * ------------------- 447c478bd9Sstevel@tonic-gate * 457c478bd9Sstevel@tonic-gate * 1.1 Overview 467c478bd9Sstevel@tonic-gate * ------------ 477c478bd9Sstevel@tonic-gate * We divide the kernel address space into a number of logically distinct 487c478bd9Sstevel@tonic-gate * pieces, or *arenas*: text, data, heap, stack, and so on. Within these 497c478bd9Sstevel@tonic-gate * arenas we often subdivide further; for example, we use heap addresses 507c478bd9Sstevel@tonic-gate * not only for the kernel heap (kmem_alloc() space), but also for DVMA, 517c478bd9Sstevel@tonic-gate * bp_mapin(), /dev/kmem, and even some device mappings like the TOD chip. 527c478bd9Sstevel@tonic-gate * The kernel address space, therefore, is most accurately described as 537c478bd9Sstevel@tonic-gate * a tree of arenas in which each node of the tree *imports* some subset 547c478bd9Sstevel@tonic-gate * of its parent. The virtual memory allocator manages these arenas and 557c478bd9Sstevel@tonic-gate * supports their natural hierarchical structure. 567c478bd9Sstevel@tonic-gate * 577c478bd9Sstevel@tonic-gate * 1.2 Arenas 587c478bd9Sstevel@tonic-gate * ---------- 597c478bd9Sstevel@tonic-gate * An arena is nothing more than a set of integers. These integers most 607c478bd9Sstevel@tonic-gate * commonly represent virtual addresses, but in fact they can represent 617c478bd9Sstevel@tonic-gate * anything at all. For example, we could use an arena containing the 627c478bd9Sstevel@tonic-gate * integers minpid through maxpid to allocate process IDs. vmem_create() 637c478bd9Sstevel@tonic-gate * and vmem_destroy() create and destroy vmem arenas. In order to 647c478bd9Sstevel@tonic-gate * differentiate between arenas used for adresses and arenas used for 657c478bd9Sstevel@tonic-gate * identifiers, the VMC_IDENTIFIER flag is passed to vmem_create(). This 667c478bd9Sstevel@tonic-gate * prevents identifier exhaustion from being diagnosed as general memory 677c478bd9Sstevel@tonic-gate * failure. 687c478bd9Sstevel@tonic-gate * 697c478bd9Sstevel@tonic-gate * 1.3 Spans 707c478bd9Sstevel@tonic-gate * --------- 717c478bd9Sstevel@tonic-gate * We represent the integers in an arena as a collection of *spans*, or 727c478bd9Sstevel@tonic-gate * contiguous ranges of integers. For example, the kernel heap consists 737c478bd9Sstevel@tonic-gate * of just one span: [kernelheap, ekernelheap). Spans can be added to an 747c478bd9Sstevel@tonic-gate * arena in two ways: explicitly, by vmem_add(), or implicitly, by 757c478bd9Sstevel@tonic-gate * importing, as described in Section 1.5 below. 767c478bd9Sstevel@tonic-gate * 777c478bd9Sstevel@tonic-gate * 1.4 Segments 787c478bd9Sstevel@tonic-gate * ------------ 797c478bd9Sstevel@tonic-gate * Spans are subdivided into *segments*, each of which is either allocated 807c478bd9Sstevel@tonic-gate * or free. A segment, like a span, is a contiguous range of integers. 817c478bd9Sstevel@tonic-gate * Each allocated segment [addr, addr + size) represents exactly one 827c478bd9Sstevel@tonic-gate * vmem_alloc(size) that returned addr. Free segments represent the space 837c478bd9Sstevel@tonic-gate * between allocated segments. If two free segments are adjacent, we 847c478bd9Sstevel@tonic-gate * coalesce them into one larger segment; that is, if segments [a, b) and 857c478bd9Sstevel@tonic-gate * [b, c) are both free, we merge them into a single segment [a, c). 867c478bd9Sstevel@tonic-gate * The segments within a span are linked together in increasing-address order 877c478bd9Sstevel@tonic-gate * so we can easily determine whether coalescing is possible. 887c478bd9Sstevel@tonic-gate * 897c478bd9Sstevel@tonic-gate * Segments never cross span boundaries. When all segments within 907c478bd9Sstevel@tonic-gate * an imported span become free, we return the span to its source. 917c478bd9Sstevel@tonic-gate * 927c478bd9Sstevel@tonic-gate * 1.5 Imported Memory 937c478bd9Sstevel@tonic-gate * ------------------- 947c478bd9Sstevel@tonic-gate * As mentioned in the overview, some arenas are logical subsets of 957c478bd9Sstevel@tonic-gate * other arenas. For example, kmem_va_arena (a virtual address cache 967c478bd9Sstevel@tonic-gate * that satisfies most kmem_slab_create() requests) is just a subset 977c478bd9Sstevel@tonic-gate * of heap_arena (the kernel heap) that provides caching for the most 987c478bd9Sstevel@tonic-gate * common slab sizes. When kmem_va_arena runs out of virtual memory, 997c478bd9Sstevel@tonic-gate * it *imports* more from the heap; we say that heap_arena is the 1007c478bd9Sstevel@tonic-gate * *vmem source* for kmem_va_arena. vmem_create() allows you to 1017c478bd9Sstevel@tonic-gate * specify any existing vmem arena as the source for your new arena. 1027c478bd9Sstevel@tonic-gate * Topologically, since every arena is a child of at most one source, 1037c478bd9Sstevel@tonic-gate * the set of all arenas forms a collection of trees. 1047c478bd9Sstevel@tonic-gate * 1057c478bd9Sstevel@tonic-gate * 1.6 Constrained Allocations 1067c478bd9Sstevel@tonic-gate * --------------------------- 1077c478bd9Sstevel@tonic-gate * Some vmem clients are quite picky about the kind of address they want. 1087c478bd9Sstevel@tonic-gate * For example, the DVMA code may need an address that is at a particular 1097c478bd9Sstevel@tonic-gate * phase with respect to some alignment (to get good cache coloring), or 1107c478bd9Sstevel@tonic-gate * that lies within certain limits (the addressable range of a device), 1117c478bd9Sstevel@tonic-gate * or that doesn't cross some boundary (a DMA counter restriction) -- 1127c478bd9Sstevel@tonic-gate * or all of the above. vmem_xalloc() allows the client to specify any 1137c478bd9Sstevel@tonic-gate * or all of these constraints. 1147c478bd9Sstevel@tonic-gate * 1157c478bd9Sstevel@tonic-gate * 1.7 The Vmem Quantum 1167c478bd9Sstevel@tonic-gate * -------------------- 1177c478bd9Sstevel@tonic-gate * Every arena has a notion of 'quantum', specified at vmem_create() time, 1187c478bd9Sstevel@tonic-gate * that defines the arena's minimum unit of currency. Most commonly the 1197c478bd9Sstevel@tonic-gate * quantum is either 1 or PAGESIZE, but any power of 2 is legal. 1207c478bd9Sstevel@tonic-gate * All vmem allocations are guaranteed to be quantum-aligned. 1217c478bd9Sstevel@tonic-gate * 1227c478bd9Sstevel@tonic-gate * 1.8 Quantum Caching 1237c478bd9Sstevel@tonic-gate * ------------------- 1247c478bd9Sstevel@tonic-gate * A vmem arena may be so hot (frequently used) that the scalability of vmem 1257c478bd9Sstevel@tonic-gate * allocation is a significant concern. We address this by allowing the most 1267c478bd9Sstevel@tonic-gate * common allocation sizes to be serviced by the kernel memory allocator, 1277c478bd9Sstevel@tonic-gate * which provides low-latency per-cpu caching. The qcache_max argument to 1287c478bd9Sstevel@tonic-gate * vmem_create() specifies the largest allocation size to cache. 1297c478bd9Sstevel@tonic-gate * 1307c478bd9Sstevel@tonic-gate * 1.9 Relationship to Kernel Memory Allocator 1317c478bd9Sstevel@tonic-gate * ------------------------------------------- 1327c478bd9Sstevel@tonic-gate * Every kmem cache has a vmem arena as its slab supplier. The kernel memory 1337c478bd9Sstevel@tonic-gate * allocator uses vmem_alloc() and vmem_free() to create and destroy slabs. 1347c478bd9Sstevel@tonic-gate * 1357c478bd9Sstevel@tonic-gate * 1367c478bd9Sstevel@tonic-gate * 2. Implementation 1377c478bd9Sstevel@tonic-gate * ----------------- 1387c478bd9Sstevel@tonic-gate * 1397c478bd9Sstevel@tonic-gate * 2.1 Segment lists and markers 1407c478bd9Sstevel@tonic-gate * ----------------------------- 1417c478bd9Sstevel@tonic-gate * The segment structure (vmem_seg_t) contains two doubly-linked lists. 1427c478bd9Sstevel@tonic-gate * 1437c478bd9Sstevel@tonic-gate * The arena list (vs_anext/vs_aprev) links all segments in the arena. 1447c478bd9Sstevel@tonic-gate * In addition to the allocated and free segments, the arena contains 1457c478bd9Sstevel@tonic-gate * special marker segments at span boundaries. Span markers simplify 1467c478bd9Sstevel@tonic-gate * coalescing and importing logic by making it easy to tell both when 1477c478bd9Sstevel@tonic-gate * we're at a span boundary (so we don't coalesce across it), and when 1487c478bd9Sstevel@tonic-gate * a span is completely free (its neighbors will both be span markers). 1497c478bd9Sstevel@tonic-gate * 1507c478bd9Sstevel@tonic-gate * Imported spans will have vs_import set. 1517c478bd9Sstevel@tonic-gate * 1527c478bd9Sstevel@tonic-gate * The next-of-kin list (vs_knext/vs_kprev) links segments of the same type: 1537c478bd9Sstevel@tonic-gate * (1) for allocated segments, vs_knext is the hash chain linkage; 1547c478bd9Sstevel@tonic-gate * (2) for free segments, vs_knext is the freelist linkage; 1557c478bd9Sstevel@tonic-gate * (3) for span marker segments, vs_knext is the next span marker. 1567c478bd9Sstevel@tonic-gate * 1577c478bd9Sstevel@tonic-gate * 2.2 Allocation hashing 1587c478bd9Sstevel@tonic-gate * ---------------------- 1597c478bd9Sstevel@tonic-gate * We maintain a hash table of all allocated segments, hashed by address. 1607c478bd9Sstevel@tonic-gate * This allows vmem_free() to discover the target segment in constant time. 1617c478bd9Sstevel@tonic-gate * vmem_update() periodically resizes hash tables to keep hash chains short. 1627c478bd9Sstevel@tonic-gate * 1637c478bd9Sstevel@tonic-gate * 2.3 Freelist management 1647c478bd9Sstevel@tonic-gate * ----------------------- 1657c478bd9Sstevel@tonic-gate * We maintain power-of-2 freelists for free segments, i.e. free segments 1667c478bd9Sstevel@tonic-gate * of size >= 2^n reside in vmp->vm_freelist[n]. To ensure constant-time 1677c478bd9Sstevel@tonic-gate * allocation, vmem_xalloc() looks not in the first freelist that *might* 1687c478bd9Sstevel@tonic-gate * satisfy the allocation, but in the first freelist that *definitely* 1697c478bd9Sstevel@tonic-gate * satisfies the allocation (unless VM_BESTFIT is specified, or all larger 1707c478bd9Sstevel@tonic-gate * freelists are empty). For example, a 1000-byte allocation will be 1717c478bd9Sstevel@tonic-gate * satisfied not from the 512..1023-byte freelist, whose members *might* 1727c478bd9Sstevel@tonic-gate * contains a 1000-byte segment, but from a 1024-byte or larger freelist, 1737c478bd9Sstevel@tonic-gate * the first member of which will *definitely* satisfy the allocation. 1747c478bd9Sstevel@tonic-gate * This ensures that vmem_xalloc() works in constant time. 1757c478bd9Sstevel@tonic-gate * 1767c478bd9Sstevel@tonic-gate * We maintain a bit map to determine quickly which freelists are non-empty. 1777c478bd9Sstevel@tonic-gate * vmp->vm_freemap & (1 << n) is non-zero iff vmp->vm_freelist[n] is non-empty. 1787c478bd9Sstevel@tonic-gate * 1797c478bd9Sstevel@tonic-gate * The different freelists are linked together into one large freelist, 1807c478bd9Sstevel@tonic-gate * with the freelist heads serving as markers. Freelist markers simplify 1817c478bd9Sstevel@tonic-gate * the maintenance of vm_freemap by making it easy to tell when we're taking 1827c478bd9Sstevel@tonic-gate * the last member of a freelist (both of its neighbors will be markers). 1837c478bd9Sstevel@tonic-gate * 1847c478bd9Sstevel@tonic-gate * 2.4 Vmem Locking 1857c478bd9Sstevel@tonic-gate * ---------------- 1867c478bd9Sstevel@tonic-gate * For simplicity, all arena state is protected by a per-arena lock. 1877c478bd9Sstevel@tonic-gate * For very hot arenas, use quantum caching for scalability. 1887c478bd9Sstevel@tonic-gate * 1897c478bd9Sstevel@tonic-gate * 2.5 Vmem Population 1907c478bd9Sstevel@tonic-gate * ------------------- 1917c478bd9Sstevel@tonic-gate * Any internal vmem routine that might need to allocate new segment 1927c478bd9Sstevel@tonic-gate * structures must prepare in advance by calling vmem_populate(), which 1937c478bd9Sstevel@tonic-gate * will preallocate enough vmem_seg_t's to get is through the entire 1947c478bd9Sstevel@tonic-gate * operation without dropping the arena lock. 1957c478bd9Sstevel@tonic-gate * 1967c478bd9Sstevel@tonic-gate * 2.6 Auditing 1977c478bd9Sstevel@tonic-gate * ------------ 1987c478bd9Sstevel@tonic-gate * If KMF_AUDIT is set in kmem_flags, we audit vmem allocations as well. 1997c478bd9Sstevel@tonic-gate * Since virtual addresses cannot be scribbled on, there is no equivalent 2007c478bd9Sstevel@tonic-gate * in vmem to redzone checking, deadbeef, or other kmem debugging features. 2017c478bd9Sstevel@tonic-gate * Moreover, we do not audit frees because segment coalescing destroys the 2027c478bd9Sstevel@tonic-gate * association between an address and its segment structure. Auditing is 2037c478bd9Sstevel@tonic-gate * thus intended primarily to keep track of who's consuming the arena. 2047c478bd9Sstevel@tonic-gate * Debugging support could certainly be extended in the future if it proves 2057c478bd9Sstevel@tonic-gate * necessary, but we do so much live checking via the allocation hash table 2067c478bd9Sstevel@tonic-gate * that even non-DEBUG systems get quite a bit of sanity checking already. 2077c478bd9Sstevel@tonic-gate */ 2087c478bd9Sstevel@tonic-gate 2097c478bd9Sstevel@tonic-gate #include <sys/vmem_impl.h> 2107c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 2117c478bd9Sstevel@tonic-gate #include <sys/kstat.h> 2127c478bd9Sstevel@tonic-gate #include <sys/param.h> 2137c478bd9Sstevel@tonic-gate #include <sys/systm.h> 2147c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 2157c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 2167c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 2177c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 2187c478bd9Sstevel@tonic-gate #include <sys/debug.h> 2197c478bd9Sstevel@tonic-gate #include <sys/panic.h> 2207c478bd9Sstevel@tonic-gate 2217c478bd9Sstevel@tonic-gate #define VMEM_INITIAL 10 /* early vmem arenas */ 2227c478bd9Sstevel@tonic-gate #define VMEM_SEG_INITIAL 200 /* early segments */ 2237c478bd9Sstevel@tonic-gate 2247c478bd9Sstevel@tonic-gate /* 2257c478bd9Sstevel@tonic-gate * Adding a new span to an arena requires two segment structures: one to 2267c478bd9Sstevel@tonic-gate * represent the span, and one to represent the free segment it contains. 2277c478bd9Sstevel@tonic-gate */ 2287c478bd9Sstevel@tonic-gate #define VMEM_SEGS_PER_SPAN_CREATE 2 2297c478bd9Sstevel@tonic-gate 2307c478bd9Sstevel@tonic-gate /* 2317c478bd9Sstevel@tonic-gate * Allocating a piece of an existing segment requires 0-2 segment structures 2327c478bd9Sstevel@tonic-gate * depending on how much of the segment we're allocating. 2337c478bd9Sstevel@tonic-gate * 2347c478bd9Sstevel@tonic-gate * To allocate the entire segment, no new segment structures are needed; we 2357c478bd9Sstevel@tonic-gate * simply move the existing segment structure from the freelist to the 2367c478bd9Sstevel@tonic-gate * allocation hash table. 2377c478bd9Sstevel@tonic-gate * 2387c478bd9Sstevel@tonic-gate * To allocate a piece from the left or right end of the segment, we must 2397c478bd9Sstevel@tonic-gate * split the segment into two pieces (allocated part and remainder), so we 2407c478bd9Sstevel@tonic-gate * need one new segment structure to represent the remainder. 2417c478bd9Sstevel@tonic-gate * 2427c478bd9Sstevel@tonic-gate * To allocate from the middle of a segment, we need two new segment strucures 2437c478bd9Sstevel@tonic-gate * to represent the remainders on either side of the allocated part. 2447c478bd9Sstevel@tonic-gate */ 2457c478bd9Sstevel@tonic-gate #define VMEM_SEGS_PER_EXACT_ALLOC 0 2467c478bd9Sstevel@tonic-gate #define VMEM_SEGS_PER_LEFT_ALLOC 1 2477c478bd9Sstevel@tonic-gate #define VMEM_SEGS_PER_RIGHT_ALLOC 1 2487c478bd9Sstevel@tonic-gate #define VMEM_SEGS_PER_MIDDLE_ALLOC 2 2497c478bd9Sstevel@tonic-gate 2507c478bd9Sstevel@tonic-gate /* 2517c478bd9Sstevel@tonic-gate * vmem_populate() preallocates segment structures for vmem to do its work. 2527c478bd9Sstevel@tonic-gate * It must preallocate enough for the worst case, which is when we must import 2537c478bd9Sstevel@tonic-gate * a new span and then allocate from the middle of it. 2547c478bd9Sstevel@tonic-gate */ 2557c478bd9Sstevel@tonic-gate #define VMEM_SEGS_PER_ALLOC_MAX \ 2567c478bd9Sstevel@tonic-gate (VMEM_SEGS_PER_SPAN_CREATE + VMEM_SEGS_PER_MIDDLE_ALLOC) 2577c478bd9Sstevel@tonic-gate 2587c478bd9Sstevel@tonic-gate /* 2597c478bd9Sstevel@tonic-gate * The segment structures themselves are allocated from vmem_seg_arena, so 2607c478bd9Sstevel@tonic-gate * we have a recursion problem when vmem_seg_arena needs to populate itself. 2617c478bd9Sstevel@tonic-gate * We address this by working out the maximum number of segment structures 2627c478bd9Sstevel@tonic-gate * this act will require, and multiplying by the maximum number of threads 2637c478bd9Sstevel@tonic-gate * that we'll allow to do it simultaneously. 2647c478bd9Sstevel@tonic-gate * 2657c478bd9Sstevel@tonic-gate * The worst-case segment consumption to populate vmem_seg_arena is as 2667c478bd9Sstevel@tonic-gate * follows (depicted as a stack trace to indicate why events are occurring): 2677c478bd9Sstevel@tonic-gate * 2687c478bd9Sstevel@tonic-gate * (In order to lower the fragmentation in the heap_arena, we specify a 2697c478bd9Sstevel@tonic-gate * minimum import size for the vmem_metadata_arena which is the same size 2707c478bd9Sstevel@tonic-gate * as the kmem_va quantum cache allocations. This causes the worst-case 2717c478bd9Sstevel@tonic-gate * allocation from the vmem_metadata_arena to be 3 segments.) 2727c478bd9Sstevel@tonic-gate * 2737c478bd9Sstevel@tonic-gate * vmem_alloc(vmem_seg_arena) -> 2 segs (span create + exact alloc) 2747c478bd9Sstevel@tonic-gate * segkmem_alloc(vmem_metadata_arena) 2757c478bd9Sstevel@tonic-gate * vmem_alloc(vmem_metadata_arena) -> 3 segs (span create + left alloc) 2767c478bd9Sstevel@tonic-gate * vmem_alloc(heap_arena) -> 1 seg (left alloc) 2777c478bd9Sstevel@tonic-gate * page_create() 2787c478bd9Sstevel@tonic-gate * hat_memload() 2797c478bd9Sstevel@tonic-gate * kmem_cache_alloc() 2807c478bd9Sstevel@tonic-gate * kmem_slab_create() 2817c478bd9Sstevel@tonic-gate * vmem_alloc(hat_memload_arena) -> 2 segs (span create + exact alloc) 2827c478bd9Sstevel@tonic-gate * segkmem_alloc(heap_arena) 2837c478bd9Sstevel@tonic-gate * vmem_alloc(heap_arena) -> 1 seg (left alloc) 2847c478bd9Sstevel@tonic-gate * page_create() 2857c478bd9Sstevel@tonic-gate * hat_memload() -> (hat layer won't recurse further) 2867c478bd9Sstevel@tonic-gate * 2877c478bd9Sstevel@tonic-gate * The worst-case consumption for each arena is 3 segment structures. 2887c478bd9Sstevel@tonic-gate * Of course, a 3-seg reserve could easily be blown by multiple threads. 2897c478bd9Sstevel@tonic-gate * Therefore, we serialize all allocations from vmem_seg_arena (which is OK 2907c478bd9Sstevel@tonic-gate * because they're rare). We cannot allow a non-blocking allocation to get 2917c478bd9Sstevel@tonic-gate * tied up behind a blocking allocation, however, so we use separate locks 29210b95589Sbonwick * for VM_SLEEP and VM_NOSLEEP allocations. Similarly, VM_PUSHPAGE allocations 29310b95589Sbonwick * must not block behind ordinary VM_SLEEPs. In addition, if the system is 2947c478bd9Sstevel@tonic-gate * panicking then we must keep enough resources for panic_thread to do its 29510b95589Sbonwick * work. Thus we have at most four threads trying to allocate from 2967c478bd9Sstevel@tonic-gate * vmem_seg_arena, and each thread consumes at most three segment structures, 29710b95589Sbonwick * so we must maintain a 12-seg reserve. 2987c478bd9Sstevel@tonic-gate */ 29910b95589Sbonwick #define VMEM_POPULATE_RESERVE 12 3007c478bd9Sstevel@tonic-gate 3017c478bd9Sstevel@tonic-gate /* 3027c478bd9Sstevel@tonic-gate * vmem_populate() ensures that each arena has VMEM_MINFREE seg structures 3037c478bd9Sstevel@tonic-gate * so that it can satisfy the worst-case allocation *and* participate in 3047c478bd9Sstevel@tonic-gate * worst-case allocation from vmem_seg_arena. 3057c478bd9Sstevel@tonic-gate */ 3067c478bd9Sstevel@tonic-gate #define VMEM_MINFREE (VMEM_POPULATE_RESERVE + VMEM_SEGS_PER_ALLOC_MAX) 3077c478bd9Sstevel@tonic-gate 3087c478bd9Sstevel@tonic-gate static vmem_t vmem0[VMEM_INITIAL]; 3097c478bd9Sstevel@tonic-gate static vmem_t *vmem_populator[VMEM_INITIAL]; 3107c478bd9Sstevel@tonic-gate static uint32_t vmem_id; 3117c478bd9Sstevel@tonic-gate static uint32_t vmem_populators; 3127c478bd9Sstevel@tonic-gate static vmem_seg_t vmem_seg0[VMEM_SEG_INITIAL]; 3137c478bd9Sstevel@tonic-gate static vmem_seg_t *vmem_segfree; 3147c478bd9Sstevel@tonic-gate static kmutex_t vmem_list_lock; 3157c478bd9Sstevel@tonic-gate static kmutex_t vmem_segfree_lock; 3167c478bd9Sstevel@tonic-gate static kmutex_t vmem_sleep_lock; 3177c478bd9Sstevel@tonic-gate static kmutex_t vmem_nosleep_lock; 31810b95589Sbonwick static kmutex_t vmem_pushpage_lock; 3197c478bd9Sstevel@tonic-gate static kmutex_t vmem_panic_lock; 3207c478bd9Sstevel@tonic-gate static vmem_t *vmem_list; 3217c478bd9Sstevel@tonic-gate static vmem_t *vmem_metadata_arena; 3227c478bd9Sstevel@tonic-gate static vmem_t *vmem_seg_arena; 3237c478bd9Sstevel@tonic-gate static vmem_t *vmem_hash_arena; 3247c478bd9Sstevel@tonic-gate static vmem_t *vmem_vmem_arena; 3257c478bd9Sstevel@tonic-gate static long vmem_update_interval = 15; /* vmem_update() every 15 seconds */ 3267c478bd9Sstevel@tonic-gate uint32_t vmem_mtbf; /* mean time between failures [default: off] */ 3277c478bd9Sstevel@tonic-gate size_t vmem_seg_size = sizeof (vmem_seg_t); 3287c478bd9Sstevel@tonic-gate 3297c478bd9Sstevel@tonic-gate static vmem_kstat_t vmem_kstat_template = { 3307c478bd9Sstevel@tonic-gate { "mem_inuse", KSTAT_DATA_UINT64 }, 3317c478bd9Sstevel@tonic-gate { "mem_import", KSTAT_DATA_UINT64 }, 3327c478bd9Sstevel@tonic-gate { "mem_total", KSTAT_DATA_UINT64 }, 3337c478bd9Sstevel@tonic-gate { "vmem_source", KSTAT_DATA_UINT32 }, 3347c478bd9Sstevel@tonic-gate { "alloc", KSTAT_DATA_UINT64 }, 3357c478bd9Sstevel@tonic-gate { "free", KSTAT_DATA_UINT64 }, 3367c478bd9Sstevel@tonic-gate { "wait", KSTAT_DATA_UINT64 }, 3377c478bd9Sstevel@tonic-gate { "fail", KSTAT_DATA_UINT64 }, 3387c478bd9Sstevel@tonic-gate { "lookup", KSTAT_DATA_UINT64 }, 3397c478bd9Sstevel@tonic-gate { "search", KSTAT_DATA_UINT64 }, 3407c478bd9Sstevel@tonic-gate { "populate_wait", KSTAT_DATA_UINT64 }, 3417c478bd9Sstevel@tonic-gate { "populate_fail", KSTAT_DATA_UINT64 }, 3427c478bd9Sstevel@tonic-gate { "contains", KSTAT_DATA_UINT64 }, 3437c478bd9Sstevel@tonic-gate { "contains_search", KSTAT_DATA_UINT64 }, 3447c478bd9Sstevel@tonic-gate }; 3457c478bd9Sstevel@tonic-gate 3467c478bd9Sstevel@tonic-gate /* 3477c478bd9Sstevel@tonic-gate * Insert/delete from arena list (type 'a') or next-of-kin list (type 'k'). 3487c478bd9Sstevel@tonic-gate */ 3497c478bd9Sstevel@tonic-gate #define VMEM_INSERT(vprev, vsp, type) \ 3507c478bd9Sstevel@tonic-gate { \ 3517c478bd9Sstevel@tonic-gate vmem_seg_t *vnext = (vprev)->vs_##type##next; \ 3527c478bd9Sstevel@tonic-gate (vsp)->vs_##type##next = (vnext); \ 3537c478bd9Sstevel@tonic-gate (vsp)->vs_##type##prev = (vprev); \ 3547c478bd9Sstevel@tonic-gate (vprev)->vs_##type##next = (vsp); \ 3557c478bd9Sstevel@tonic-gate (vnext)->vs_##type##prev = (vsp); \ 3567c478bd9Sstevel@tonic-gate } 3577c478bd9Sstevel@tonic-gate 3587c478bd9Sstevel@tonic-gate #define VMEM_DELETE(vsp, type) \ 3597c478bd9Sstevel@tonic-gate { \ 3607c478bd9Sstevel@tonic-gate vmem_seg_t *vprev = (vsp)->vs_##type##prev; \ 3617c478bd9Sstevel@tonic-gate vmem_seg_t *vnext = (vsp)->vs_##type##next; \ 3627c478bd9Sstevel@tonic-gate (vprev)->vs_##type##next = (vnext); \ 3637c478bd9Sstevel@tonic-gate (vnext)->vs_##type##prev = (vprev); \ 3647c478bd9Sstevel@tonic-gate } 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate /* 3677c478bd9Sstevel@tonic-gate * Get a vmem_seg_t from the global segfree list. 3687c478bd9Sstevel@tonic-gate */ 3697c478bd9Sstevel@tonic-gate static vmem_seg_t * 3707c478bd9Sstevel@tonic-gate vmem_getseg_global(void) 3717c478bd9Sstevel@tonic-gate { 3727c478bd9Sstevel@tonic-gate vmem_seg_t *vsp; 3737c478bd9Sstevel@tonic-gate 3747c478bd9Sstevel@tonic-gate mutex_enter(&vmem_segfree_lock); 3757c478bd9Sstevel@tonic-gate if ((vsp = vmem_segfree) != NULL) 3767c478bd9Sstevel@tonic-gate vmem_segfree = vsp->vs_knext; 3777c478bd9Sstevel@tonic-gate mutex_exit(&vmem_segfree_lock); 3787c478bd9Sstevel@tonic-gate 3797c478bd9Sstevel@tonic-gate return (vsp); 3807c478bd9Sstevel@tonic-gate } 3817c478bd9Sstevel@tonic-gate 3827c478bd9Sstevel@tonic-gate /* 3837c478bd9Sstevel@tonic-gate * Put a vmem_seg_t on the global segfree list. 3847c478bd9Sstevel@tonic-gate */ 3857c478bd9Sstevel@tonic-gate static void 3867c478bd9Sstevel@tonic-gate vmem_putseg_global(vmem_seg_t *vsp) 3877c478bd9Sstevel@tonic-gate { 3887c478bd9Sstevel@tonic-gate mutex_enter(&vmem_segfree_lock); 3897c478bd9Sstevel@tonic-gate vsp->vs_knext = vmem_segfree; 3907c478bd9Sstevel@tonic-gate vmem_segfree = vsp; 3917c478bd9Sstevel@tonic-gate mutex_exit(&vmem_segfree_lock); 3927c478bd9Sstevel@tonic-gate } 3937c478bd9Sstevel@tonic-gate 3947c478bd9Sstevel@tonic-gate /* 3957c478bd9Sstevel@tonic-gate * Get a vmem_seg_t from vmp's segfree list. 3967c478bd9Sstevel@tonic-gate */ 3977c478bd9Sstevel@tonic-gate static vmem_seg_t * 3987c478bd9Sstevel@tonic-gate vmem_getseg(vmem_t *vmp) 3997c478bd9Sstevel@tonic-gate { 4007c478bd9Sstevel@tonic-gate vmem_seg_t *vsp; 4017c478bd9Sstevel@tonic-gate 4027c478bd9Sstevel@tonic-gate ASSERT(vmp->vm_nsegfree > 0); 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate vsp = vmp->vm_segfree; 4057c478bd9Sstevel@tonic-gate vmp->vm_segfree = vsp->vs_knext; 4067c478bd9Sstevel@tonic-gate vmp->vm_nsegfree--; 4077c478bd9Sstevel@tonic-gate 4087c478bd9Sstevel@tonic-gate return (vsp); 4097c478bd9Sstevel@tonic-gate } 4107c478bd9Sstevel@tonic-gate 4117c478bd9Sstevel@tonic-gate /* 4127c478bd9Sstevel@tonic-gate * Put a vmem_seg_t on vmp's segfree list. 4137c478bd9Sstevel@tonic-gate */ 4147c478bd9Sstevel@tonic-gate static void 4157c478bd9Sstevel@tonic-gate vmem_putseg(vmem_t *vmp, vmem_seg_t *vsp) 4167c478bd9Sstevel@tonic-gate { 4177c478bd9Sstevel@tonic-gate vsp->vs_knext = vmp->vm_segfree; 4187c478bd9Sstevel@tonic-gate vmp->vm_segfree = vsp; 4197c478bd9Sstevel@tonic-gate vmp->vm_nsegfree++; 4207c478bd9Sstevel@tonic-gate } 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate /* 4237c478bd9Sstevel@tonic-gate * Add vsp to the appropriate freelist. 4247c478bd9Sstevel@tonic-gate */ 4257c478bd9Sstevel@tonic-gate static void 4267c478bd9Sstevel@tonic-gate vmem_freelist_insert(vmem_t *vmp, vmem_seg_t *vsp) 4277c478bd9Sstevel@tonic-gate { 4287c478bd9Sstevel@tonic-gate vmem_seg_t *vprev; 4297c478bd9Sstevel@tonic-gate 4307c478bd9Sstevel@tonic-gate ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp); 4317c478bd9Sstevel@tonic-gate 4327c478bd9Sstevel@tonic-gate vprev = (vmem_seg_t *)&vmp->vm_freelist[highbit(VS_SIZE(vsp)) - 1]; 4337c478bd9Sstevel@tonic-gate vsp->vs_type = VMEM_FREE; 4347c478bd9Sstevel@tonic-gate vmp->vm_freemap |= VS_SIZE(vprev); 4357c478bd9Sstevel@tonic-gate VMEM_INSERT(vprev, vsp, k); 4367c478bd9Sstevel@tonic-gate 4377c478bd9Sstevel@tonic-gate cv_broadcast(&vmp->vm_cv); 4387c478bd9Sstevel@tonic-gate } 4397c478bd9Sstevel@tonic-gate 4407c478bd9Sstevel@tonic-gate /* 4417c478bd9Sstevel@tonic-gate * Take vsp from the freelist. 4427c478bd9Sstevel@tonic-gate */ 4437c478bd9Sstevel@tonic-gate static void 4447c478bd9Sstevel@tonic-gate vmem_freelist_delete(vmem_t *vmp, vmem_seg_t *vsp) 4457c478bd9Sstevel@tonic-gate { 4467c478bd9Sstevel@tonic-gate ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp); 4477c478bd9Sstevel@tonic-gate ASSERT(vsp->vs_type == VMEM_FREE); 4487c478bd9Sstevel@tonic-gate 4497c478bd9Sstevel@tonic-gate if (vsp->vs_knext->vs_start == 0 && vsp->vs_kprev->vs_start == 0) { 4507c478bd9Sstevel@tonic-gate /* 4517c478bd9Sstevel@tonic-gate * The segments on both sides of 'vsp' are freelist heads, 4527c478bd9Sstevel@tonic-gate * so taking vsp leaves the freelist at vsp->vs_kprev empty. 4537c478bd9Sstevel@tonic-gate */ 4547c478bd9Sstevel@tonic-gate ASSERT(vmp->vm_freemap & VS_SIZE(vsp->vs_kprev)); 4557c478bd9Sstevel@tonic-gate vmp->vm_freemap ^= VS_SIZE(vsp->vs_kprev); 4567c478bd9Sstevel@tonic-gate } 4577c478bd9Sstevel@tonic-gate VMEM_DELETE(vsp, k); 4587c478bd9Sstevel@tonic-gate } 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate /* 4617c478bd9Sstevel@tonic-gate * Add vsp to the allocated-segment hash table and update kstats. 4627c478bd9Sstevel@tonic-gate */ 4637c478bd9Sstevel@tonic-gate static void 4647c478bd9Sstevel@tonic-gate vmem_hash_insert(vmem_t *vmp, vmem_seg_t *vsp) 4657c478bd9Sstevel@tonic-gate { 4667c478bd9Sstevel@tonic-gate vmem_seg_t **bucket; 4677c478bd9Sstevel@tonic-gate 4687c478bd9Sstevel@tonic-gate vsp->vs_type = VMEM_ALLOC; 4697c478bd9Sstevel@tonic-gate bucket = VMEM_HASH(vmp, vsp->vs_start); 4707c478bd9Sstevel@tonic-gate vsp->vs_knext = *bucket; 4717c478bd9Sstevel@tonic-gate *bucket = vsp; 4727c478bd9Sstevel@tonic-gate 4737c478bd9Sstevel@tonic-gate if (vmem_seg_size == sizeof (vmem_seg_t)) { 4747c478bd9Sstevel@tonic-gate vsp->vs_depth = (uint8_t)getpcstack(vsp->vs_stack, 4757c478bd9Sstevel@tonic-gate VMEM_STACK_DEPTH); 4767c478bd9Sstevel@tonic-gate vsp->vs_thread = curthread; 4777c478bd9Sstevel@tonic-gate vsp->vs_timestamp = gethrtime(); 4787c478bd9Sstevel@tonic-gate } else { 4797c478bd9Sstevel@tonic-gate vsp->vs_depth = 0; 4807c478bd9Sstevel@tonic-gate } 4817c478bd9Sstevel@tonic-gate 4827c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_alloc.value.ui64++; 4837c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_mem_inuse.value.ui64 += VS_SIZE(vsp); 4847c478bd9Sstevel@tonic-gate } 4857c478bd9Sstevel@tonic-gate 4867c478bd9Sstevel@tonic-gate /* 4877c478bd9Sstevel@tonic-gate * Remove vsp from the allocated-segment hash table and update kstats. 4887c478bd9Sstevel@tonic-gate */ 4897c478bd9Sstevel@tonic-gate static vmem_seg_t * 4907c478bd9Sstevel@tonic-gate vmem_hash_delete(vmem_t *vmp, uintptr_t addr, size_t size) 4917c478bd9Sstevel@tonic-gate { 4927c478bd9Sstevel@tonic-gate vmem_seg_t *vsp, **prev_vspp; 4937c478bd9Sstevel@tonic-gate 4947c478bd9Sstevel@tonic-gate prev_vspp = VMEM_HASH(vmp, addr); 4957c478bd9Sstevel@tonic-gate while ((vsp = *prev_vspp) != NULL) { 4967c478bd9Sstevel@tonic-gate if (vsp->vs_start == addr) { 4977c478bd9Sstevel@tonic-gate *prev_vspp = vsp->vs_knext; 4987c478bd9Sstevel@tonic-gate break; 4997c478bd9Sstevel@tonic-gate } 5007c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_lookup.value.ui64++; 5017c478bd9Sstevel@tonic-gate prev_vspp = &vsp->vs_knext; 5027c478bd9Sstevel@tonic-gate } 5037c478bd9Sstevel@tonic-gate 5047c478bd9Sstevel@tonic-gate if (vsp == NULL) 5057c478bd9Sstevel@tonic-gate panic("vmem_hash_delete(%p, %lx, %lu): bad free", 506*903a11ebSrh87107 (void *)vmp, addr, size); 5077c478bd9Sstevel@tonic-gate if (VS_SIZE(vsp) != size) 5087c478bd9Sstevel@tonic-gate panic("vmem_hash_delete(%p, %lx, %lu): wrong size (expect %lu)", 509*903a11ebSrh87107 (void *)vmp, addr, size, VS_SIZE(vsp)); 5107c478bd9Sstevel@tonic-gate 5117c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_free.value.ui64++; 5127c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_mem_inuse.value.ui64 -= size; 5137c478bd9Sstevel@tonic-gate 5147c478bd9Sstevel@tonic-gate return (vsp); 5157c478bd9Sstevel@tonic-gate } 5167c478bd9Sstevel@tonic-gate 5177c478bd9Sstevel@tonic-gate /* 5187c478bd9Sstevel@tonic-gate * Create a segment spanning the range [start, end) and add it to the arena. 5197c478bd9Sstevel@tonic-gate */ 5207c478bd9Sstevel@tonic-gate static vmem_seg_t * 5217c478bd9Sstevel@tonic-gate vmem_seg_create(vmem_t *vmp, vmem_seg_t *vprev, uintptr_t start, uintptr_t end) 5227c478bd9Sstevel@tonic-gate { 5237c478bd9Sstevel@tonic-gate vmem_seg_t *newseg = vmem_getseg(vmp); 5247c478bd9Sstevel@tonic-gate 5257c478bd9Sstevel@tonic-gate newseg->vs_start = start; 5267c478bd9Sstevel@tonic-gate newseg->vs_end = end; 5277c478bd9Sstevel@tonic-gate newseg->vs_type = 0; 5287c478bd9Sstevel@tonic-gate newseg->vs_import = 0; 5297c478bd9Sstevel@tonic-gate 5307c478bd9Sstevel@tonic-gate VMEM_INSERT(vprev, newseg, a); 5317c478bd9Sstevel@tonic-gate 5327c478bd9Sstevel@tonic-gate return (newseg); 5337c478bd9Sstevel@tonic-gate } 5347c478bd9Sstevel@tonic-gate 5357c478bd9Sstevel@tonic-gate /* 5367c478bd9Sstevel@tonic-gate * Remove segment vsp from the arena. 5377c478bd9Sstevel@tonic-gate */ 5387c478bd9Sstevel@tonic-gate static void 5397c478bd9Sstevel@tonic-gate vmem_seg_destroy(vmem_t *vmp, vmem_seg_t *vsp) 5407c478bd9Sstevel@tonic-gate { 5417c478bd9Sstevel@tonic-gate ASSERT(vsp->vs_type != VMEM_ROTOR); 5427c478bd9Sstevel@tonic-gate VMEM_DELETE(vsp, a); 5437c478bd9Sstevel@tonic-gate 5447c478bd9Sstevel@tonic-gate vmem_putseg(vmp, vsp); 5457c478bd9Sstevel@tonic-gate } 5467c478bd9Sstevel@tonic-gate 5477c478bd9Sstevel@tonic-gate /* 5487c478bd9Sstevel@tonic-gate * Add the span [vaddr, vaddr + size) to vmp and update kstats. 5497c478bd9Sstevel@tonic-gate */ 5507c478bd9Sstevel@tonic-gate static vmem_seg_t * 5517c478bd9Sstevel@tonic-gate vmem_span_create(vmem_t *vmp, void *vaddr, size_t size, uint8_t import) 5527c478bd9Sstevel@tonic-gate { 5537c478bd9Sstevel@tonic-gate vmem_seg_t *newseg, *span; 5547c478bd9Sstevel@tonic-gate uintptr_t start = (uintptr_t)vaddr; 5557c478bd9Sstevel@tonic-gate uintptr_t end = start + size; 5567c478bd9Sstevel@tonic-gate 5577c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&vmp->vm_lock)); 5587c478bd9Sstevel@tonic-gate 5597c478bd9Sstevel@tonic-gate if ((start | end) & (vmp->vm_quantum - 1)) 5607c478bd9Sstevel@tonic-gate panic("vmem_span_create(%p, %p, %lu): misaligned", 561*903a11ebSrh87107 (void *)vmp, vaddr, size); 5627c478bd9Sstevel@tonic-gate 5637c478bd9Sstevel@tonic-gate span = vmem_seg_create(vmp, vmp->vm_seg0.vs_aprev, start, end); 5647c478bd9Sstevel@tonic-gate span->vs_type = VMEM_SPAN; 5657c478bd9Sstevel@tonic-gate span->vs_import = import; 5667c478bd9Sstevel@tonic-gate VMEM_INSERT(vmp->vm_seg0.vs_kprev, span, k); 5677c478bd9Sstevel@tonic-gate 5687c478bd9Sstevel@tonic-gate newseg = vmem_seg_create(vmp, span, start, end); 5697c478bd9Sstevel@tonic-gate vmem_freelist_insert(vmp, newseg); 5707c478bd9Sstevel@tonic-gate 5717c478bd9Sstevel@tonic-gate if (import) 5727c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_mem_import.value.ui64 += size; 5737c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_mem_total.value.ui64 += size; 5747c478bd9Sstevel@tonic-gate 5757c478bd9Sstevel@tonic-gate return (newseg); 5767c478bd9Sstevel@tonic-gate } 5777c478bd9Sstevel@tonic-gate 5787c478bd9Sstevel@tonic-gate /* 5797c478bd9Sstevel@tonic-gate * Remove span vsp from vmp and update kstats. 5807c478bd9Sstevel@tonic-gate */ 5817c478bd9Sstevel@tonic-gate static void 5827c478bd9Sstevel@tonic-gate vmem_span_destroy(vmem_t *vmp, vmem_seg_t *vsp) 5837c478bd9Sstevel@tonic-gate { 5847c478bd9Sstevel@tonic-gate vmem_seg_t *span = vsp->vs_aprev; 5857c478bd9Sstevel@tonic-gate size_t size = VS_SIZE(vsp); 5867c478bd9Sstevel@tonic-gate 5877c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&vmp->vm_lock)); 5887c478bd9Sstevel@tonic-gate ASSERT(span->vs_type == VMEM_SPAN); 5897c478bd9Sstevel@tonic-gate 5907c478bd9Sstevel@tonic-gate if (span->vs_import) 5917c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_mem_import.value.ui64 -= size; 5927c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_mem_total.value.ui64 -= size; 5937c478bd9Sstevel@tonic-gate 5947c478bd9Sstevel@tonic-gate VMEM_DELETE(span, k); 5957c478bd9Sstevel@tonic-gate 5967c478bd9Sstevel@tonic-gate vmem_seg_destroy(vmp, vsp); 5977c478bd9Sstevel@tonic-gate vmem_seg_destroy(vmp, span); 5987c478bd9Sstevel@tonic-gate } 5997c478bd9Sstevel@tonic-gate 6007c478bd9Sstevel@tonic-gate /* 6017c478bd9Sstevel@tonic-gate * Allocate the subrange [addr, addr + size) from segment vsp. 6027c478bd9Sstevel@tonic-gate * If there are leftovers on either side, place them on the freelist. 6037c478bd9Sstevel@tonic-gate * Returns a pointer to the segment representing [addr, addr + size). 6047c478bd9Sstevel@tonic-gate */ 6057c478bd9Sstevel@tonic-gate static vmem_seg_t * 6067c478bd9Sstevel@tonic-gate vmem_seg_alloc(vmem_t *vmp, vmem_seg_t *vsp, uintptr_t addr, size_t size) 6077c478bd9Sstevel@tonic-gate { 6087c478bd9Sstevel@tonic-gate uintptr_t vs_start = vsp->vs_start; 6097c478bd9Sstevel@tonic-gate uintptr_t vs_end = vsp->vs_end; 6107c478bd9Sstevel@tonic-gate size_t vs_size = vs_end - vs_start; 6117c478bd9Sstevel@tonic-gate size_t realsize = P2ROUNDUP(size, vmp->vm_quantum); 6127c478bd9Sstevel@tonic-gate uintptr_t addr_end = addr + realsize; 6137c478bd9Sstevel@tonic-gate 6147c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(vs_start, vmp->vm_quantum) == 0); 6157c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(addr, vmp->vm_quantum) == 0); 6167c478bd9Sstevel@tonic-gate ASSERT(vsp->vs_type == VMEM_FREE); 6177c478bd9Sstevel@tonic-gate ASSERT(addr >= vs_start && addr_end - 1 <= vs_end - 1); 6187c478bd9Sstevel@tonic-gate ASSERT(addr - 1 <= addr_end - 1); 6197c478bd9Sstevel@tonic-gate 6207c478bd9Sstevel@tonic-gate /* 6217c478bd9Sstevel@tonic-gate * If we're allocating from the start of the segment, and the 6227c478bd9Sstevel@tonic-gate * remainder will be on the same freelist, we can save quite 6237c478bd9Sstevel@tonic-gate * a bit of work. 6247c478bd9Sstevel@tonic-gate */ 6257c478bd9Sstevel@tonic-gate if (P2SAMEHIGHBIT(vs_size, vs_size - realsize) && addr == vs_start) { 6267c478bd9Sstevel@tonic-gate ASSERT(highbit(vs_size) == highbit(vs_size - realsize)); 6277c478bd9Sstevel@tonic-gate vsp->vs_start = addr_end; 6287c478bd9Sstevel@tonic-gate vsp = vmem_seg_create(vmp, vsp->vs_aprev, addr, addr + size); 6297c478bd9Sstevel@tonic-gate vmem_hash_insert(vmp, vsp); 6307c478bd9Sstevel@tonic-gate return (vsp); 6317c478bd9Sstevel@tonic-gate } 6327c478bd9Sstevel@tonic-gate 6337c478bd9Sstevel@tonic-gate vmem_freelist_delete(vmp, vsp); 6347c478bd9Sstevel@tonic-gate 6357c478bd9Sstevel@tonic-gate if (vs_end != addr_end) 6367c478bd9Sstevel@tonic-gate vmem_freelist_insert(vmp, 6377c478bd9Sstevel@tonic-gate vmem_seg_create(vmp, vsp, addr_end, vs_end)); 6387c478bd9Sstevel@tonic-gate 6397c478bd9Sstevel@tonic-gate if (vs_start != addr) 6407c478bd9Sstevel@tonic-gate vmem_freelist_insert(vmp, 6417c478bd9Sstevel@tonic-gate vmem_seg_create(vmp, vsp->vs_aprev, vs_start, addr)); 6427c478bd9Sstevel@tonic-gate 6437c478bd9Sstevel@tonic-gate vsp->vs_start = addr; 6447c478bd9Sstevel@tonic-gate vsp->vs_end = addr + size; 6457c478bd9Sstevel@tonic-gate 6467c478bd9Sstevel@tonic-gate vmem_hash_insert(vmp, vsp); 6477c478bd9Sstevel@tonic-gate return (vsp); 6487c478bd9Sstevel@tonic-gate } 6497c478bd9Sstevel@tonic-gate 6507c478bd9Sstevel@tonic-gate /* 6517c478bd9Sstevel@tonic-gate * Returns 1 if we are populating, 0 otherwise. 6527c478bd9Sstevel@tonic-gate * Call it if we want to prevent recursion from HAT. 6537c478bd9Sstevel@tonic-gate */ 6547c478bd9Sstevel@tonic-gate int 6557c478bd9Sstevel@tonic-gate vmem_is_populator() 6567c478bd9Sstevel@tonic-gate { 6577c478bd9Sstevel@tonic-gate return (mutex_owner(&vmem_sleep_lock) == curthread || 6587c478bd9Sstevel@tonic-gate mutex_owner(&vmem_nosleep_lock) == curthread || 65910b95589Sbonwick mutex_owner(&vmem_pushpage_lock) == curthread || 6607c478bd9Sstevel@tonic-gate mutex_owner(&vmem_panic_lock) == curthread); 6617c478bd9Sstevel@tonic-gate } 6627c478bd9Sstevel@tonic-gate 6637c478bd9Sstevel@tonic-gate /* 6647c478bd9Sstevel@tonic-gate * Populate vmp's segfree list with VMEM_MINFREE vmem_seg_t structures. 6657c478bd9Sstevel@tonic-gate */ 6667c478bd9Sstevel@tonic-gate static int 6677c478bd9Sstevel@tonic-gate vmem_populate(vmem_t *vmp, int vmflag) 6687c478bd9Sstevel@tonic-gate { 6697c478bd9Sstevel@tonic-gate char *p; 6707c478bd9Sstevel@tonic-gate vmem_seg_t *vsp; 6717c478bd9Sstevel@tonic-gate ssize_t nseg; 6727c478bd9Sstevel@tonic-gate size_t size; 6737c478bd9Sstevel@tonic-gate kmutex_t *lp; 6747c478bd9Sstevel@tonic-gate int i; 6757c478bd9Sstevel@tonic-gate 6767c478bd9Sstevel@tonic-gate while (vmp->vm_nsegfree < VMEM_MINFREE && 6777c478bd9Sstevel@tonic-gate (vsp = vmem_getseg_global()) != NULL) 6787c478bd9Sstevel@tonic-gate vmem_putseg(vmp, vsp); 6797c478bd9Sstevel@tonic-gate 6807c478bd9Sstevel@tonic-gate if (vmp->vm_nsegfree >= VMEM_MINFREE) 6817c478bd9Sstevel@tonic-gate return (1); 6827c478bd9Sstevel@tonic-gate 6837c478bd9Sstevel@tonic-gate /* 6847c478bd9Sstevel@tonic-gate * If we're already populating, tap the reserve. 6857c478bd9Sstevel@tonic-gate */ 6867c478bd9Sstevel@tonic-gate if (vmem_is_populator()) { 6877c478bd9Sstevel@tonic-gate ASSERT(vmp->vm_cflags & VMC_POPULATOR); 6887c478bd9Sstevel@tonic-gate return (1); 6897c478bd9Sstevel@tonic-gate } 6907c478bd9Sstevel@tonic-gate 6917c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 6927c478bd9Sstevel@tonic-gate 6937c478bd9Sstevel@tonic-gate if (panic_thread == curthread) 6947c478bd9Sstevel@tonic-gate lp = &vmem_panic_lock; 6957c478bd9Sstevel@tonic-gate else if (vmflag & VM_NOSLEEP) 6967c478bd9Sstevel@tonic-gate lp = &vmem_nosleep_lock; 69710b95589Sbonwick else if (vmflag & VM_PUSHPAGE) 69810b95589Sbonwick lp = &vmem_pushpage_lock; 6997c478bd9Sstevel@tonic-gate else 7007c478bd9Sstevel@tonic-gate lp = &vmem_sleep_lock; 7017c478bd9Sstevel@tonic-gate 7027c478bd9Sstevel@tonic-gate mutex_enter(lp); 7037c478bd9Sstevel@tonic-gate 7047c478bd9Sstevel@tonic-gate nseg = VMEM_MINFREE + vmem_populators * VMEM_POPULATE_RESERVE; 7057c478bd9Sstevel@tonic-gate size = P2ROUNDUP(nseg * vmem_seg_size, vmem_seg_arena->vm_quantum); 7067c478bd9Sstevel@tonic-gate nseg = size / vmem_seg_size; 7077c478bd9Sstevel@tonic-gate 7087c478bd9Sstevel@tonic-gate /* 7097c478bd9Sstevel@tonic-gate * The following vmem_alloc() may need to populate vmem_seg_arena 7107c478bd9Sstevel@tonic-gate * and all the things it imports from. When doing so, it will tap 7117c478bd9Sstevel@tonic-gate * each arena's reserve to prevent recursion (see the block comment 7127c478bd9Sstevel@tonic-gate * above the definition of VMEM_POPULATE_RESERVE). 7137c478bd9Sstevel@tonic-gate */ 7147c478bd9Sstevel@tonic-gate p = vmem_alloc(vmem_seg_arena, size, vmflag & VM_KMFLAGS); 7157c478bd9Sstevel@tonic-gate if (p == NULL) { 7167c478bd9Sstevel@tonic-gate mutex_exit(lp); 7177c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 7187c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_populate_fail.value.ui64++; 7197c478bd9Sstevel@tonic-gate return (0); 7207c478bd9Sstevel@tonic-gate } 7217c478bd9Sstevel@tonic-gate 7227c478bd9Sstevel@tonic-gate /* 7237c478bd9Sstevel@tonic-gate * Restock the arenas that may have been depleted during population. 7247c478bd9Sstevel@tonic-gate */ 7257c478bd9Sstevel@tonic-gate for (i = 0; i < vmem_populators; i++) { 7267c478bd9Sstevel@tonic-gate mutex_enter(&vmem_populator[i]->vm_lock); 7277c478bd9Sstevel@tonic-gate while (vmem_populator[i]->vm_nsegfree < VMEM_POPULATE_RESERVE) 7287c478bd9Sstevel@tonic-gate vmem_putseg(vmem_populator[i], 7297c478bd9Sstevel@tonic-gate (vmem_seg_t *)(p + --nseg * vmem_seg_size)); 7307c478bd9Sstevel@tonic-gate mutex_exit(&vmem_populator[i]->vm_lock); 7317c478bd9Sstevel@tonic-gate } 7327c478bd9Sstevel@tonic-gate 7337c478bd9Sstevel@tonic-gate mutex_exit(lp); 7347c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 7357c478bd9Sstevel@tonic-gate 7367c478bd9Sstevel@tonic-gate /* 7377c478bd9Sstevel@tonic-gate * Now take our own segments. 7387c478bd9Sstevel@tonic-gate */ 7397c478bd9Sstevel@tonic-gate ASSERT(nseg >= VMEM_MINFREE); 7407c478bd9Sstevel@tonic-gate while (vmp->vm_nsegfree < VMEM_MINFREE) 7417c478bd9Sstevel@tonic-gate vmem_putseg(vmp, (vmem_seg_t *)(p + --nseg * vmem_seg_size)); 7427c478bd9Sstevel@tonic-gate 7437c478bd9Sstevel@tonic-gate /* 7447c478bd9Sstevel@tonic-gate * Give the remainder to charity. 7457c478bd9Sstevel@tonic-gate */ 7467c478bd9Sstevel@tonic-gate while (nseg > 0) 7477c478bd9Sstevel@tonic-gate vmem_putseg_global((vmem_seg_t *)(p + --nseg * vmem_seg_size)); 7487c478bd9Sstevel@tonic-gate 7497c478bd9Sstevel@tonic-gate return (1); 7507c478bd9Sstevel@tonic-gate } 7517c478bd9Sstevel@tonic-gate 7527c478bd9Sstevel@tonic-gate /* 7537c478bd9Sstevel@tonic-gate * Advance a walker from its previous position to 'afterme'. 7547c478bd9Sstevel@tonic-gate * Note: may drop and reacquire vmp->vm_lock. 7557c478bd9Sstevel@tonic-gate */ 7567c478bd9Sstevel@tonic-gate static void 7577c478bd9Sstevel@tonic-gate vmem_advance(vmem_t *vmp, vmem_seg_t *walker, vmem_seg_t *afterme) 7587c478bd9Sstevel@tonic-gate { 7597c478bd9Sstevel@tonic-gate vmem_seg_t *vprev = walker->vs_aprev; 7607c478bd9Sstevel@tonic-gate vmem_seg_t *vnext = walker->vs_anext; 7617c478bd9Sstevel@tonic-gate vmem_seg_t *vsp = NULL; 7627c478bd9Sstevel@tonic-gate 7637c478bd9Sstevel@tonic-gate VMEM_DELETE(walker, a); 7647c478bd9Sstevel@tonic-gate 7657c478bd9Sstevel@tonic-gate if (afterme != NULL) 7667c478bd9Sstevel@tonic-gate VMEM_INSERT(afterme, walker, a); 7677c478bd9Sstevel@tonic-gate 7687c478bd9Sstevel@tonic-gate /* 7697c478bd9Sstevel@tonic-gate * The walker segment's presence may have prevented its neighbors 7707c478bd9Sstevel@tonic-gate * from coalescing. If so, coalesce them now. 7717c478bd9Sstevel@tonic-gate */ 7727c478bd9Sstevel@tonic-gate if (vprev->vs_type == VMEM_FREE) { 7737c478bd9Sstevel@tonic-gate if (vnext->vs_type == VMEM_FREE) { 7747c478bd9Sstevel@tonic-gate ASSERT(vprev->vs_end == vnext->vs_start); 7757c478bd9Sstevel@tonic-gate vmem_freelist_delete(vmp, vnext); 7767c478bd9Sstevel@tonic-gate vmem_freelist_delete(vmp, vprev); 7777c478bd9Sstevel@tonic-gate vprev->vs_end = vnext->vs_end; 7787c478bd9Sstevel@tonic-gate vmem_freelist_insert(vmp, vprev); 7797c478bd9Sstevel@tonic-gate vmem_seg_destroy(vmp, vnext); 7807c478bd9Sstevel@tonic-gate } 7817c478bd9Sstevel@tonic-gate vsp = vprev; 7827c478bd9Sstevel@tonic-gate } else if (vnext->vs_type == VMEM_FREE) { 7837c478bd9Sstevel@tonic-gate vsp = vnext; 7847c478bd9Sstevel@tonic-gate } 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate /* 7877c478bd9Sstevel@tonic-gate * vsp could represent a complete imported span, 7887c478bd9Sstevel@tonic-gate * in which case we must return it to the source. 7897c478bd9Sstevel@tonic-gate */ 7907c478bd9Sstevel@tonic-gate if (vsp != NULL && vsp->vs_aprev->vs_import && 7917c478bd9Sstevel@tonic-gate vmp->vm_source_free != NULL && 7927c478bd9Sstevel@tonic-gate vsp->vs_aprev->vs_type == VMEM_SPAN && 7937c478bd9Sstevel@tonic-gate vsp->vs_anext->vs_type == VMEM_SPAN) { 7947c478bd9Sstevel@tonic-gate void *vaddr = (void *)vsp->vs_start; 7957c478bd9Sstevel@tonic-gate size_t size = VS_SIZE(vsp); 7967c478bd9Sstevel@tonic-gate ASSERT(size == VS_SIZE(vsp->vs_aprev)); 7977c478bd9Sstevel@tonic-gate vmem_freelist_delete(vmp, vsp); 7987c478bd9Sstevel@tonic-gate vmem_span_destroy(vmp, vsp); 7997c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 8007c478bd9Sstevel@tonic-gate vmp->vm_source_free(vmp->vm_source, vaddr, size); 8017c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 8027c478bd9Sstevel@tonic-gate } 8037c478bd9Sstevel@tonic-gate } 8047c478bd9Sstevel@tonic-gate 8057c478bd9Sstevel@tonic-gate /* 8067c478bd9Sstevel@tonic-gate * VM_NEXTFIT allocations deliberately cycle through all virtual addresses 8077c478bd9Sstevel@tonic-gate * in an arena, so that we avoid reusing addresses for as long as possible. 8087c478bd9Sstevel@tonic-gate * This helps to catch used-after-freed bugs. It's also the perfect policy 8097c478bd9Sstevel@tonic-gate * for allocating things like process IDs, where we want to cycle through 8107c478bd9Sstevel@tonic-gate * all values in order. 8117c478bd9Sstevel@tonic-gate */ 8127c478bd9Sstevel@tonic-gate static void * 8137c478bd9Sstevel@tonic-gate vmem_nextfit_alloc(vmem_t *vmp, size_t size, int vmflag) 8147c478bd9Sstevel@tonic-gate { 8157c478bd9Sstevel@tonic-gate vmem_seg_t *vsp, *rotor; 8167c478bd9Sstevel@tonic-gate uintptr_t addr; 8177c478bd9Sstevel@tonic-gate size_t realsize = P2ROUNDUP(size, vmp->vm_quantum); 8187c478bd9Sstevel@tonic-gate size_t vs_size; 8197c478bd9Sstevel@tonic-gate 8207c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate if (vmp->vm_nsegfree < VMEM_MINFREE && !vmem_populate(vmp, vmflag)) { 8237c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 8247c478bd9Sstevel@tonic-gate return (NULL); 8257c478bd9Sstevel@tonic-gate } 8267c478bd9Sstevel@tonic-gate 8277c478bd9Sstevel@tonic-gate /* 8287c478bd9Sstevel@tonic-gate * The common case is that the segment right after the rotor is free, 8297c478bd9Sstevel@tonic-gate * and large enough that extracting 'size' bytes won't change which 8307c478bd9Sstevel@tonic-gate * freelist it's on. In this case we can avoid a *lot* of work. 8317c478bd9Sstevel@tonic-gate * Instead of the normal vmem_seg_alloc(), we just advance the start 8327c478bd9Sstevel@tonic-gate * address of the victim segment. Instead of moving the rotor, we 8337c478bd9Sstevel@tonic-gate * create the new segment structure *behind the rotor*, which has 8347c478bd9Sstevel@tonic-gate * the same effect. And finally, we know we don't have to coalesce 8357c478bd9Sstevel@tonic-gate * the rotor's neighbors because the new segment lies between them. 8367c478bd9Sstevel@tonic-gate */ 8377c478bd9Sstevel@tonic-gate rotor = &vmp->vm_rotor; 8387c478bd9Sstevel@tonic-gate vsp = rotor->vs_anext; 8397c478bd9Sstevel@tonic-gate if (vsp->vs_type == VMEM_FREE && (vs_size = VS_SIZE(vsp)) > realsize && 8407c478bd9Sstevel@tonic-gate P2SAMEHIGHBIT(vs_size, vs_size - realsize)) { 8417c478bd9Sstevel@tonic-gate ASSERT(highbit(vs_size) == highbit(vs_size - realsize)); 8427c478bd9Sstevel@tonic-gate addr = vsp->vs_start; 8437c478bd9Sstevel@tonic-gate vsp->vs_start = addr + realsize; 8447c478bd9Sstevel@tonic-gate vmem_hash_insert(vmp, 8457c478bd9Sstevel@tonic-gate vmem_seg_create(vmp, rotor->vs_aprev, addr, addr + size)); 8467c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 8477c478bd9Sstevel@tonic-gate return ((void *)addr); 8487c478bd9Sstevel@tonic-gate } 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate /* 8517c478bd9Sstevel@tonic-gate * Starting at the rotor, look for a segment large enough to 8527c478bd9Sstevel@tonic-gate * satisfy the allocation. 8537c478bd9Sstevel@tonic-gate */ 8547c478bd9Sstevel@tonic-gate for (;;) { 8557c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_search.value.ui64++; 8567c478bd9Sstevel@tonic-gate if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size) 8577c478bd9Sstevel@tonic-gate break; 8587c478bd9Sstevel@tonic-gate vsp = vsp->vs_anext; 8597c478bd9Sstevel@tonic-gate if (vsp == rotor) { 8607c478bd9Sstevel@tonic-gate /* 8617c478bd9Sstevel@tonic-gate * We've come full circle. One possibility is that the 8627c478bd9Sstevel@tonic-gate * there's actually enough space, but the rotor itself 8637c478bd9Sstevel@tonic-gate * is preventing the allocation from succeeding because 8647c478bd9Sstevel@tonic-gate * it's sitting between two free segments. Therefore, 8657c478bd9Sstevel@tonic-gate * we advance the rotor and see if that liberates a 8667c478bd9Sstevel@tonic-gate * suitable segment. 8677c478bd9Sstevel@tonic-gate */ 8687c478bd9Sstevel@tonic-gate vmem_advance(vmp, rotor, rotor->vs_anext); 8697c478bd9Sstevel@tonic-gate vsp = rotor->vs_aprev; 8707c478bd9Sstevel@tonic-gate if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size) 8717c478bd9Sstevel@tonic-gate break; 8727c478bd9Sstevel@tonic-gate /* 8737c478bd9Sstevel@tonic-gate * If there's a lower arena we can import from, or it's 8747c478bd9Sstevel@tonic-gate * a VM_NOSLEEP allocation, let vmem_xalloc() handle it. 8757c478bd9Sstevel@tonic-gate * Otherwise, wait until another thread frees something. 8767c478bd9Sstevel@tonic-gate */ 8777c478bd9Sstevel@tonic-gate if (vmp->vm_source_alloc != NULL || 8787c478bd9Sstevel@tonic-gate (vmflag & VM_NOSLEEP)) { 8797c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 8807c478bd9Sstevel@tonic-gate return (vmem_xalloc(vmp, size, vmp->vm_quantum, 8817c478bd9Sstevel@tonic-gate 0, 0, NULL, NULL, vmflag & VM_KMFLAGS)); 8827c478bd9Sstevel@tonic-gate } 8837c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_wait.value.ui64++; 8847c478bd9Sstevel@tonic-gate cv_wait(&vmp->vm_cv, &vmp->vm_lock); 8857c478bd9Sstevel@tonic-gate vsp = rotor->vs_anext; 8867c478bd9Sstevel@tonic-gate } 8877c478bd9Sstevel@tonic-gate } 8887c478bd9Sstevel@tonic-gate 8897c478bd9Sstevel@tonic-gate /* 8907c478bd9Sstevel@tonic-gate * We found a segment. Extract enough space to satisfy the allocation. 8917c478bd9Sstevel@tonic-gate */ 8927c478bd9Sstevel@tonic-gate addr = vsp->vs_start; 8937c478bd9Sstevel@tonic-gate vsp = vmem_seg_alloc(vmp, vsp, addr, size); 8947c478bd9Sstevel@tonic-gate ASSERT(vsp->vs_type == VMEM_ALLOC && 8957c478bd9Sstevel@tonic-gate vsp->vs_start == addr && vsp->vs_end == addr + size); 8967c478bd9Sstevel@tonic-gate 8977c478bd9Sstevel@tonic-gate /* 8987c478bd9Sstevel@tonic-gate * Advance the rotor to right after the newly-allocated segment. 8997c478bd9Sstevel@tonic-gate * That's where the next VM_NEXTFIT allocation will begin searching. 9007c478bd9Sstevel@tonic-gate */ 9017c478bd9Sstevel@tonic-gate vmem_advance(vmp, rotor, vsp); 9027c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 9037c478bd9Sstevel@tonic-gate return ((void *)addr); 9047c478bd9Sstevel@tonic-gate } 9057c478bd9Sstevel@tonic-gate 9067c478bd9Sstevel@tonic-gate /* 9077c478bd9Sstevel@tonic-gate * Checks if vmp is guaranteed to have a size-byte buffer somewhere on its 9087c478bd9Sstevel@tonic-gate * freelist. If size is not a power-of-2, it can return a false-negative. 9097c478bd9Sstevel@tonic-gate * 9107c478bd9Sstevel@tonic-gate * Used to decide if a newly imported span is superfluous after re-acquiring 9117c478bd9Sstevel@tonic-gate * the arena lock. 9127c478bd9Sstevel@tonic-gate */ 9137c478bd9Sstevel@tonic-gate static int 9147c478bd9Sstevel@tonic-gate vmem_canalloc(vmem_t *vmp, size_t size) 9157c478bd9Sstevel@tonic-gate { 9167c478bd9Sstevel@tonic-gate int hb; 9177c478bd9Sstevel@tonic-gate int flist = 0; 9187c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&vmp->vm_lock)); 9197c478bd9Sstevel@tonic-gate 9207c478bd9Sstevel@tonic-gate if ((size & (size - 1)) == 0) 9217c478bd9Sstevel@tonic-gate flist = lowbit(P2ALIGN(vmp->vm_freemap, size)); 9227c478bd9Sstevel@tonic-gate else if ((hb = highbit(size)) < VMEM_FREELISTS) 9237c478bd9Sstevel@tonic-gate flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb)); 9247c478bd9Sstevel@tonic-gate 9257c478bd9Sstevel@tonic-gate return (flist); 9267c478bd9Sstevel@tonic-gate } 9277c478bd9Sstevel@tonic-gate 9287c478bd9Sstevel@tonic-gate /* 9297c478bd9Sstevel@tonic-gate * Allocate size bytes at offset phase from an align boundary such that the 9307c478bd9Sstevel@tonic-gate * resulting segment [addr, addr + size) is a subset of [minaddr, maxaddr) 9317c478bd9Sstevel@tonic-gate * that does not straddle a nocross-aligned boundary. 9327c478bd9Sstevel@tonic-gate */ 9337c478bd9Sstevel@tonic-gate void * 9347c478bd9Sstevel@tonic-gate vmem_xalloc(vmem_t *vmp, size_t size, size_t align_arg, size_t phase, 9357c478bd9Sstevel@tonic-gate size_t nocross, void *minaddr, void *maxaddr, int vmflag) 9367c478bd9Sstevel@tonic-gate { 9377c478bd9Sstevel@tonic-gate vmem_seg_t *vsp; 9387c478bd9Sstevel@tonic-gate vmem_seg_t *vbest = NULL; 9397c478bd9Sstevel@tonic-gate uintptr_t addr, taddr, start, end; 9407c478bd9Sstevel@tonic-gate uintptr_t align = (align_arg != 0) ? align_arg : vmp->vm_quantum; 9417c478bd9Sstevel@tonic-gate void *vaddr, *xvaddr = NULL; 9427c478bd9Sstevel@tonic-gate size_t xsize; 9437c478bd9Sstevel@tonic-gate int hb, flist, resv; 9447c478bd9Sstevel@tonic-gate uint32_t mtbf; 9457c478bd9Sstevel@tonic-gate 9467c478bd9Sstevel@tonic-gate if ((align | phase | nocross) & (vmp->vm_quantum - 1)) 9477c478bd9Sstevel@tonic-gate panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): " 9487c478bd9Sstevel@tonic-gate "parameters not vm_quantum aligned", 9497c478bd9Sstevel@tonic-gate (void *)vmp, size, align_arg, phase, nocross, 9507c478bd9Sstevel@tonic-gate minaddr, maxaddr, vmflag); 9517c478bd9Sstevel@tonic-gate 9527c478bd9Sstevel@tonic-gate if (nocross != 0 && 9537c478bd9Sstevel@tonic-gate (align > nocross || P2ROUNDUP(phase + size, align) > nocross)) 9547c478bd9Sstevel@tonic-gate panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): " 9557c478bd9Sstevel@tonic-gate "overconstrained allocation", 9567c478bd9Sstevel@tonic-gate (void *)vmp, size, align_arg, phase, nocross, 9577c478bd9Sstevel@tonic-gate minaddr, maxaddr, vmflag); 9587c478bd9Sstevel@tonic-gate 9597c478bd9Sstevel@tonic-gate if (phase >= align || (align & (align - 1)) != 0 || 9607c478bd9Sstevel@tonic-gate (nocross & (nocross - 1)) != 0) 9617c478bd9Sstevel@tonic-gate panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): " 9627c478bd9Sstevel@tonic-gate "parameters inconsistent or invalid", 9637c478bd9Sstevel@tonic-gate (void *)vmp, size, align_arg, phase, nocross, 9647c478bd9Sstevel@tonic-gate minaddr, maxaddr, vmflag); 9657c478bd9Sstevel@tonic-gate 9667c478bd9Sstevel@tonic-gate if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 && 9677c478bd9Sstevel@tonic-gate (vmflag & (VM_NOSLEEP | VM_PANIC)) == VM_NOSLEEP) 9687c478bd9Sstevel@tonic-gate return (NULL); 9697c478bd9Sstevel@tonic-gate 9707c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 9717c478bd9Sstevel@tonic-gate for (;;) { 9727c478bd9Sstevel@tonic-gate if (vmp->vm_nsegfree < VMEM_MINFREE && 9737c478bd9Sstevel@tonic-gate !vmem_populate(vmp, vmflag)) 9747c478bd9Sstevel@tonic-gate break; 9757c478bd9Sstevel@tonic-gate do_alloc: 9767c478bd9Sstevel@tonic-gate /* 9777c478bd9Sstevel@tonic-gate * highbit() returns the highest bit + 1, which is exactly 9787c478bd9Sstevel@tonic-gate * what we want: we want to search the first freelist whose 9797c478bd9Sstevel@tonic-gate * members are *definitely* large enough to satisfy our 9807c478bd9Sstevel@tonic-gate * allocation. However, there are certain cases in which we 9817c478bd9Sstevel@tonic-gate * want to look at the next-smallest freelist (which *might* 9827c478bd9Sstevel@tonic-gate * be able to satisfy the allocation): 9837c478bd9Sstevel@tonic-gate * 9847c478bd9Sstevel@tonic-gate * (1) The size is exactly a power of 2, in which case 9857c478bd9Sstevel@tonic-gate * the smaller freelist is always big enough; 9867c478bd9Sstevel@tonic-gate * 9877c478bd9Sstevel@tonic-gate * (2) All other freelists are empty; 9887c478bd9Sstevel@tonic-gate * 9897c478bd9Sstevel@tonic-gate * (3) We're in the highest possible freelist, which is 9907c478bd9Sstevel@tonic-gate * always empty (e.g. the 4GB freelist on 32-bit systems); 9917c478bd9Sstevel@tonic-gate * 9927c478bd9Sstevel@tonic-gate * (4) We're doing a best-fit or first-fit allocation. 9937c478bd9Sstevel@tonic-gate */ 9947c478bd9Sstevel@tonic-gate if ((size & (size - 1)) == 0) { 9957c478bd9Sstevel@tonic-gate flist = lowbit(P2ALIGN(vmp->vm_freemap, size)); 9967c478bd9Sstevel@tonic-gate } else { 9977c478bd9Sstevel@tonic-gate hb = highbit(size); 9987c478bd9Sstevel@tonic-gate if ((vmp->vm_freemap >> hb) == 0 || 9997c478bd9Sstevel@tonic-gate hb == VMEM_FREELISTS || 10007c478bd9Sstevel@tonic-gate (vmflag & (VM_BESTFIT | VM_FIRSTFIT))) 10017c478bd9Sstevel@tonic-gate hb--; 10027c478bd9Sstevel@tonic-gate flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb)); 10037c478bd9Sstevel@tonic-gate } 10047c478bd9Sstevel@tonic-gate 10057c478bd9Sstevel@tonic-gate for (vbest = NULL, vsp = (flist == 0) ? NULL : 10067c478bd9Sstevel@tonic-gate vmp->vm_freelist[flist - 1].vs_knext; 10077c478bd9Sstevel@tonic-gate vsp != NULL; vsp = vsp->vs_knext) { 10087c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_search.value.ui64++; 10097c478bd9Sstevel@tonic-gate if (vsp->vs_start == 0) { 10107c478bd9Sstevel@tonic-gate /* 10117c478bd9Sstevel@tonic-gate * We're moving up to a larger freelist, 10127c478bd9Sstevel@tonic-gate * so if we've already found a candidate, 10137c478bd9Sstevel@tonic-gate * the fit can't possibly get any better. 10147c478bd9Sstevel@tonic-gate */ 10157c478bd9Sstevel@tonic-gate if (vbest != NULL) 10167c478bd9Sstevel@tonic-gate break; 10177c478bd9Sstevel@tonic-gate /* 10187c478bd9Sstevel@tonic-gate * Find the next non-empty freelist. 10197c478bd9Sstevel@tonic-gate */ 10207c478bd9Sstevel@tonic-gate flist = lowbit(P2ALIGN(vmp->vm_freemap, 10217c478bd9Sstevel@tonic-gate VS_SIZE(vsp))); 10227c478bd9Sstevel@tonic-gate if (flist-- == 0) 10237c478bd9Sstevel@tonic-gate break; 10247c478bd9Sstevel@tonic-gate vsp = (vmem_seg_t *)&vmp->vm_freelist[flist]; 10257c478bd9Sstevel@tonic-gate ASSERT(vsp->vs_knext->vs_type == VMEM_FREE); 10267c478bd9Sstevel@tonic-gate continue; 10277c478bd9Sstevel@tonic-gate } 10287c478bd9Sstevel@tonic-gate if (vsp->vs_end - 1 < (uintptr_t)minaddr) 10297c478bd9Sstevel@tonic-gate continue; 10307c478bd9Sstevel@tonic-gate if (vsp->vs_start > (uintptr_t)maxaddr - 1) 10317c478bd9Sstevel@tonic-gate continue; 10327c478bd9Sstevel@tonic-gate start = MAX(vsp->vs_start, (uintptr_t)minaddr); 10337c478bd9Sstevel@tonic-gate end = MIN(vsp->vs_end - 1, (uintptr_t)maxaddr - 1) + 1; 10347c478bd9Sstevel@tonic-gate taddr = P2PHASEUP(start, align, phase); 10357c478bd9Sstevel@tonic-gate if (P2CROSS(taddr, taddr + size - 1, nocross)) 10367c478bd9Sstevel@tonic-gate taddr += 10377c478bd9Sstevel@tonic-gate P2ROUNDUP(P2NPHASE(taddr, nocross), align); 10387c478bd9Sstevel@tonic-gate if ((taddr - start) + size > end - start || 10397c478bd9Sstevel@tonic-gate (vbest != NULL && VS_SIZE(vsp) >= VS_SIZE(vbest))) 10407c478bd9Sstevel@tonic-gate continue; 10417c478bd9Sstevel@tonic-gate vbest = vsp; 10427c478bd9Sstevel@tonic-gate addr = taddr; 10437c478bd9Sstevel@tonic-gate if (!(vmflag & VM_BESTFIT) || VS_SIZE(vbest) == size) 10447c478bd9Sstevel@tonic-gate break; 10457c478bd9Sstevel@tonic-gate } 10467c478bd9Sstevel@tonic-gate if (vbest != NULL) 10477c478bd9Sstevel@tonic-gate break; 10487c478bd9Sstevel@tonic-gate ASSERT(xvaddr == NULL); 10497c478bd9Sstevel@tonic-gate if (size == 0) 10507c478bd9Sstevel@tonic-gate panic("vmem_xalloc(): size == 0"); 10517c478bd9Sstevel@tonic-gate if (vmp->vm_source_alloc != NULL && nocross == 0 && 10527c478bd9Sstevel@tonic-gate minaddr == NULL && maxaddr == NULL) { 10537c478bd9Sstevel@tonic-gate size_t aneeded, asize; 10547c478bd9Sstevel@tonic-gate size_t aquantum = MAX(vmp->vm_quantum, 10557c478bd9Sstevel@tonic-gate vmp->vm_source->vm_quantum); 10567c478bd9Sstevel@tonic-gate size_t aphase = phase; 1057aaa10e67Sha137994 if ((align > aquantum) && 1058aaa10e67Sha137994 !(vmp->vm_cflags & VMC_XALIGN)) { 10597c478bd9Sstevel@tonic-gate aphase = (P2PHASE(phase, aquantum) != 0) ? 10607c478bd9Sstevel@tonic-gate align - vmp->vm_quantum : align - aquantum; 10617c478bd9Sstevel@tonic-gate ASSERT(aphase >= phase); 10627c478bd9Sstevel@tonic-gate } 10637c478bd9Sstevel@tonic-gate aneeded = MAX(size + aphase, vmp->vm_min_import); 10647c478bd9Sstevel@tonic-gate asize = P2ROUNDUP(aneeded, aquantum); 10657c478bd9Sstevel@tonic-gate 10667c478bd9Sstevel@tonic-gate /* 10677c478bd9Sstevel@tonic-gate * Determine how many segment structures we'll consume. 10687c478bd9Sstevel@tonic-gate * The calculation must be precise because if we're 10697c478bd9Sstevel@tonic-gate * here on behalf of vmem_populate(), we are taking 10707c478bd9Sstevel@tonic-gate * segments from a very limited reserve. 10717c478bd9Sstevel@tonic-gate */ 10727c478bd9Sstevel@tonic-gate if (size == asize && !(vmp->vm_cflags & VMC_XALLOC)) 10737c478bd9Sstevel@tonic-gate resv = VMEM_SEGS_PER_SPAN_CREATE + 10747c478bd9Sstevel@tonic-gate VMEM_SEGS_PER_EXACT_ALLOC; 10757c478bd9Sstevel@tonic-gate else if (phase == 0 && 10767c478bd9Sstevel@tonic-gate align <= vmp->vm_source->vm_quantum) 10777c478bd9Sstevel@tonic-gate resv = VMEM_SEGS_PER_SPAN_CREATE + 10787c478bd9Sstevel@tonic-gate VMEM_SEGS_PER_LEFT_ALLOC; 10797c478bd9Sstevel@tonic-gate else 10807c478bd9Sstevel@tonic-gate resv = VMEM_SEGS_PER_ALLOC_MAX; 10817c478bd9Sstevel@tonic-gate 10827c478bd9Sstevel@tonic-gate ASSERT(vmp->vm_nsegfree >= resv); 10837c478bd9Sstevel@tonic-gate vmp->vm_nsegfree -= resv; /* reserve our segs */ 10847c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 10857c478bd9Sstevel@tonic-gate if (vmp->vm_cflags & VMC_XALLOC) { 10867c478bd9Sstevel@tonic-gate size_t oasize = asize; 10877c478bd9Sstevel@tonic-gate vaddr = ((vmem_ximport_t *) 10887c478bd9Sstevel@tonic-gate vmp->vm_source_alloc)(vmp->vm_source, 1089aaa10e67Sha137994 &asize, align, vmflag & VM_KMFLAGS); 10907c478bd9Sstevel@tonic-gate ASSERT(asize >= oasize); 10917c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(asize, 10927c478bd9Sstevel@tonic-gate vmp->vm_source->vm_quantum) == 0); 1093aaa10e67Sha137994 ASSERT(!(vmp->vm_cflags & VMC_XALIGN) || 1094aaa10e67Sha137994 IS_P2ALIGNED(vaddr, align)); 10957c478bd9Sstevel@tonic-gate } else { 10967c478bd9Sstevel@tonic-gate vaddr = vmp->vm_source_alloc(vmp->vm_source, 10977c478bd9Sstevel@tonic-gate asize, vmflag & VM_KMFLAGS); 10987c478bd9Sstevel@tonic-gate } 10997c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 11007c478bd9Sstevel@tonic-gate vmp->vm_nsegfree += resv; /* claim reservation */ 11017c478bd9Sstevel@tonic-gate aneeded = size + align - vmp->vm_quantum; 11027c478bd9Sstevel@tonic-gate aneeded = P2ROUNDUP(aneeded, vmp->vm_quantum); 11037c478bd9Sstevel@tonic-gate if (vaddr != NULL) { 11047c478bd9Sstevel@tonic-gate /* 11057c478bd9Sstevel@tonic-gate * Since we dropped the vmem lock while 11067c478bd9Sstevel@tonic-gate * calling the import function, other 11077c478bd9Sstevel@tonic-gate * threads could have imported space 11087c478bd9Sstevel@tonic-gate * and made our import unnecessary. In 11097c478bd9Sstevel@tonic-gate * order to save space, we return 11107c478bd9Sstevel@tonic-gate * excess imports immediately. 11117c478bd9Sstevel@tonic-gate */ 11127c478bd9Sstevel@tonic-gate if (asize > aneeded && 11137c478bd9Sstevel@tonic-gate vmp->vm_source_free != NULL && 11147c478bd9Sstevel@tonic-gate vmem_canalloc(vmp, aneeded)) { 11157c478bd9Sstevel@tonic-gate ASSERT(resv >= 11167c478bd9Sstevel@tonic-gate VMEM_SEGS_PER_MIDDLE_ALLOC); 11177c478bd9Sstevel@tonic-gate xvaddr = vaddr; 11187c478bd9Sstevel@tonic-gate xsize = asize; 11197c478bd9Sstevel@tonic-gate goto do_alloc; 11207c478bd9Sstevel@tonic-gate } 11217c478bd9Sstevel@tonic-gate vbest = vmem_span_create(vmp, vaddr, asize, 1); 11227c478bd9Sstevel@tonic-gate addr = P2PHASEUP(vbest->vs_start, align, phase); 11237c478bd9Sstevel@tonic-gate break; 11247c478bd9Sstevel@tonic-gate } else if (vmem_canalloc(vmp, aneeded)) { 11257c478bd9Sstevel@tonic-gate /* 11267c478bd9Sstevel@tonic-gate * Our import failed, but another thread 11277c478bd9Sstevel@tonic-gate * added sufficient free memory to the arena 11287c478bd9Sstevel@tonic-gate * to satisfy our request. Go back and 11297c478bd9Sstevel@tonic-gate * grab it. 11307c478bd9Sstevel@tonic-gate */ 11317c478bd9Sstevel@tonic-gate ASSERT(resv >= VMEM_SEGS_PER_MIDDLE_ALLOC); 11327c478bd9Sstevel@tonic-gate goto do_alloc; 11337c478bd9Sstevel@tonic-gate } 11347c478bd9Sstevel@tonic-gate } 11357c478bd9Sstevel@tonic-gate 11367c478bd9Sstevel@tonic-gate /* 11377c478bd9Sstevel@tonic-gate * If the requestor chooses to fail the allocation attempt 11387c478bd9Sstevel@tonic-gate * rather than reap wait and retry - get out of the loop. 11397c478bd9Sstevel@tonic-gate */ 11407c478bd9Sstevel@tonic-gate if (vmflag & VM_ABORT) 11417c478bd9Sstevel@tonic-gate break; 11427c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 11437c478bd9Sstevel@tonic-gate if (vmp->vm_cflags & VMC_IDENTIFIER) 11447c478bd9Sstevel@tonic-gate kmem_reap_idspace(); 11457c478bd9Sstevel@tonic-gate else 11467c478bd9Sstevel@tonic-gate kmem_reap(); 11477c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 11487c478bd9Sstevel@tonic-gate if (vmflag & VM_NOSLEEP) 11497c478bd9Sstevel@tonic-gate break; 11507c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_wait.value.ui64++; 11517c478bd9Sstevel@tonic-gate cv_wait(&vmp->vm_cv, &vmp->vm_lock); 11527c478bd9Sstevel@tonic-gate } 11537c478bd9Sstevel@tonic-gate if (vbest != NULL) { 11547c478bd9Sstevel@tonic-gate ASSERT(vbest->vs_type == VMEM_FREE); 11557c478bd9Sstevel@tonic-gate ASSERT(vbest->vs_knext != vbest); 11567c478bd9Sstevel@tonic-gate (void) vmem_seg_alloc(vmp, vbest, addr, size); 11577c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 11587c478bd9Sstevel@tonic-gate if (xvaddr) 11597c478bd9Sstevel@tonic-gate vmp->vm_source_free(vmp->vm_source, xvaddr, xsize); 11607c478bd9Sstevel@tonic-gate ASSERT(P2PHASE(addr, align) == phase); 11617c478bd9Sstevel@tonic-gate ASSERT(!P2CROSS(addr, addr + size - 1, nocross)); 11627c478bd9Sstevel@tonic-gate ASSERT(addr >= (uintptr_t)minaddr); 11637c478bd9Sstevel@tonic-gate ASSERT(addr + size - 1 <= (uintptr_t)maxaddr - 1); 11647c478bd9Sstevel@tonic-gate return ((void *)addr); 11657c478bd9Sstevel@tonic-gate } 11667c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_fail.value.ui64++; 11677c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 11687c478bd9Sstevel@tonic-gate if (vmflag & VM_PANIC) 11697c478bd9Sstevel@tonic-gate panic("vmem_xalloc(%p, %lu, %lu, %lu, %lu, %p, %p, %x): " 11707c478bd9Sstevel@tonic-gate "cannot satisfy mandatory allocation", 11717c478bd9Sstevel@tonic-gate (void *)vmp, size, align_arg, phase, nocross, 11727c478bd9Sstevel@tonic-gate minaddr, maxaddr, vmflag); 11737c478bd9Sstevel@tonic-gate ASSERT(xvaddr == NULL); 11747c478bd9Sstevel@tonic-gate return (NULL); 11757c478bd9Sstevel@tonic-gate } 11767c478bd9Sstevel@tonic-gate 11777c478bd9Sstevel@tonic-gate /* 11787c478bd9Sstevel@tonic-gate * Free the segment [vaddr, vaddr + size), where vaddr was a constrained 11797c478bd9Sstevel@tonic-gate * allocation. vmem_xalloc() and vmem_xfree() must always be paired because 11807c478bd9Sstevel@tonic-gate * both routines bypass the quantum caches. 11817c478bd9Sstevel@tonic-gate */ 11827c478bd9Sstevel@tonic-gate void 11837c478bd9Sstevel@tonic-gate vmem_xfree(vmem_t *vmp, void *vaddr, size_t size) 11847c478bd9Sstevel@tonic-gate { 11857c478bd9Sstevel@tonic-gate vmem_seg_t *vsp, *vnext, *vprev; 11867c478bd9Sstevel@tonic-gate 11877c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 11887c478bd9Sstevel@tonic-gate 11897c478bd9Sstevel@tonic-gate vsp = vmem_hash_delete(vmp, (uintptr_t)vaddr, size); 11907c478bd9Sstevel@tonic-gate vsp->vs_end = P2ROUNDUP(vsp->vs_end, vmp->vm_quantum); 11917c478bd9Sstevel@tonic-gate 11927c478bd9Sstevel@tonic-gate /* 11937c478bd9Sstevel@tonic-gate * Attempt to coalesce with the next segment. 11947c478bd9Sstevel@tonic-gate */ 11957c478bd9Sstevel@tonic-gate vnext = vsp->vs_anext; 11967c478bd9Sstevel@tonic-gate if (vnext->vs_type == VMEM_FREE) { 11977c478bd9Sstevel@tonic-gate ASSERT(vsp->vs_end == vnext->vs_start); 11987c478bd9Sstevel@tonic-gate vmem_freelist_delete(vmp, vnext); 11997c478bd9Sstevel@tonic-gate vsp->vs_end = vnext->vs_end; 12007c478bd9Sstevel@tonic-gate vmem_seg_destroy(vmp, vnext); 12017c478bd9Sstevel@tonic-gate } 12027c478bd9Sstevel@tonic-gate 12037c478bd9Sstevel@tonic-gate /* 12047c478bd9Sstevel@tonic-gate * Attempt to coalesce with the previous segment. 12057c478bd9Sstevel@tonic-gate */ 12067c478bd9Sstevel@tonic-gate vprev = vsp->vs_aprev; 12077c478bd9Sstevel@tonic-gate if (vprev->vs_type == VMEM_FREE) { 12087c478bd9Sstevel@tonic-gate ASSERT(vprev->vs_end == vsp->vs_start); 12097c478bd9Sstevel@tonic-gate vmem_freelist_delete(vmp, vprev); 12107c478bd9Sstevel@tonic-gate vprev->vs_end = vsp->vs_end; 12117c478bd9Sstevel@tonic-gate vmem_seg_destroy(vmp, vsp); 12127c478bd9Sstevel@tonic-gate vsp = vprev; 12137c478bd9Sstevel@tonic-gate } 12147c478bd9Sstevel@tonic-gate 12157c478bd9Sstevel@tonic-gate /* 12167c478bd9Sstevel@tonic-gate * If the entire span is free, return it to the source. 12177c478bd9Sstevel@tonic-gate */ 12187c478bd9Sstevel@tonic-gate if (vsp->vs_aprev->vs_import && vmp->vm_source_free != NULL && 12197c478bd9Sstevel@tonic-gate vsp->vs_aprev->vs_type == VMEM_SPAN && 12207c478bd9Sstevel@tonic-gate vsp->vs_anext->vs_type == VMEM_SPAN) { 12217c478bd9Sstevel@tonic-gate vaddr = (void *)vsp->vs_start; 12227c478bd9Sstevel@tonic-gate size = VS_SIZE(vsp); 12237c478bd9Sstevel@tonic-gate ASSERT(size == VS_SIZE(vsp->vs_aprev)); 12247c478bd9Sstevel@tonic-gate vmem_span_destroy(vmp, vsp); 12257c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 12267c478bd9Sstevel@tonic-gate vmp->vm_source_free(vmp->vm_source, vaddr, size); 12277c478bd9Sstevel@tonic-gate } else { 12287c478bd9Sstevel@tonic-gate vmem_freelist_insert(vmp, vsp); 12297c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 12307c478bd9Sstevel@tonic-gate } 12317c478bd9Sstevel@tonic-gate } 12327c478bd9Sstevel@tonic-gate 12337c478bd9Sstevel@tonic-gate /* 12347c478bd9Sstevel@tonic-gate * Allocate size bytes from arena vmp. Returns the allocated address 12357c478bd9Sstevel@tonic-gate * on success, NULL on failure. vmflag specifies VM_SLEEP or VM_NOSLEEP, 12367c478bd9Sstevel@tonic-gate * and may also specify best-fit, first-fit, or next-fit allocation policy 12377c478bd9Sstevel@tonic-gate * instead of the default instant-fit policy. VM_SLEEP allocations are 12387c478bd9Sstevel@tonic-gate * guaranteed to succeed. 12397c478bd9Sstevel@tonic-gate */ 12407c478bd9Sstevel@tonic-gate void * 12417c478bd9Sstevel@tonic-gate vmem_alloc(vmem_t *vmp, size_t size, int vmflag) 12427c478bd9Sstevel@tonic-gate { 12437c478bd9Sstevel@tonic-gate vmem_seg_t *vsp; 12447c478bd9Sstevel@tonic-gate uintptr_t addr; 12457c478bd9Sstevel@tonic-gate int hb; 12467c478bd9Sstevel@tonic-gate int flist = 0; 12477c478bd9Sstevel@tonic-gate uint32_t mtbf; 12487c478bd9Sstevel@tonic-gate 12497c478bd9Sstevel@tonic-gate if (size - 1 < vmp->vm_qcache_max) 12507c478bd9Sstevel@tonic-gate return (kmem_cache_alloc(vmp->vm_qcache[(size - 1) >> 12517c478bd9Sstevel@tonic-gate vmp->vm_qshift], vmflag & VM_KMFLAGS)); 12527c478bd9Sstevel@tonic-gate 12537c478bd9Sstevel@tonic-gate if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 && 12547c478bd9Sstevel@tonic-gate (vmflag & (VM_NOSLEEP | VM_PANIC)) == VM_NOSLEEP) 12557c478bd9Sstevel@tonic-gate return (NULL); 12567c478bd9Sstevel@tonic-gate 12577c478bd9Sstevel@tonic-gate if (vmflag & VM_NEXTFIT) 12587c478bd9Sstevel@tonic-gate return (vmem_nextfit_alloc(vmp, size, vmflag)); 12597c478bd9Sstevel@tonic-gate 12607c478bd9Sstevel@tonic-gate if (vmflag & (VM_BESTFIT | VM_FIRSTFIT)) 12617c478bd9Sstevel@tonic-gate return (vmem_xalloc(vmp, size, vmp->vm_quantum, 0, 0, 12627c478bd9Sstevel@tonic-gate NULL, NULL, vmflag)); 12637c478bd9Sstevel@tonic-gate 12647c478bd9Sstevel@tonic-gate /* 12657c478bd9Sstevel@tonic-gate * Unconstrained instant-fit allocation from the segment list. 12667c478bd9Sstevel@tonic-gate */ 12677c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 12687c478bd9Sstevel@tonic-gate 12697c478bd9Sstevel@tonic-gate if (vmp->vm_nsegfree >= VMEM_MINFREE || vmem_populate(vmp, vmflag)) { 12707c478bd9Sstevel@tonic-gate if ((size & (size - 1)) == 0) 12717c478bd9Sstevel@tonic-gate flist = lowbit(P2ALIGN(vmp->vm_freemap, size)); 12727c478bd9Sstevel@tonic-gate else if ((hb = highbit(size)) < VMEM_FREELISTS) 12737c478bd9Sstevel@tonic-gate flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb)); 12747c478bd9Sstevel@tonic-gate } 12757c478bd9Sstevel@tonic-gate 12767c478bd9Sstevel@tonic-gate if (flist-- == 0) { 12777c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 12787c478bd9Sstevel@tonic-gate return (vmem_xalloc(vmp, size, vmp->vm_quantum, 12797c478bd9Sstevel@tonic-gate 0, 0, NULL, NULL, vmflag)); 12807c478bd9Sstevel@tonic-gate } 12817c478bd9Sstevel@tonic-gate 12827c478bd9Sstevel@tonic-gate ASSERT(size <= (1UL << flist)); 12837c478bd9Sstevel@tonic-gate vsp = vmp->vm_freelist[flist].vs_knext; 12847c478bd9Sstevel@tonic-gate addr = vsp->vs_start; 12857c478bd9Sstevel@tonic-gate (void) vmem_seg_alloc(vmp, vsp, addr, size); 12867c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 12877c478bd9Sstevel@tonic-gate return ((void *)addr); 12887c478bd9Sstevel@tonic-gate } 12897c478bd9Sstevel@tonic-gate 12907c478bd9Sstevel@tonic-gate /* 12917c478bd9Sstevel@tonic-gate * Free the segment [vaddr, vaddr + size). 12927c478bd9Sstevel@tonic-gate */ 12937c478bd9Sstevel@tonic-gate void 12947c478bd9Sstevel@tonic-gate vmem_free(vmem_t *vmp, void *vaddr, size_t size) 12957c478bd9Sstevel@tonic-gate { 12967c478bd9Sstevel@tonic-gate if (size - 1 < vmp->vm_qcache_max) 12977c478bd9Sstevel@tonic-gate kmem_cache_free(vmp->vm_qcache[(size - 1) >> vmp->vm_qshift], 12987c478bd9Sstevel@tonic-gate vaddr); 12997c478bd9Sstevel@tonic-gate else 13007c478bd9Sstevel@tonic-gate vmem_xfree(vmp, vaddr, size); 13017c478bd9Sstevel@tonic-gate } 13027c478bd9Sstevel@tonic-gate 13037c478bd9Sstevel@tonic-gate /* 13047c478bd9Sstevel@tonic-gate * Determine whether arena vmp contains the segment [vaddr, vaddr + size). 13057c478bd9Sstevel@tonic-gate */ 13067c478bd9Sstevel@tonic-gate int 13077c478bd9Sstevel@tonic-gate vmem_contains(vmem_t *vmp, void *vaddr, size_t size) 13087c478bd9Sstevel@tonic-gate { 13097c478bd9Sstevel@tonic-gate uintptr_t start = (uintptr_t)vaddr; 13107c478bd9Sstevel@tonic-gate uintptr_t end = start + size; 13117c478bd9Sstevel@tonic-gate vmem_seg_t *vsp; 13127c478bd9Sstevel@tonic-gate vmem_seg_t *seg0 = &vmp->vm_seg0; 13137c478bd9Sstevel@tonic-gate 13147c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 13157c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_contains.value.ui64++; 13167c478bd9Sstevel@tonic-gate for (vsp = seg0->vs_knext; vsp != seg0; vsp = vsp->vs_knext) { 13177c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_contains_search.value.ui64++; 13187c478bd9Sstevel@tonic-gate ASSERT(vsp->vs_type == VMEM_SPAN); 13197c478bd9Sstevel@tonic-gate if (start >= vsp->vs_start && end - 1 <= vsp->vs_end - 1) 13207c478bd9Sstevel@tonic-gate break; 13217c478bd9Sstevel@tonic-gate } 13227c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 13237c478bd9Sstevel@tonic-gate return (vsp != seg0); 13247c478bd9Sstevel@tonic-gate } 13257c478bd9Sstevel@tonic-gate 13267c478bd9Sstevel@tonic-gate /* 13277c478bd9Sstevel@tonic-gate * Add the span [vaddr, vaddr + size) to arena vmp. 13287c478bd9Sstevel@tonic-gate */ 13297c478bd9Sstevel@tonic-gate void * 13307c478bd9Sstevel@tonic-gate vmem_add(vmem_t *vmp, void *vaddr, size_t size, int vmflag) 13317c478bd9Sstevel@tonic-gate { 13327c478bd9Sstevel@tonic-gate if (vaddr == NULL || size == 0) 1333*903a11ebSrh87107 panic("vmem_add(%p, %p, %lu): bad arguments", 1334*903a11ebSrh87107 (void *)vmp, vaddr, size); 13357c478bd9Sstevel@tonic-gate 13367c478bd9Sstevel@tonic-gate ASSERT(!vmem_contains(vmp, vaddr, size)); 13377c478bd9Sstevel@tonic-gate 13387c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 13397c478bd9Sstevel@tonic-gate if (vmem_populate(vmp, vmflag)) 13407c478bd9Sstevel@tonic-gate (void) vmem_span_create(vmp, vaddr, size, 0); 13417c478bd9Sstevel@tonic-gate else 13427c478bd9Sstevel@tonic-gate vaddr = NULL; 13437c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 13447c478bd9Sstevel@tonic-gate return (vaddr); 13457c478bd9Sstevel@tonic-gate } 13467c478bd9Sstevel@tonic-gate 13477c478bd9Sstevel@tonic-gate /* 13487c478bd9Sstevel@tonic-gate * Walk the vmp arena, applying func to each segment matching typemask. 13497c478bd9Sstevel@tonic-gate * If VMEM_REENTRANT is specified, the arena lock is dropped across each 13507c478bd9Sstevel@tonic-gate * call to func(); otherwise, it is held for the duration of vmem_walk() 13517c478bd9Sstevel@tonic-gate * to ensure a consistent snapshot. Note that VMEM_REENTRANT callbacks 13527c478bd9Sstevel@tonic-gate * are *not* necessarily consistent, so they may only be used when a hint 13537c478bd9Sstevel@tonic-gate * is adequate. 13547c478bd9Sstevel@tonic-gate */ 13557c478bd9Sstevel@tonic-gate void 13567c478bd9Sstevel@tonic-gate vmem_walk(vmem_t *vmp, int typemask, 13577c478bd9Sstevel@tonic-gate void (*func)(void *, void *, size_t), void *arg) 13587c478bd9Sstevel@tonic-gate { 13597c478bd9Sstevel@tonic-gate vmem_seg_t *vsp; 13607c478bd9Sstevel@tonic-gate vmem_seg_t *seg0 = &vmp->vm_seg0; 13617c478bd9Sstevel@tonic-gate vmem_seg_t walker; 13627c478bd9Sstevel@tonic-gate 13637c478bd9Sstevel@tonic-gate if (typemask & VMEM_WALKER) 13647c478bd9Sstevel@tonic-gate return; 13657c478bd9Sstevel@tonic-gate 13667c478bd9Sstevel@tonic-gate bzero(&walker, sizeof (walker)); 13677c478bd9Sstevel@tonic-gate walker.vs_type = VMEM_WALKER; 13687c478bd9Sstevel@tonic-gate 13697c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 13707c478bd9Sstevel@tonic-gate VMEM_INSERT(seg0, &walker, a); 13717c478bd9Sstevel@tonic-gate for (vsp = seg0->vs_anext; vsp != seg0; vsp = vsp->vs_anext) { 13727c478bd9Sstevel@tonic-gate if (vsp->vs_type & typemask) { 13737c478bd9Sstevel@tonic-gate void *start = (void *)vsp->vs_start; 13747c478bd9Sstevel@tonic-gate size_t size = VS_SIZE(vsp); 13757c478bd9Sstevel@tonic-gate if (typemask & VMEM_REENTRANT) { 13767c478bd9Sstevel@tonic-gate vmem_advance(vmp, &walker, vsp); 13777c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 13787c478bd9Sstevel@tonic-gate func(arg, start, size); 13797c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 13807c478bd9Sstevel@tonic-gate vsp = &walker; 13817c478bd9Sstevel@tonic-gate } else { 13827c478bd9Sstevel@tonic-gate func(arg, start, size); 13837c478bd9Sstevel@tonic-gate } 13847c478bd9Sstevel@tonic-gate } 13857c478bd9Sstevel@tonic-gate } 13867c478bd9Sstevel@tonic-gate vmem_advance(vmp, &walker, NULL); 13877c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 13887c478bd9Sstevel@tonic-gate } 13897c478bd9Sstevel@tonic-gate 13907c478bd9Sstevel@tonic-gate /* 13917c478bd9Sstevel@tonic-gate * Return the total amount of memory whose type matches typemask. Thus: 13927c478bd9Sstevel@tonic-gate * 13937c478bd9Sstevel@tonic-gate * typemask VMEM_ALLOC yields total memory allocated (in use). 13947c478bd9Sstevel@tonic-gate * typemask VMEM_FREE yields total memory free (available). 13957c478bd9Sstevel@tonic-gate * typemask (VMEM_ALLOC | VMEM_FREE) yields total arena size. 13967c478bd9Sstevel@tonic-gate */ 13977c478bd9Sstevel@tonic-gate size_t 13987c478bd9Sstevel@tonic-gate vmem_size(vmem_t *vmp, int typemask) 13997c478bd9Sstevel@tonic-gate { 14007c478bd9Sstevel@tonic-gate uint64_t size = 0; 14017c478bd9Sstevel@tonic-gate 14027c478bd9Sstevel@tonic-gate if (typemask & VMEM_ALLOC) 14037c478bd9Sstevel@tonic-gate size += vmp->vm_kstat.vk_mem_inuse.value.ui64; 14047c478bd9Sstevel@tonic-gate if (typemask & VMEM_FREE) 14057c478bd9Sstevel@tonic-gate size += vmp->vm_kstat.vk_mem_total.value.ui64 - 14067c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_mem_inuse.value.ui64; 14077c478bd9Sstevel@tonic-gate return ((size_t)size); 14087c478bd9Sstevel@tonic-gate } 14097c478bd9Sstevel@tonic-gate 14107c478bd9Sstevel@tonic-gate /* 14117c478bd9Sstevel@tonic-gate * Create an arena called name whose initial span is [base, base + size). 14127c478bd9Sstevel@tonic-gate * The arena's natural unit of currency is quantum, so vmem_alloc() 14137c478bd9Sstevel@tonic-gate * guarantees quantum-aligned results. The arena may import new spans 14147c478bd9Sstevel@tonic-gate * by invoking afunc() on source, and may return those spans by invoking 14157c478bd9Sstevel@tonic-gate * ffunc() on source. To make small allocations fast and scalable, 14167c478bd9Sstevel@tonic-gate * the arena offers high-performance caching for each integer multiple 14177c478bd9Sstevel@tonic-gate * of quantum up to qcache_max. 14187c478bd9Sstevel@tonic-gate */ 14197c478bd9Sstevel@tonic-gate static vmem_t * 14207c478bd9Sstevel@tonic-gate vmem_create_common(const char *name, void *base, size_t size, size_t quantum, 14217c478bd9Sstevel@tonic-gate void *(*afunc)(vmem_t *, size_t, int), 14227c478bd9Sstevel@tonic-gate void (*ffunc)(vmem_t *, void *, size_t), 14237c478bd9Sstevel@tonic-gate vmem_t *source, size_t qcache_max, int vmflag) 14247c478bd9Sstevel@tonic-gate { 14257c478bd9Sstevel@tonic-gate int i; 14267c478bd9Sstevel@tonic-gate size_t nqcache; 14277c478bd9Sstevel@tonic-gate vmem_t *vmp, *cur, **vmpp; 14287c478bd9Sstevel@tonic-gate vmem_seg_t *vsp; 14297c478bd9Sstevel@tonic-gate vmem_freelist_t *vfp; 14307c478bd9Sstevel@tonic-gate uint32_t id = atomic_add_32_nv(&vmem_id, 1); 14317c478bd9Sstevel@tonic-gate 14327c478bd9Sstevel@tonic-gate if (vmem_vmem_arena != NULL) { 14337c478bd9Sstevel@tonic-gate vmp = vmem_alloc(vmem_vmem_arena, sizeof (vmem_t), 14347c478bd9Sstevel@tonic-gate vmflag & VM_KMFLAGS); 14357c478bd9Sstevel@tonic-gate } else { 14367c478bd9Sstevel@tonic-gate ASSERT(id <= VMEM_INITIAL); 14377c478bd9Sstevel@tonic-gate vmp = &vmem0[id - 1]; 14387c478bd9Sstevel@tonic-gate } 14397c478bd9Sstevel@tonic-gate 14407c478bd9Sstevel@tonic-gate /* An identifier arena must inherit from another identifier arena */ 14417c478bd9Sstevel@tonic-gate ASSERT(source == NULL || ((source->vm_cflags & VMC_IDENTIFIER) == 14427c478bd9Sstevel@tonic-gate (vmflag & VMC_IDENTIFIER))); 14437c478bd9Sstevel@tonic-gate 14447c478bd9Sstevel@tonic-gate if (vmp == NULL) 14457c478bd9Sstevel@tonic-gate return (NULL); 14467c478bd9Sstevel@tonic-gate bzero(vmp, sizeof (vmem_t)); 14477c478bd9Sstevel@tonic-gate 14487c478bd9Sstevel@tonic-gate (void) snprintf(vmp->vm_name, VMEM_NAMELEN, "%s", name); 14497c478bd9Sstevel@tonic-gate mutex_init(&vmp->vm_lock, NULL, MUTEX_DEFAULT, NULL); 14507c478bd9Sstevel@tonic-gate cv_init(&vmp->vm_cv, NULL, CV_DEFAULT, NULL); 14517c478bd9Sstevel@tonic-gate vmp->vm_cflags = vmflag; 14527c478bd9Sstevel@tonic-gate vmflag &= VM_KMFLAGS; 14537c478bd9Sstevel@tonic-gate 14547c478bd9Sstevel@tonic-gate vmp->vm_quantum = quantum; 14557c478bd9Sstevel@tonic-gate vmp->vm_qshift = highbit(quantum) - 1; 14567c478bd9Sstevel@tonic-gate nqcache = MIN(qcache_max >> vmp->vm_qshift, VMEM_NQCACHE_MAX); 14577c478bd9Sstevel@tonic-gate 14587c478bd9Sstevel@tonic-gate for (i = 0; i <= VMEM_FREELISTS; i++) { 14597c478bd9Sstevel@tonic-gate vfp = &vmp->vm_freelist[i]; 14607c478bd9Sstevel@tonic-gate vfp->vs_end = 1UL << i; 14617c478bd9Sstevel@tonic-gate vfp->vs_knext = (vmem_seg_t *)(vfp + 1); 14627c478bd9Sstevel@tonic-gate vfp->vs_kprev = (vmem_seg_t *)(vfp - 1); 14637c478bd9Sstevel@tonic-gate } 14647c478bd9Sstevel@tonic-gate 14657c478bd9Sstevel@tonic-gate vmp->vm_freelist[0].vs_kprev = NULL; 14667c478bd9Sstevel@tonic-gate vmp->vm_freelist[VMEM_FREELISTS].vs_knext = NULL; 14677c478bd9Sstevel@tonic-gate vmp->vm_freelist[VMEM_FREELISTS].vs_end = 0; 14687c478bd9Sstevel@tonic-gate vmp->vm_hash_table = vmp->vm_hash0; 14697c478bd9Sstevel@tonic-gate vmp->vm_hash_mask = VMEM_HASH_INITIAL - 1; 14707c478bd9Sstevel@tonic-gate vmp->vm_hash_shift = highbit(vmp->vm_hash_mask); 14717c478bd9Sstevel@tonic-gate 14727c478bd9Sstevel@tonic-gate vsp = &vmp->vm_seg0; 14737c478bd9Sstevel@tonic-gate vsp->vs_anext = vsp; 14747c478bd9Sstevel@tonic-gate vsp->vs_aprev = vsp; 14757c478bd9Sstevel@tonic-gate vsp->vs_knext = vsp; 14767c478bd9Sstevel@tonic-gate vsp->vs_kprev = vsp; 14777c478bd9Sstevel@tonic-gate vsp->vs_type = VMEM_SPAN; 14787c478bd9Sstevel@tonic-gate 14797c478bd9Sstevel@tonic-gate vsp = &vmp->vm_rotor; 14807c478bd9Sstevel@tonic-gate vsp->vs_type = VMEM_ROTOR; 14817c478bd9Sstevel@tonic-gate VMEM_INSERT(&vmp->vm_seg0, vsp, a); 14827c478bd9Sstevel@tonic-gate 14837c478bd9Sstevel@tonic-gate bcopy(&vmem_kstat_template, &vmp->vm_kstat, sizeof (vmem_kstat_t)); 14847c478bd9Sstevel@tonic-gate 14857c478bd9Sstevel@tonic-gate vmp->vm_id = id; 14867c478bd9Sstevel@tonic-gate if (source != NULL) 14877c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_source_id.value.ui32 = source->vm_id; 14887c478bd9Sstevel@tonic-gate vmp->vm_source = source; 14897c478bd9Sstevel@tonic-gate vmp->vm_source_alloc = afunc; 14907c478bd9Sstevel@tonic-gate vmp->vm_source_free = ffunc; 14917c478bd9Sstevel@tonic-gate 14927c478bd9Sstevel@tonic-gate /* 14937c478bd9Sstevel@tonic-gate * Some arenas (like vmem_metadata and kmem_metadata) cannot 14947c478bd9Sstevel@tonic-gate * use quantum caching to lower fragmentation. Instead, we 14957c478bd9Sstevel@tonic-gate * increase their imports, giving a similar effect. 14967c478bd9Sstevel@tonic-gate */ 14977c478bd9Sstevel@tonic-gate if (vmp->vm_cflags & VMC_NO_QCACHE) { 14987c478bd9Sstevel@tonic-gate vmp->vm_min_import = 14997c478bd9Sstevel@tonic-gate VMEM_QCACHE_SLABSIZE(nqcache << vmp->vm_qshift); 15007c478bd9Sstevel@tonic-gate nqcache = 0; 15017c478bd9Sstevel@tonic-gate } 15027c478bd9Sstevel@tonic-gate 15037c478bd9Sstevel@tonic-gate if (nqcache != 0) { 15047c478bd9Sstevel@tonic-gate ASSERT(!(vmflag & VM_NOSLEEP)); 15057c478bd9Sstevel@tonic-gate vmp->vm_qcache_max = nqcache << vmp->vm_qshift; 15067c478bd9Sstevel@tonic-gate for (i = 0; i < nqcache; i++) { 15077c478bd9Sstevel@tonic-gate char buf[VMEM_NAMELEN + 21]; 15087c478bd9Sstevel@tonic-gate (void) sprintf(buf, "%s_%lu", vmp->vm_name, 15097c478bd9Sstevel@tonic-gate (i + 1) * quantum); 15107c478bd9Sstevel@tonic-gate vmp->vm_qcache[i] = kmem_cache_create(buf, 15117c478bd9Sstevel@tonic-gate (i + 1) * quantum, quantum, NULL, NULL, NULL, 15127c478bd9Sstevel@tonic-gate NULL, vmp, KMC_QCACHE | KMC_NOTOUCH); 15137c478bd9Sstevel@tonic-gate } 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate 15167c478bd9Sstevel@tonic-gate if ((vmp->vm_ksp = kstat_create("vmem", vmp->vm_id, vmp->vm_name, 15177c478bd9Sstevel@tonic-gate "vmem", KSTAT_TYPE_NAMED, sizeof (vmem_kstat_t) / 15187c478bd9Sstevel@tonic-gate sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL)) != NULL) { 15197c478bd9Sstevel@tonic-gate vmp->vm_ksp->ks_data = &vmp->vm_kstat; 15207c478bd9Sstevel@tonic-gate kstat_install(vmp->vm_ksp); 15217c478bd9Sstevel@tonic-gate } 15227c478bd9Sstevel@tonic-gate 15237c478bd9Sstevel@tonic-gate mutex_enter(&vmem_list_lock); 15247c478bd9Sstevel@tonic-gate vmpp = &vmem_list; 15257c478bd9Sstevel@tonic-gate while ((cur = *vmpp) != NULL) 15267c478bd9Sstevel@tonic-gate vmpp = &cur->vm_next; 15277c478bd9Sstevel@tonic-gate *vmpp = vmp; 15287c478bd9Sstevel@tonic-gate mutex_exit(&vmem_list_lock); 15297c478bd9Sstevel@tonic-gate 15307c478bd9Sstevel@tonic-gate if (vmp->vm_cflags & VMC_POPULATOR) { 15317c478bd9Sstevel@tonic-gate ASSERT(vmem_populators < VMEM_INITIAL); 15327c478bd9Sstevel@tonic-gate vmem_populator[atomic_add_32_nv(&vmem_populators, 1) - 1] = vmp; 15337c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 15347c478bd9Sstevel@tonic-gate (void) vmem_populate(vmp, vmflag | VM_PANIC); 15357c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 15367c478bd9Sstevel@tonic-gate } 15377c478bd9Sstevel@tonic-gate 15387c478bd9Sstevel@tonic-gate if ((base || size) && vmem_add(vmp, base, size, vmflag) == NULL) { 15397c478bd9Sstevel@tonic-gate vmem_destroy(vmp); 15407c478bd9Sstevel@tonic-gate return (NULL); 15417c478bd9Sstevel@tonic-gate } 15427c478bd9Sstevel@tonic-gate 15437c478bd9Sstevel@tonic-gate return (vmp); 15447c478bd9Sstevel@tonic-gate } 15457c478bd9Sstevel@tonic-gate 15467c478bd9Sstevel@tonic-gate vmem_t * 15477c478bd9Sstevel@tonic-gate vmem_xcreate(const char *name, void *base, size_t size, size_t quantum, 15487c478bd9Sstevel@tonic-gate vmem_ximport_t *afunc, vmem_free_t *ffunc, vmem_t *source, 15497c478bd9Sstevel@tonic-gate size_t qcache_max, int vmflag) 15507c478bd9Sstevel@tonic-gate { 15517c478bd9Sstevel@tonic-gate ASSERT(!(vmflag & (VMC_POPULATOR | VMC_XALLOC))); 15527c478bd9Sstevel@tonic-gate vmflag &= ~(VMC_POPULATOR | VMC_XALLOC); 15537c478bd9Sstevel@tonic-gate 15547c478bd9Sstevel@tonic-gate return (vmem_create_common(name, base, size, quantum, 15557c478bd9Sstevel@tonic-gate (vmem_alloc_t *)afunc, ffunc, source, qcache_max, 15567c478bd9Sstevel@tonic-gate vmflag | VMC_XALLOC)); 15577c478bd9Sstevel@tonic-gate } 15587c478bd9Sstevel@tonic-gate 15597c478bd9Sstevel@tonic-gate vmem_t * 15607c478bd9Sstevel@tonic-gate vmem_create(const char *name, void *base, size_t size, size_t quantum, 15617c478bd9Sstevel@tonic-gate vmem_alloc_t *afunc, vmem_free_t *ffunc, vmem_t *source, 15627c478bd9Sstevel@tonic-gate size_t qcache_max, int vmflag) 15637c478bd9Sstevel@tonic-gate { 1564aaa10e67Sha137994 ASSERT(!(vmflag & (VMC_XALLOC | VMC_XALIGN))); 1565aaa10e67Sha137994 vmflag &= ~(VMC_XALLOC | VMC_XALIGN); 15667c478bd9Sstevel@tonic-gate 15677c478bd9Sstevel@tonic-gate return (vmem_create_common(name, base, size, quantum, 15687c478bd9Sstevel@tonic-gate afunc, ffunc, source, qcache_max, vmflag)); 15697c478bd9Sstevel@tonic-gate } 15707c478bd9Sstevel@tonic-gate 15717c478bd9Sstevel@tonic-gate /* 15727c478bd9Sstevel@tonic-gate * Destroy arena vmp. 15737c478bd9Sstevel@tonic-gate */ 15747c478bd9Sstevel@tonic-gate void 15757c478bd9Sstevel@tonic-gate vmem_destroy(vmem_t *vmp) 15767c478bd9Sstevel@tonic-gate { 15777c478bd9Sstevel@tonic-gate vmem_t *cur, **vmpp; 15787c478bd9Sstevel@tonic-gate vmem_seg_t *seg0 = &vmp->vm_seg0; 15797c478bd9Sstevel@tonic-gate vmem_seg_t *vsp; 15807c478bd9Sstevel@tonic-gate size_t leaked; 15817c478bd9Sstevel@tonic-gate int i; 15827c478bd9Sstevel@tonic-gate 15837c478bd9Sstevel@tonic-gate mutex_enter(&vmem_list_lock); 15847c478bd9Sstevel@tonic-gate vmpp = &vmem_list; 15857c478bd9Sstevel@tonic-gate while ((cur = *vmpp) != vmp) 15867c478bd9Sstevel@tonic-gate vmpp = &cur->vm_next; 15877c478bd9Sstevel@tonic-gate *vmpp = vmp->vm_next; 15887c478bd9Sstevel@tonic-gate mutex_exit(&vmem_list_lock); 15897c478bd9Sstevel@tonic-gate 15907c478bd9Sstevel@tonic-gate for (i = 0; i < VMEM_NQCACHE_MAX; i++) 15917c478bd9Sstevel@tonic-gate if (vmp->vm_qcache[i]) 15927c478bd9Sstevel@tonic-gate kmem_cache_destroy(vmp->vm_qcache[i]); 15937c478bd9Sstevel@tonic-gate 15947c478bd9Sstevel@tonic-gate leaked = vmem_size(vmp, VMEM_ALLOC); 15957c478bd9Sstevel@tonic-gate if (leaked != 0) 15967c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "vmem_destroy('%s'): leaked %lu %s", 15977c478bd9Sstevel@tonic-gate vmp->vm_name, leaked, (vmp->vm_cflags & VMC_IDENTIFIER) ? 15987c478bd9Sstevel@tonic-gate "identifiers" : "bytes"); 15997c478bd9Sstevel@tonic-gate 16007c478bd9Sstevel@tonic-gate if (vmp->vm_hash_table != vmp->vm_hash0) 16017c478bd9Sstevel@tonic-gate vmem_free(vmem_hash_arena, vmp->vm_hash_table, 16027c478bd9Sstevel@tonic-gate (vmp->vm_hash_mask + 1) * sizeof (void *)); 16037c478bd9Sstevel@tonic-gate 16047c478bd9Sstevel@tonic-gate /* 16057c478bd9Sstevel@tonic-gate * Give back the segment structures for anything that's left in the 16067c478bd9Sstevel@tonic-gate * arena, e.g. the primary spans and their free segments. 16077c478bd9Sstevel@tonic-gate */ 16087c478bd9Sstevel@tonic-gate VMEM_DELETE(&vmp->vm_rotor, a); 16097c478bd9Sstevel@tonic-gate for (vsp = seg0->vs_anext; vsp != seg0; vsp = vsp->vs_anext) 16107c478bd9Sstevel@tonic-gate vmem_putseg_global(vsp); 16117c478bd9Sstevel@tonic-gate 16127c478bd9Sstevel@tonic-gate while (vmp->vm_nsegfree > 0) 16137c478bd9Sstevel@tonic-gate vmem_putseg_global(vmem_getseg(vmp)); 16147c478bd9Sstevel@tonic-gate 16157c478bd9Sstevel@tonic-gate kstat_delete(vmp->vm_ksp); 16167c478bd9Sstevel@tonic-gate 16177c478bd9Sstevel@tonic-gate mutex_destroy(&vmp->vm_lock); 16187c478bd9Sstevel@tonic-gate cv_destroy(&vmp->vm_cv); 16197c478bd9Sstevel@tonic-gate vmem_free(vmem_vmem_arena, vmp, sizeof (vmem_t)); 16207c478bd9Sstevel@tonic-gate } 16217c478bd9Sstevel@tonic-gate 16227c478bd9Sstevel@tonic-gate /* 16237c478bd9Sstevel@tonic-gate * Resize vmp's hash table to keep the average lookup depth near 1.0. 16247c478bd9Sstevel@tonic-gate */ 16257c478bd9Sstevel@tonic-gate static void 16267c478bd9Sstevel@tonic-gate vmem_hash_rescale(vmem_t *vmp) 16277c478bd9Sstevel@tonic-gate { 16287c478bd9Sstevel@tonic-gate vmem_seg_t **old_table, **new_table, *vsp; 16297c478bd9Sstevel@tonic-gate size_t old_size, new_size, h, nseg; 16307c478bd9Sstevel@tonic-gate 16317c478bd9Sstevel@tonic-gate nseg = (size_t)(vmp->vm_kstat.vk_alloc.value.ui64 - 16327c478bd9Sstevel@tonic-gate vmp->vm_kstat.vk_free.value.ui64); 16337c478bd9Sstevel@tonic-gate 16347c478bd9Sstevel@tonic-gate new_size = MAX(VMEM_HASH_INITIAL, 1 << (highbit(3 * nseg + 4) - 2)); 16357c478bd9Sstevel@tonic-gate old_size = vmp->vm_hash_mask + 1; 16367c478bd9Sstevel@tonic-gate 16377c478bd9Sstevel@tonic-gate if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 16387c478bd9Sstevel@tonic-gate return; 16397c478bd9Sstevel@tonic-gate 16407c478bd9Sstevel@tonic-gate new_table = vmem_alloc(vmem_hash_arena, new_size * sizeof (void *), 16417c478bd9Sstevel@tonic-gate VM_NOSLEEP); 16427c478bd9Sstevel@tonic-gate if (new_table == NULL) 16437c478bd9Sstevel@tonic-gate return; 16447c478bd9Sstevel@tonic-gate bzero(new_table, new_size * sizeof (void *)); 16457c478bd9Sstevel@tonic-gate 16467c478bd9Sstevel@tonic-gate mutex_enter(&vmp->vm_lock); 16477c478bd9Sstevel@tonic-gate 16487c478bd9Sstevel@tonic-gate old_size = vmp->vm_hash_mask + 1; 16497c478bd9Sstevel@tonic-gate old_table = vmp->vm_hash_table; 16507c478bd9Sstevel@tonic-gate 16517c478bd9Sstevel@tonic-gate vmp->vm_hash_mask = new_size - 1; 16527c478bd9Sstevel@tonic-gate vmp->vm_hash_table = new_table; 16537c478bd9Sstevel@tonic-gate vmp->vm_hash_shift = highbit(vmp->vm_hash_mask); 16547c478bd9Sstevel@tonic-gate 16557c478bd9Sstevel@tonic-gate for (h = 0; h < old_size; h++) { 16567c478bd9Sstevel@tonic-gate vsp = old_table[h]; 16577c478bd9Sstevel@tonic-gate while (vsp != NULL) { 16587c478bd9Sstevel@tonic-gate uintptr_t addr = vsp->vs_start; 16597c478bd9Sstevel@tonic-gate vmem_seg_t *next_vsp = vsp->vs_knext; 16607c478bd9Sstevel@tonic-gate vmem_seg_t **hash_bucket = VMEM_HASH(vmp, addr); 16617c478bd9Sstevel@tonic-gate vsp->vs_knext = *hash_bucket; 16627c478bd9Sstevel@tonic-gate *hash_bucket = vsp; 16637c478bd9Sstevel@tonic-gate vsp = next_vsp; 16647c478bd9Sstevel@tonic-gate } 16657c478bd9Sstevel@tonic-gate } 16667c478bd9Sstevel@tonic-gate 16677c478bd9Sstevel@tonic-gate mutex_exit(&vmp->vm_lock); 16687c478bd9Sstevel@tonic-gate 16697c478bd9Sstevel@tonic-gate if (old_table != vmp->vm_hash0) 16707c478bd9Sstevel@tonic-gate vmem_free(vmem_hash_arena, old_table, 16717c478bd9Sstevel@tonic-gate old_size * sizeof (void *)); 16727c478bd9Sstevel@tonic-gate } 16737c478bd9Sstevel@tonic-gate 16747c478bd9Sstevel@tonic-gate /* 16757c478bd9Sstevel@tonic-gate * Perform periodic maintenance on all vmem arenas. 16767c478bd9Sstevel@tonic-gate */ 16777c478bd9Sstevel@tonic-gate void 16787c478bd9Sstevel@tonic-gate vmem_update(void *dummy) 16797c478bd9Sstevel@tonic-gate { 16807c478bd9Sstevel@tonic-gate vmem_t *vmp; 16817c478bd9Sstevel@tonic-gate 16827c478bd9Sstevel@tonic-gate mutex_enter(&vmem_list_lock); 16837c478bd9Sstevel@tonic-gate for (vmp = vmem_list; vmp != NULL; vmp = vmp->vm_next) { 16847c478bd9Sstevel@tonic-gate /* 16857c478bd9Sstevel@tonic-gate * If threads are waiting for resources, wake them up 16867c478bd9Sstevel@tonic-gate * periodically so they can issue another kmem_reap() 16877c478bd9Sstevel@tonic-gate * to reclaim resources cached by the slab allocator. 16887c478bd9Sstevel@tonic-gate */ 16897c478bd9Sstevel@tonic-gate cv_broadcast(&vmp->vm_cv); 16907c478bd9Sstevel@tonic-gate 16917c478bd9Sstevel@tonic-gate /* 16927c478bd9Sstevel@tonic-gate * Rescale the hash table to keep the hash chains short. 16937c478bd9Sstevel@tonic-gate */ 16947c478bd9Sstevel@tonic-gate vmem_hash_rescale(vmp); 16957c478bd9Sstevel@tonic-gate } 16967c478bd9Sstevel@tonic-gate mutex_exit(&vmem_list_lock); 16977c478bd9Sstevel@tonic-gate 16987c478bd9Sstevel@tonic-gate (void) timeout(vmem_update, dummy, vmem_update_interval * hz); 16997c478bd9Sstevel@tonic-gate } 17007c478bd9Sstevel@tonic-gate 17017c478bd9Sstevel@tonic-gate /* 17027c478bd9Sstevel@tonic-gate * Prepare vmem for use. 17037c478bd9Sstevel@tonic-gate */ 17047c478bd9Sstevel@tonic-gate vmem_t * 17057c478bd9Sstevel@tonic-gate vmem_init(const char *heap_name, 17067c478bd9Sstevel@tonic-gate void *heap_start, size_t heap_size, size_t heap_quantum, 17077c478bd9Sstevel@tonic-gate void *(*heap_alloc)(vmem_t *, size_t, int), 17087c478bd9Sstevel@tonic-gate void (*heap_free)(vmem_t *, void *, size_t)) 17097c478bd9Sstevel@tonic-gate { 17107c478bd9Sstevel@tonic-gate uint32_t id; 17117c478bd9Sstevel@tonic-gate int nseg = VMEM_SEG_INITIAL; 17127c478bd9Sstevel@tonic-gate vmem_t *heap; 17137c478bd9Sstevel@tonic-gate 17147c478bd9Sstevel@tonic-gate while (--nseg >= 0) 17157c478bd9Sstevel@tonic-gate vmem_putseg_global(&vmem_seg0[nseg]); 17167c478bd9Sstevel@tonic-gate 17177c478bd9Sstevel@tonic-gate heap = vmem_create(heap_name, 17187c478bd9Sstevel@tonic-gate heap_start, heap_size, heap_quantum, 17197c478bd9Sstevel@tonic-gate NULL, NULL, NULL, 0, 17207c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_POPULATOR); 17217c478bd9Sstevel@tonic-gate 17227c478bd9Sstevel@tonic-gate vmem_metadata_arena = vmem_create("vmem_metadata", 17237c478bd9Sstevel@tonic-gate NULL, 0, heap_quantum, 17247c478bd9Sstevel@tonic-gate vmem_alloc, vmem_free, heap, 8 * heap_quantum, 17257c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_POPULATOR | VMC_NO_QCACHE); 17267c478bd9Sstevel@tonic-gate 17277c478bd9Sstevel@tonic-gate vmem_seg_arena = vmem_create("vmem_seg", 17287c478bd9Sstevel@tonic-gate NULL, 0, heap_quantum, 17297c478bd9Sstevel@tonic-gate heap_alloc, heap_free, vmem_metadata_arena, 0, 17307c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_POPULATOR); 17317c478bd9Sstevel@tonic-gate 17327c478bd9Sstevel@tonic-gate vmem_hash_arena = vmem_create("vmem_hash", 17337c478bd9Sstevel@tonic-gate NULL, 0, 8, 17347c478bd9Sstevel@tonic-gate heap_alloc, heap_free, vmem_metadata_arena, 0, 17357c478bd9Sstevel@tonic-gate VM_SLEEP); 17367c478bd9Sstevel@tonic-gate 17377c478bd9Sstevel@tonic-gate vmem_vmem_arena = vmem_create("vmem_vmem", 17387c478bd9Sstevel@tonic-gate vmem0, sizeof (vmem0), 1, 17397c478bd9Sstevel@tonic-gate heap_alloc, heap_free, vmem_metadata_arena, 0, 17407c478bd9Sstevel@tonic-gate VM_SLEEP); 17417c478bd9Sstevel@tonic-gate 17427c478bd9Sstevel@tonic-gate for (id = 0; id < vmem_id; id++) 17437c478bd9Sstevel@tonic-gate (void) vmem_xalloc(vmem_vmem_arena, sizeof (vmem_t), 17447c478bd9Sstevel@tonic-gate 1, 0, 0, &vmem0[id], &vmem0[id + 1], 17457c478bd9Sstevel@tonic-gate VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 17467c478bd9Sstevel@tonic-gate 17477c478bd9Sstevel@tonic-gate return (heap); 17487c478bd9Sstevel@tonic-gate } 1749