/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * seg_kmem is the primary kernel memory segment driver. It * maps the kernel heap [kernelheap, ekernelheap), module text, * and all memory which was allocated before the VM was initialized * into kas. * * Pages which belong to seg_kmem are hashed into &kvp vnode at * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1. * They must never be paged out since segkmem_fault() is a no-op to * prevent recursive faults. * * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86 * supports relocation the #ifdef kludges can be removed. * * seg_kmem pages may be subject to relocation by page_relocate(), * provided that the HAT supports it; if this is so, segkmem_reloc * will be set to a nonzero value. All boot time allocated memory as * well as static memory is considered off limits to relocation. * Pages are "relocatable" if p_state does not have P_NORELOC set, so * we request P_NORELOC pages for memory that isn't safe to relocate. * * The kernel heap is logically divided up into four pieces: * * heap32_arena is for allocations that require 32-bit absolute * virtual addresses (e.g. code that uses 32-bit pointers/offsets). * * heap_core is for allocations that require 2GB *relative* * offsets; in other words all memory from heap_core is within * 2GB of all other memory from the same arena. This is a requirement * of the addressing modes of some processors in supervisor code. * * heap_arena is the general heap arena. * * static_arena is the static memory arena. Allocations from it * are not subject to relocation so it is safe to use the memory * physical address as well as the virtual address (e.g. the VA to * PA translations are static). Caches may import from static_arena; * all other static memory allocations should use static_alloc_arena. * * On some platforms which have limited virtual address space, seg_kmem * may share [kernelheap, ekernelheap) with seg_kp; if this is so, * segkp_bitmap is non-NULL, and each bit represents a page of virtual * address space which is actually seg_kp mapped. */ extern ulong_t *segkp_bitmap; /* Is set if segkp is from the kernel heap */ char *kernelheap; /* start of primary kernel heap */ char *ekernelheap; /* end of primary kernel heap */ struct seg kvseg; /* primary kernel heap segment */ struct seg kvseg_core; /* "core" kernel heap segment */ vmem_t *heap_arena; /* primary kernel heap arena */ vmem_t *heap_core_arena; /* core kernel heap arena */ char *heap_core_base; /* start of core kernel heap arena */ char *heap_lp_base; /* start of kernel large page heap arena */ char *heap_lp_end; /* end of kernel large page heap arena */ vmem_t *hat_memload_arena; /* HAT translation data */ struct seg kvseg32; /* 32-bit kernel heap segment */ vmem_t *heap32_arena; /* 32-bit kernel heap arena */ vmem_t *heaptext_arena; /* heaptext arena */ struct as kas; /* kernel address space */ struct vnode kvp; /* vnode for all segkmem pages */ int segkmem_reloc; /* enable/disable relocatable segkmem pages */ vmem_t *static_arena; /* arena for caches to import static memory */ vmem_t *static_alloc_arena; /* arena for allocating static memory */ /* * seg_kmem driver can map part of the kernel heap with large pages. * Currently this functionality is implemented for sparc platforms only. * * The large page size "segkmem_lpsize" for kernel heap is selected in the * platform specific code. It can also be modified via /etc/system file. * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to * match segkmem_lpsize. * * At boot time we carve from kernel heap arena a range of virtual addresses * that will be used for large page mappings. This range [heap_lp_base, * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also * create "kmem_lp_arena" that caches memory already backed up by large * pages. kmem_lp_arena imports virtual segments from heap_lp_arena. */ size_t segkmem_lpsize; static uint_t segkmem_lpshift = PAGESHIFT; size_t segkmem_kmemlp_quantum = 0x400000; /* 4MB */ size_t segkmem_heaplp_quantum; vmem_t *heap_lp_arena; static vmem_t *kmem_lp_arena; static vmem_t *segkmem_ppa_arena; static segkmem_lpcb_t segkmem_lpcb; /* * We use "segkmem_kmemlp_max" to limit the total amount of physical memory * consumed by the large page heap. By default this parameter is set to 1/8 of * physmem but can be adjusted through /etc/system either directly or * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem * we allow for large page heap. */ size_t segkmem_kmemlp_max; static uint_t segkmem_kmemlp_pcnt; /* * Getting large pages for kernel heap could be problematic due to * physical memory fragmentation. That's why we allow to preallocate * "segkmem_kmemlp_min" bytes at boot time. */ static size_t segkmem_kmemlp_min; /* * Throttling is used to avoid expensive tries to allocate large pages * for kernel heap when a lot of succesive attempts to do so fail. */ static ulong_t segkmem_lpthrottle_max = 0x400000; static ulong_t segkmem_lpthrottle_start = 0x40; static ulong_t segkmem_use_lpthrottle = 1; /* * Freed pages accumulate on a garbage list until segkmem is ready, * at which point we call segkmem_gc() to free it all. */ typedef struct segkmem_gc_list { struct segkmem_gc_list *gc_next; vmem_t *gc_arena; size_t gc_size; } segkmem_gc_list_t; static segkmem_gc_list_t *segkmem_gc_list; /* * Allocations from the hat_memload arena add VM_MEMLOAD to their * vmflags so that segkmem_xalloc() can inform the hat layer that it needs * to take steps to prevent infinite recursion. HAT allocations also * must be non-relocatable to prevent recursive page faults. */ static void * hat_memload_alloc(vmem_t *vmp, size_t size, int flags) { flags |= (VM_MEMLOAD | VM_NORELOC); return (segkmem_alloc(vmp, size, flags)); } /* * Allocations from static_arena arena (or any other arena that uses * segkmem_alloc_permanent()) require non-relocatable (permanently * wired) memory pages, since these pages are referenced by physical * as well as virtual address. */ void * segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags) { return (segkmem_alloc(vmp, size, flags | VM_NORELOC)); } /* * Initialize kernel heap boundaries. */ void kernelheap_init( void *heap_start, void *heap_end, char *first_avail, void *core_start, void *core_end) { uintptr_t textbase; size_t core_size; size_t heap_size; vmem_t *heaptext_parent; size_t heap_lp_size = 0; kernelheap = heap_start; ekernelheap = heap_end; #ifdef __sparc heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4); heap_lp_base = ekernelheap - heap_lp_size; heap_lp_end = heap_lp_base + heap_lp_size; #endif /* __sparc */ /* * If this platform has a 'core' heap area, then the space for * overflow module text should be carved out of the end of that * heap. Otherwise, it gets carved out of the general purpose * heap. */ core_size = (uintptr_t)core_end - (uintptr_t)core_start; if (core_size > 0) { ASSERT(core_size >= HEAPTEXT_SIZE); textbase = (uintptr_t)core_end - HEAPTEXT_SIZE; core_size -= HEAPTEXT_SIZE; } #ifndef __sparc else { ekernelheap -= HEAPTEXT_SIZE; textbase = (uintptr_t)ekernelheap; } #endif heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap; heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE, segkmem_alloc, segkmem_free); if (core_size > 0) { heap_core_arena = vmem_create("heap_core", core_start, core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP); heap_core_base = core_start; } else { heap_core_arena = heap_arena; heap_core_base = kernelheap; } /* * reserve space for the large page heap. If large pages for kernel * heap is enabled large page heap arean will be created later in the * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated * range will be returned back to the heap_arena. */ if (heap_lp_size) { (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0, heap_lp_base, heap_lp_end, VM_NOSLEEP | VM_BESTFIT | VM_PANIC); } /* * Remove the already-spoken-for memory range [kernelheap, first_avail). */ (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE, 0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC); #ifdef __sparc heap32_arena = vmem_create("heap32", (void *)SYSBASE32, SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP); textbase = SYSLIMIT32 - HEAPTEXT_SIZE; heaptext_parent = NULL; #else /* __sparc */ heap32_arena = heap_core_arena; heaptext_parent = heap_core_arena; #endif /* __sparc */ heaptext_arena = vmem_create("heaptext", (void *)textbase, HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP); /* * Create a set of arenas for memory with static translations * (e.g. VA -> PA translations cannot change). Since using * kernel pages by physical address implies it isn't safe to * walk across page boundaries, the static_arena quantum must * be PAGESIZE. Any kmem caches that require static memory * should source from static_arena, while direct allocations * should only use static_alloc_arena. */ static_arena = vmem_create("static", NULL, 0, PAGESIZE, segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); static_alloc_arena = vmem_create("static_alloc", NULL, 0, sizeof (uint64_t), vmem_alloc, vmem_free, static_arena, 0, VM_SLEEP); /* * Create an arena for translation data (ptes, hmes, or hblks). * We need an arena for this because hat_memload() is essential * to vmem_populate() (see comments in common/os/vmem.c). * * Note: any kmem cache that allocates from hat_memload_arena * must be created as a KMC_NOHASH cache (i.e. no external slab * and bufctl structures to allocate) so that slab creation doesn't * require anything more than a single vmem_alloc(). */ hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE, hat_memload_alloc, segkmem_free, heap_arena, 0, VM_SLEEP | VMC_POPULATOR); } /* * Grow kernel heap downward. */ void kernelheap_extend(void *range_start, void *range_end) { size_t len = (uintptr_t)range_end - (uintptr_t)range_start; ASSERT(range_start < range_end && range_end == kernelheap); if (vmem_add(heap_arena, range_start, len, VM_NOSLEEP) == NULL) { cmn_err(CE_WARN, "Could not grow kernel heap below 0x%p", (void *)kernelheap); } else { kernelheap = range_start; } } void boot_mapin(caddr_t addr, size_t size) { caddr_t eaddr; page_t *pp; pfn_t pfnum; if (page_resv(btop(size), KM_NOSLEEP) == 0) panic("boot_mapin: page_resv failed"); for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { pfnum = va_to_pfn(addr); if ((pp = page_numtopp_nolock(pfnum)) == NULL) panic("boot_mapin(): No pp for pfnum = %lx", pfnum); /* * must break up any large pages that may have constituent * pages being utilized for BOP_ALLOC()'s before calling * page_numtopp().The locking code (ie. page_reclaim()) * can't handle them */ if (pp->p_szc != 0) page_boot_demote(pp); pp = page_numtopp(pfnum, SE_EXCL); if (pp == NULL || PP_ISFREE(pp)) panic("boot_alloc: pp is NULL or free"); /* * If the cage is on but doesn't yet contain this page, * mark it as non-relocatable. */ if (kcage_on && !PP_ISNORELOC(pp)) PP_SETNORELOC(pp); (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL); pp->p_lckcnt = 1; #if defined(__x86) page_downgrade(pp); #else page_unlock(pp); #endif } } /* * Get pages from boot and hash them into the kernel's vp. * Used after page structs have been allocated, but before segkmem is ready. */ void * boot_alloc(void *inaddr, size_t size, uint_t align) { caddr_t addr = inaddr; if (bootops == NULL) prom_panic("boot_alloc: attempt to allocate memory after " "BOP_GONE"); size = ptob(btopr(size)); if (BOP_ALLOC(bootops, addr, size, align) != addr) panic("boot_alloc: BOP_ALLOC failed"); boot_mapin((caddr_t)addr, size); return (addr); } static void segkmem_badop() { panic("segkmem_badop"); } #define SEGKMEM_BADOP(t) (t(*)())segkmem_badop /*ARGSUSED*/ static faultcode_t segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size, enum fault_type type, enum seg_rw rw) { pgcnt_t npages; spgcnt_t pg; page_t *pp; ASSERT(RW_READ_HELD(&seg->s_as->a_lock)); if (seg->s_as != &kas || size > seg->s_size || addr < seg->s_base || addr + size > seg->s_base + seg->s_size) panic("segkmem_fault: bad args"); if (segkp_bitmap && seg == &kvseg) { /* * If it is one of segkp pages, call segkp_fault. */ if (BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) return (SEGOP_FAULT(hat, segkp, addr, size, type, rw)); } if (rw != S_READ && rw != S_WRITE && rw != S_OTHER) return (FC_NOSUPPORT); npages = btopr(size); switch (type) { case F_SOFTLOCK: /* lock down already-loaded translations */ for (pg = 0; pg < npages; pg++) { pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_SHARED); if (pp == NULL) { /* * Hmm, no page. Does a kernel mapping * exist for it? */ if (!hat_probe(kas.a_hat, addr)) { addr -= PAGESIZE; while (--pg >= 0) { pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr); if (pp) page_unlock(pp); addr -= PAGESIZE; } return (FC_NOMAP); } } addr += PAGESIZE; } if (rw == S_OTHER) hat_reserve(seg->s_as, addr, size); return (0); case F_SOFTUNLOCK: while (npages--) { pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr); if (pp) page_unlock(pp); addr += PAGESIZE; } return (0); default: return (FC_NOSUPPORT); } /*NOTREACHED*/ } static int segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) { ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); if (seg->s_as != &kas || size > seg->s_size || addr < seg->s_base || addr + size > seg->s_base + seg->s_size) panic("segkmem_setprot: bad args"); if (segkp_bitmap && seg == &kvseg) { /* * If it is one of segkp pages, call segkp. */ if (BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) return (SEGOP_SETPROT(segkp, addr, size, prot)); } if (prot == 0) hat_unload(kas.a_hat, addr, size, HAT_UNLOAD); else hat_chgprot(kas.a_hat, addr, size, prot); return (0); } /* * This is a dummy segkmem function overloaded to call segkp * when segkp is under the heap. */ /* ARGSUSED */ static int segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) { ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); if (seg->s_as != &kas) segkmem_badop(); if (segkp_bitmap && seg == &kvseg) { /* * If it is one of segkp pages, call into segkp. */ if (BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) return (SEGOP_CHECKPROT(segkp, addr, size, prot)); } segkmem_badop(); return (0); } /* * This is a dummy segkmem function overloaded to call segkp * when segkp is under the heap. */ /* ARGSUSED */ static int segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta) { ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); if (seg->s_as != &kas) segkmem_badop(); if (segkp_bitmap && seg == &kvseg) { /* * If it is one of segkp pages, call into segkp. */ if (BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) return (SEGOP_KLUSTER(segkp, addr, delta)); } segkmem_badop(); return (0); } static void segkmem_xdump_range(void *arg, void *start, size_t size) { struct as *as = arg; caddr_t addr = start; caddr_t addr_end = addr + size; while (addr < addr_end) { pfn_t pfn = hat_getpfnum(kas.a_hat, addr); if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn)) dump_addpage(as, addr, pfn); addr += PAGESIZE; dump_timeleft = dump_timeout; } } static void segkmem_dump_range(void *arg, void *start, size_t size) { caddr_t addr = start; caddr_t addr_end = addr + size; /* * If we are about to start dumping the range of addresses we * carved out of the kernel heap for the large page heap walk * heap_lp_arena to find what segments are actually populated */ if (SEGKMEM_USE_LARGEPAGES && addr == heap_lp_base && addr_end == heap_lp_end && vmem_size(heap_lp_arena, VMEM_ALLOC) < size) { vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT, segkmem_xdump_range, arg); } else { segkmem_xdump_range(arg, start, size); } } static void segkmem_dump(struct seg *seg) { /* * The kernel's heap_arena (represented by kvseg) is a very large * VA space, most of which is typically unused. To speed up dumping * we use vmem_walk() to quickly find the pieces of heap_arena that * are actually in use. We do the same for heap32_arena and * heap_core. * * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage() * may ultimately need to allocate memory. Reentrant walks are * necessarily imperfect snapshots. The kernel heap continues * to change during a live crash dump, for example. For a normal * crash dump, however, we know that there won't be any other threads * messing with the heap. Therefore, at worst, we may fail to dump * the pages that get allocated by the act of dumping; but we will * always dump every page that was allocated when the walk began. * * The other segkmem segments are dense (fully populated), so there's * no need to use this technique when dumping them. * * Note: when adding special dump handling for any new sparsely- * populated segments, be sure to add similar handling to the ::kgrep * code in mdb. */ if (seg == &kvseg) { vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT, segkmem_dump_range, seg->s_as); #ifndef __sparc vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT, segkmem_dump_range, seg->s_as); #endif } else if (seg == &kvseg_core) { vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT, segkmem_dump_range, seg->s_as); } else if (seg == &kvseg32) { vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT, segkmem_dump_range, seg->s_as); vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT, segkmem_dump_range, seg->s_as); } else { segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size); } } /* * lock/unlock kmem pages over a given range [addr, addr+len). * Returns a shadow list of pages in ppp. If there are holes * in the range (e.g. some of the kernel mappings do not have * underlying page_ts) returns ENOTSUP so that as_pagelock() * will handle the range via as_fault(F_SOFTLOCK). */ /*ARGSUSED*/ static int segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len, page_t ***ppp, enum lock_type type, enum seg_rw rw) { page_t **pplist, *pp; pgcnt_t npages; spgcnt_t pg; size_t nb; ASSERT(ppp != NULL); if (segkp_bitmap && seg == &kvseg) { /* * If it is one of segkp pages, call into segkp. */ if (BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) return (SEGOP_PAGELOCK(segkp, addr, len, ppp, type, rw)); } if (type == L_PAGERECLAIM) return (ENOTSUP); npages = btopr(len); nb = sizeof (page_t *) * npages; if (type == L_PAGEUNLOCK) { pplist = *ppp; ASSERT(pplist != NULL); for (pg = 0; pg < npages; pg++) { pp = pplist[pg]; page_unlock(pp); } kmem_free(pplist, nb); return (0); } ASSERT(type == L_PAGELOCK); pplist = kmem_alloc(nb, KM_NOSLEEP); if (pplist == NULL) { *ppp = NULL; return (ENOTSUP); /* take the slow path */ } for (pg = 0; pg < npages; pg++) { pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_SHARED); if (pp == NULL) { while (--pg >= 0) page_unlock(pplist[pg]); kmem_free(pplist, nb); *ppp = NULL; return (ENOTSUP); } pplist[pg] = pp; addr += PAGESIZE; } *ppp = pplist; return (0); } /* * This is a dummy segkmem function overloaded to call segkp * when segkp is under the heap. */ /* ARGSUSED */ static int segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) { ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock)); if (seg->s_as != &kas) segkmem_badop(); if (segkp_bitmap && seg == &kvseg) { /* * If it is one of segkp pages, call into segkp. */ if (BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base)))) return (SEGOP_GETMEMID(segkp, addr, memidp)); } segkmem_badop(); return (0); } /*ARGSUSED*/ static lgrp_mem_policy_info_t * segkmem_getpolicy(struct seg *seg, caddr_t addr) { return (NULL); } /*ARGSUSED*/ static int segkmem_capable(struct seg *seg, segcapability_t capability) { if (capability == S_CAPABILITY_NOMINFLT) return (1); return (0); } static struct seg_ops segkmem_ops = { SEGKMEM_BADOP(int), /* dup */ SEGKMEM_BADOP(int), /* unmap */ SEGKMEM_BADOP(void), /* free */ segkmem_fault, SEGKMEM_BADOP(faultcode_t), /* faulta */ segkmem_setprot, segkmem_checkprot, segkmem_kluster, SEGKMEM_BADOP(size_t), /* swapout */ SEGKMEM_BADOP(int), /* sync */ SEGKMEM_BADOP(size_t), /* incore */ SEGKMEM_BADOP(int), /* lockop */ SEGKMEM_BADOP(int), /* getprot */ SEGKMEM_BADOP(u_offset_t), /* getoffset */ SEGKMEM_BADOP(int), /* gettype */ SEGKMEM_BADOP(int), /* getvp */ SEGKMEM_BADOP(int), /* advise */ segkmem_dump, segkmem_pagelock, SEGKMEM_BADOP(int), /* setpgsz */ segkmem_getmemid, segkmem_getpolicy, /* getpolicy */ segkmem_capable, /* capable */ }; int segkmem_create(struct seg *seg) { ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock)); seg->s_ops = &segkmem_ops; seg->s_data = NULL; kas.a_size += seg->s_size; return (0); } /*ARGSUSED*/ page_t * segkmem_page_create(void *addr, size_t size, int vmflag, void *arg) { struct seg kseg; int pgflags; kseg.s_as = &kas; pgflags = PG_EXCL; if (segkmem_reloc == 0 || (vmflag & VM_NORELOC)) pgflags |= PG_NORELOC; if ((vmflag & VM_NOSLEEP) == 0) pgflags |= PG_WAIT; if (vmflag & VM_PANIC) pgflags |= PG_PANIC; if (vmflag & VM_PUSHPAGE) pgflags |= PG_PUSHPAGE; return (page_create_va(&kvp, (u_offset_t)(uintptr_t)addr, size, pgflags, &kseg, addr)); } /* * Allocate pages to back the virtual address range [addr, addr + size). * If addr is NULL, allocate the virtual address space as well. */ void * segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg) { page_t *ppl; caddr_t addr = inaddr; pgcnt_t npages = btopr(size); int allocflag; if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL) return (NULL); ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) { if (inaddr == NULL) vmem_free(vmp, addr, size); return (NULL); } ppl = page_create_func(addr, size, vmflag, pcarg); if (ppl == NULL) { if (inaddr == NULL) vmem_free(vmp, addr, size); page_unresv(npages); return (NULL); } /* * Under certain conditions, we need to let the HAT layer know * that it cannot safely allocate memory. Allocations from * the hat_memload vmem arena always need this, to prevent * infinite recursion. * * In addition, the x86 hat cannot safely do memory * allocations while in vmem_populate(), because there * is no simple bound on its usage. */ if (vmflag & VM_MEMLOAD) allocflag = HAT_NO_KALLOC; #if defined(__x86) else if (vmem_is_populator()) allocflag = HAT_NO_KALLOC; #endif else allocflag = 0; while (ppl != NULL) { page_t *pp = ppl; page_sub(&ppl, pp); ASSERT(page_iolock_assert(pp)); ASSERT(PAGE_EXCL(pp)); page_io_unlock(pp); hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, HAT_LOAD_LOCK | allocflag); pp->p_lckcnt = 1; #if defined(__x86) page_downgrade(pp); #else if (vmflag & SEGKMEM_SHARELOCKED) page_downgrade(pp); else page_unlock(pp); #endif } return (addr); } void * segkmem_alloc(vmem_t *vmp, size_t size, int vmflag) { void *addr; segkmem_gc_list_t *gcp, **prev_gcpp; if (kvseg.s_base == NULL) { #ifndef __sparc if (bootops->bsys_alloc == NULL) halt("Memory allocation between bop_alloc() and " "kmem_alloc().\n"); #endif /* * There's not a lot of memory to go around during boot, * so recycle it if we can. */ for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL; prev_gcpp = &gcp->gc_next) { if (gcp->gc_arena == vmp && gcp->gc_size == size) { *prev_gcpp = gcp->gc_next; return (gcp); } } addr = vmem_alloc(vmp, size, vmflag | VM_PANIC); if (boot_alloc(addr, size, BO_NO_ALIGN) != addr) panic("segkmem_alloc: boot_alloc failed"); return (addr); } return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, segkmem_page_create, NULL)); } /* * Any changes to this routine must also be carried over to * devmap_free_pages() in the seg_dev driver. This is because * we currently don't have a special kernel segment for non-paged * kernel memory that is exported by drivers to user space. */ void segkmem_free(vmem_t *vmp, void *inaddr, size_t size) { page_t *pp; caddr_t addr = inaddr; caddr_t eaddr; pgcnt_t npages = btopr(size); ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); if (kvseg.s_base == NULL) { segkmem_gc_list_t *gc = inaddr; gc->gc_arena = vmp; gc->gc_size = size; gc->gc_next = segkmem_gc_list; segkmem_gc_list = gc; return; } hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { #if defined(__x86) pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr); if (pp == NULL) panic("segkmem_free: page not found"); if (!page_tryupgrade(pp)) { /* * Some other thread has a sharelock. Wait for * it to drop the lock so we can free this page. */ page_unlock(pp); pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL); } #else pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL); #endif if (pp == NULL) panic("segkmem_free: page not found"); /* Clear p_lckcnt so page_destroy() doesn't update availrmem */ pp->p_lckcnt = 0; page_destroy(pp, 0); } page_unresv(npages); if (vmp != NULL) vmem_free(vmp, inaddr, size); } void segkmem_gc(void) { ASSERT(kvseg.s_base != NULL); while (segkmem_gc_list != NULL) { segkmem_gc_list_t *gc = segkmem_gc_list; segkmem_gc_list = gc->gc_next; segkmem_free(gc->gc_arena, gc, gc->gc_size); } } /* * Legacy entry points from here to end of file. */ void segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot, pfn_t pfn, uint_t flags) { hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK); hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot, flags | HAT_LOAD_LOCK); } void segkmem_mapout(struct seg *seg, void *addr, size_t size) { hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK); } void * kmem_getpages(pgcnt_t npages, int kmflag) { return (kmem_alloc(ptob(npages), kmflag)); } void kmem_freepages(void *addr, pgcnt_t npages) { kmem_free(addr, ptob(npages)); } /* * segkmem_page_create_large() allocates a large page to be used for the kmem * caches. If kpr is enabled we ask for a relocatable page unless requested * otherwise. If kpr is disabled we have to ask for a non-reloc page */ static page_t * segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg) { int pgflags; pgflags = PG_EXCL; if (segkmem_reloc == 0 || (vmflag & VM_NORELOC)) pgflags |= PG_NORELOC; if (!(vmflag & VM_NOSLEEP)) pgflags |= PG_WAIT; if (vmflag & VM_PUSHPAGE) pgflags |= PG_PUSHPAGE; return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size, pgflags, &kvseg, addr, arg)); } /* * Allocate a large page to back the virtual address range * [addr, addr + size). If addr is NULL, allocate the virtual address * space as well. */ static void * segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg) { caddr_t addr = inaddr, pa; size_t lpsize = segkmem_lpsize; pgcnt_t npages = btopr(size); pgcnt_t nbpages = btop(lpsize); pgcnt_t nlpages = size >> segkmem_lpshift; size_t ppasize = nbpages * sizeof (page_t *); page_t *pp, *rootpp, **ppa, *pplist = NULL; int i; if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) { return (NULL); } /* * allocate an array we need for hat_memload_array. * we use a separate arena to avoid recursion. * we will not need this array when hat_memload_array learns pp++ */ if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) { goto fail_array_alloc; } if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL) goto fail_vmem_alloc; ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0); /* create all the pages */ for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) { if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL) goto fail_page_create; page_list_concat(&pplist, &pp); } /* at this point we have all the resource to complete the request */ while ((rootpp = pplist) != NULL) { for (i = 0; i < nbpages; i++) { ASSERT(pplist != NULL); pp = pplist; page_sub(&pplist, pp); ASSERT(page_iolock_assert(pp)); page_io_unlock(pp); ppa[i] = pp; } /* * Load the locked entry. It's OK to preload the entry into the * TSB since we now support large mappings in the kernel TSB. */ hat_memload_array(kas.a_hat, (caddr_t)(uintptr_t)rootpp->p_offset, lpsize, ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr, HAT_LOAD_LOCK); for (--i; i >= 0; --i) { ppa[i]->p_lckcnt = 1; page_unlock(ppa[i]); } } vmem_free(segkmem_ppa_arena, ppa, ppasize); return (addr); fail_page_create: while ((rootpp = pplist) != NULL) { for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) { ASSERT(pp != NULL); page_sub(&pplist, pp); ASSERT(page_iolock_assert(pp)); page_io_unlock(pp); } page_destroy_pages(rootpp); } if (inaddr == NULL) vmem_free(vmp, addr, size); fail_vmem_alloc: vmem_free(segkmem_ppa_arena, ppa, ppasize); fail_array_alloc: page_unresv(npages); return (NULL); } static void segkmem_free_one_lp(caddr_t addr, size_t size) { page_t *pp, *rootpp = NULL; pgcnt_t pgs_left = btopr(size); ASSERT(size == segkmem_lpsize); hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) { pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL); if (pp == NULL) panic("segkmem_free_one_lp: page not found"); ASSERT(PAGE_EXCL(pp)); pp->p_lckcnt = 0; if (rootpp == NULL) rootpp = pp; } ASSERT(rootpp != NULL); page_destroy_pages(rootpp); /* page_unresv() is done by the caller */ } /* * This function is called to import new spans into the vmem arenas like * kmem_default_arena and kmem_oversize_arena. It first tries to import * spans from large page arena - kmem_lp_arena. In order to do this it might * have to "upgrade the requested size" to kmem_lp_arena quantum. If * it was not able to satisfy the upgraded request it then calls regular * segkmem_alloc() that satisfies the request by importing from "*vmp" arena */ void * segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, int vmflag) { size_t size; kthread_t *t = curthread; segkmem_lpcb_t *lpcb = &segkmem_lpcb; ASSERT(sizep != NULL); size = *sizep; if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) && !(vmflag & SEGKMEM_SHARELOCKED)) { size_t kmemlp_qnt = segkmem_kmemlp_quantum; size_t asize = P2ROUNDUP(size, kmemlp_qnt); void *addr = NULL; ulong_t *lpthrtp = &lpcb->lp_throttle; ulong_t lpthrt = *lpthrtp; int dowakeup = 0; int doalloc = 1; ASSERT(kmem_lp_arena != NULL); ASSERT(asize >= size); if (lpthrt != 0) { /* try to update the throttle value */ lpthrt = atomic_add_long_nv(lpthrtp, 1); if (lpthrt >= segkmem_lpthrottle_max) { lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, segkmem_lpthrottle_max / 4); } /* * when we get above throttle start do an exponential * backoff at trying large pages and reaping */ if (lpthrt > segkmem_lpthrottle_start && (lpthrt & (lpthrt - 1))) { lpcb->allocs_throttled++; lpthrt--; if ((lpthrt & (lpthrt - 1)) == 0) kmem_reap(); return (segkmem_alloc(vmp, size, vmflag)); } } if (!(vmflag & VM_NOSLEEP) && segkmem_heaplp_quantum >= (8 * kmemlp_qnt) && vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt && asize < (segkmem_heaplp_quantum - kmemlp_qnt)) { /* * we are low on free memory in kmem_lp_arena * we let only one guy to allocate heap_lp * quantum size chunk that everybody is going to * share */ mutex_enter(&lpcb->lp_lock); if (lpcb->lp_wait) { /* we are not the first one - wait */ cv_wait(&lpcb->lp_cv, &lpcb->lp_lock); if (vmem_size(kmem_lp_arena, VMEM_FREE) < kmemlp_qnt) { doalloc = 0; } } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt) { /* * we are the first one, make sure we import * a large page */ if (asize == kmemlp_qnt) asize += kmemlp_qnt; dowakeup = 1; lpcb->lp_wait = 1; } mutex_exit(&lpcb->lp_lock); } /* * VM_ABORT flag prevents sleeps in vmem_xalloc when * large pages are not available. In that case this allocation * attempt will fail and we will retry allocation with small * pages. We also do not want to panic if this allocation fails * because we are going to retry. */ if (doalloc) { addr = vmem_alloc(kmem_lp_arena, asize, (vmflag | VM_ABORT) & ~VM_PANIC); if (dowakeup) { mutex_enter(&lpcb->lp_lock); ASSERT(lpcb->lp_wait != 0); lpcb->lp_wait = 0; cv_broadcast(&lpcb->lp_cv); mutex_exit(&lpcb->lp_lock); } } if (addr != NULL) { *sizep = asize; *lpthrtp = 0; return (addr); } if (vmflag & VM_NOSLEEP) lpcb->nosleep_allocs_failed++; else lpcb->sleep_allocs_failed++; lpcb->alloc_bytes_failed += size; /* if large page throttling is not started yet do it */ if (segkmem_use_lpthrottle && lpthrt == 0) { lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1); } } return (segkmem_alloc(vmp, size, vmflag)); } void segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size) { if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) { segkmem_free(vmp, inaddr, size); } else { vmem_free(kmem_lp_arena, inaddr, size); } } /* * segkmem_alloc_lpi() imports virtual memory from large page heap arena * into kmem_lp arena. In the process it maps the imported segment with * large pages */ static void * segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag) { segkmem_lpcb_t *lpcb = &segkmem_lpcb; void *addr; ASSERT(size != 0); ASSERT(vmp == heap_lp_arena); /* do not allow large page heap grow beyound limits */ if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) { lpcb->allocs_limited++; return (NULL); } addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0, segkmem_page_create_large, NULL); return (addr); } /* * segkmem_free_lpi() returns virtual memory back into large page heap arena * from kmem_lp arena. Beore doing this it unmaps the segment and frees * large pages used to map it. */ static void segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size) { pgcnt_t nlpages = size >> segkmem_lpshift; size_t lpsize = segkmem_lpsize; caddr_t addr = inaddr; pgcnt_t npages = btopr(size); int i; ASSERT(vmp == heap_lp_arena); ASSERT(IS_KMEM_VA_LARGEPAGE(addr)); ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0); for (i = 0; i < nlpages; i++) { segkmem_free_one_lp(addr, lpsize); addr += lpsize; } page_unresv(npages); vmem_free(vmp, inaddr, size); } /* * This function is called at system boot time by kmem_init right after * /etc/system file has been read. It checks based on hardware configuration * and /etc/system settings if system is going to use large pages. The * initialiazation necessary to actually start using large pages * happens later in the process after segkmem_heap_lp_init() is called. */ int segkmem_lpsetup() { int use_large_pages = 0; #ifdef __sparc size_t memtotal = physmem * PAGESIZE; if (heap_lp_base == NULL) { segkmem_lpsize = PAGESIZE; return (0); } /* get a platform dependent value of large page size for kernel heap */ segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize); if (segkmem_lpsize <= PAGESIZE) { /* * put virtual space reserved for the large page kernel * back to the regular heap */ vmem_xfree(heap_arena, heap_lp_base, heap_lp_end - heap_lp_base); heap_lp_base = NULL; heap_lp_end = NULL; segkmem_lpsize = PAGESIZE; return (0); } /* set heap_lp quantum if necessary */ if (segkmem_heaplp_quantum == 0 || (segkmem_heaplp_quantum & (segkmem_heaplp_quantum - 1)) || P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) { segkmem_heaplp_quantum = segkmem_lpsize; } /* set kmem_lp quantum if necessary */ if (segkmem_kmemlp_quantum == 0 || (segkmem_kmemlp_quantum & (segkmem_kmemlp_quantum - 1)) || segkmem_kmemlp_quantum > segkmem_heaplp_quantum) { segkmem_kmemlp_quantum = segkmem_heaplp_quantum; } /* set total amount of memory allowed for large page kernel heap */ if (segkmem_kmemlp_max == 0) { if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100) segkmem_kmemlp_pcnt = 12; segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100; } segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max, segkmem_heaplp_quantum); /* fix lp kmem preallocation request if necesssary */ if (segkmem_kmemlp_min) { segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min, segkmem_heaplp_quantum); if (segkmem_kmemlp_min > segkmem_kmemlp_max) segkmem_kmemlp_min = segkmem_kmemlp_max; } use_large_pages = 1; segkmem_lpshift = page_get_shift(page_szc(segkmem_lpsize)); #endif return (use_large_pages); } #ifdef __sparc static void * segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag) { size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *); void *addr; if (ppaquantum <= PAGESIZE) return (segkmem_alloc(vmp, size, vmflag)); ASSERT((size & (ppaquantum - 1)) == 0); addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag); if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0, segkmem_page_create, NULL) == NULL) { vmem_xfree(vmp, addr, size); addr = NULL; } return (addr); } static void segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size) { size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *); ASSERT(addr != NULL); if (ppaquantum <= PAGESIZE) { segkmem_free(vmp, addr, size); } else { segkmem_free(NULL, addr, size); vmem_xfree(vmp, addr, size); } } void segkmem_heap_lp_init() { segkmem_lpcb_t *lpcb = &segkmem_lpcb; size_t heap_lp_size = heap_lp_end - heap_lp_base; size_t lpsize = segkmem_lpsize; size_t ppaquantum; void *addr; if (segkmem_lpsize <= PAGESIZE) { ASSERT(heap_lp_base == NULL); ASSERT(heap_lp_end == NULL); return; } ASSERT(segkmem_heaplp_quantum >= lpsize); ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0); ASSERT(lpcb->lp_uselp == 0); ASSERT(heap_lp_base != NULL); ASSERT(heap_lp_end != NULL); ASSERT(heap_lp_base < heap_lp_end); ASSERT(heap_lp_arena == NULL); ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0); ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0); /* create large page heap arena */ heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size, segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP); ASSERT(heap_lp_arena != NULL); /* This arena caches memory already mapped by large pages */ kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum, segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP); ASSERT(kmem_lp_arena != NULL); mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL); /* * this arena is used for the array of page_t pointers necessary * to call hat_mem_load_array */ ppaquantum = btopr(lpsize) * sizeof (page_t *); segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum, segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum, VM_SLEEP); ASSERT(segkmem_ppa_arena != NULL); /* prealloacate some memory for the lp kernel heap */ if (segkmem_kmemlp_min) { ASSERT(P2PHASE(segkmem_kmemlp_min, segkmem_heaplp_quantum) == 0); if ((addr = segkmem_alloc_lpi(heap_lp_arena, segkmem_kmemlp_min, VM_SLEEP)) != NULL) { addr = vmem_add(kmem_lp_arena, addr, segkmem_kmemlp_min, VM_SLEEP); ASSERT(addr != NULL); } } lpcb->lp_uselp = 1; } #endif