xref: /titanic_44/usr/src/uts/common/vm/seg_kmem.c (revision 7c8de9202c10c8c49a901bff2e373864b545bd57)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/types.h>
30 #include <sys/t_lock.h>
31 #include <sys/param.h>
32 #include <sys/sysmacros.h>
33 #include <sys/tuneable.h>
34 #include <sys/systm.h>
35 #include <sys/vm.h>
36 #include <sys/kmem.h>
37 #include <sys/vmem.h>
38 #include <sys/mman.h>
39 #include <sys/cmn_err.h>
40 #include <sys/debug.h>
41 #include <sys/dumphdr.h>
42 #include <sys/bootconf.h>
43 #include <sys/lgrp.h>
44 #include <vm/seg_kmem.h>
45 #include <vm/hat.h>
46 #include <vm/page.h>
47 #include <vm/vm_dep.h>
48 #include <vm/faultcode.h>
49 #include <sys/promif.h>
50 #include <vm/seg_kp.h>
51 #include <sys/bitmap.h>
52 #include <sys/mem_cage.h>
53 
54 /*
55  * seg_kmem is the primary kernel memory segment driver.  It
56  * maps the kernel heap [kernelheap, ekernelheap), module text,
57  * and all memory which was allocated before the VM was initialized
58  * into kas.
59  *
60  * Pages which belong to seg_kmem are hashed into &kvp vnode at
61  * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1.
62  * They must never be paged out since segkmem_fault() is a no-op to
63  * prevent recursive faults.
64  *
65  * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
66  * __x86 and are unlocked (p_sharelock == 0) on __sparc.  Once __x86
67  * supports relocation the #ifdef kludges can be removed.
68  *
69  * seg_kmem pages may be subject to relocation by page_relocate(),
70  * provided that the HAT supports it; if this is so, segkmem_reloc
71  * will be set to a nonzero value. All boot time allocated memory as
72  * well as static memory is considered off limits to relocation.
73  * Pages are "relocatable" if p_state does not have P_NORELOC set, so
74  * we request P_NORELOC pages for memory that isn't safe to relocate.
75  *
76  * The kernel heap is logically divided up into four pieces:
77  *
78  *   heap32_arena is for allocations that require 32-bit absolute
79  *   virtual addresses (e.g. code that uses 32-bit pointers/offsets).
80  *
81  *   heap_core is for allocations that require 2GB *relative*
82  *   offsets; in other words all memory from heap_core is within
83  *   2GB of all other memory from the same arena. This is a requirement
84  *   of the addressing modes of some processors in supervisor code.
85  *
86  *   heap_arena is the general heap arena.
87  *
88  *   static_arena is the static memory arena.  Allocations from it
89  *   are not subject to relocation so it is safe to use the memory
90  *   physical address as well as the virtual address (e.g. the VA to
91  *   PA translations are static).  Caches may import from static_arena;
92  *   all other static memory allocations should use static_alloc_arena.
93  *
94  * On some platforms which have limited virtual address space, seg_kmem
95  * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
96  * segkp_bitmap is non-NULL, and each bit represents a page of virtual
97  * address space which is actually seg_kp mapped.
98  */
99 
100 extern ulong_t *segkp_bitmap;   /* Is set if segkp is from the kernel heap */
101 
102 char *kernelheap;		/* start of primary kernel heap */
103 char *ekernelheap;		/* end of primary kernel heap */
104 struct seg kvseg;		/* primary kernel heap segment */
105 struct seg kvseg_core;		/* "core" kernel heap segment */
106 vmem_t *heap_arena;		/* primary kernel heap arena */
107 vmem_t *heap_core_arena;	/* core kernel heap arena */
108 char *heap_core_base;		/* start of core kernel heap arena */
109 char *heap_lp_base;		/* start of kernel large page heap arena */
110 char *heap_lp_end;		/* end of kernel large page heap arena */
111 vmem_t *hat_memload_arena;	/* HAT translation data */
112 struct seg kvseg32;		/* 32-bit kernel heap segment */
113 vmem_t *heap32_arena;		/* 32-bit kernel heap arena */
114 vmem_t *heaptext_arena;		/* heaptext arena */
115 struct as kas;			/* kernel address space */
116 struct vnode kvp;		/* vnode for all segkmem pages */
117 int segkmem_reloc;		/* enable/disable relocatable segkmem pages */
118 vmem_t *static_arena;		/* arena for caches to import static memory */
119 vmem_t *static_alloc_arena;	/* arena for allocating static memory */
120 
121 /*
122  * seg_kmem driver can map part of the kernel heap with large pages.
123  * Currently this functionality is implemented for sparc platforms only.
124  *
125  * The large page size "segkmem_lpsize" for kernel heap is selected in the
126  * platform specific code. It can also be modified via /etc/system file.
127  * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
128  * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
129  * match segkmem_lpsize.
130  *
131  * At boot time we carve from kernel heap arena a range of virtual addresses
132  * that will be used for large page mappings. This range [heap_lp_base,
133  * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
134  * create "kmem_lp_arena" that caches memory already backed up by large
135  * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
136  */
137 
138 size_t	segkmem_lpsize;
139 static  uint_t	segkmem_lpshift = PAGESHIFT;
140 
141 size_t  segkmem_kmemlp_quantum = 0x400000;	/* 4MB */
142 size_t  segkmem_heaplp_quantum;
143 vmem_t *heap_lp_arena;
144 static  vmem_t *kmem_lp_arena;
145 static  vmem_t *segkmem_ppa_arena;
146 static	segkmem_lpcb_t segkmem_lpcb;
147 
148 /*
149  * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
150  * consumed by the large page heap. By default this parameter is set to 1/8 of
151  * physmem but can be adjusted through /etc/system either directly or
152  * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
153  * we allow for large page heap.
154  */
155 size_t  segkmem_kmemlp_max;
156 static  uint_t  segkmem_kmemlp_pcnt;
157 
158 /*
159  * Getting large pages for kernel heap could be problematic due to
160  * physical memory fragmentation. That's why we allow to preallocate
161  * "segkmem_kmemlp_min" bytes at boot time.
162  */
163 static  size_t	segkmem_kmemlp_min;
164 
165 /*
166  * Throttling is used to avoid expensive tries to allocate large pages
167  * for kernel heap when a lot of succesive attempts to do so fail.
168  */
169 static  ulong_t segkmem_lpthrottle_max = 0x400000;
170 static  ulong_t segkmem_lpthrottle_start = 0x40;
171 static  ulong_t segkmem_use_lpthrottle = 1;
172 
173 /*
174  * Freed pages accumulate on a garbage list until segkmem is ready,
175  * at which point we call segkmem_gc() to free it all.
176  */
177 typedef struct segkmem_gc_list {
178 	struct segkmem_gc_list	*gc_next;
179 	vmem_t			*gc_arena;
180 	size_t			gc_size;
181 } segkmem_gc_list_t;
182 
183 static segkmem_gc_list_t *segkmem_gc_list;
184 
185 /*
186  * Allocations from the hat_memload arena add VM_MEMLOAD to their
187  * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
188  * to take steps to prevent infinite recursion.  HAT allocations also
189  * must be non-relocatable to prevent recursive page faults.
190  */
191 static void *
192 hat_memload_alloc(vmem_t *vmp, size_t size, int flags)
193 {
194 	flags |= (VM_MEMLOAD | VM_NORELOC);
195 	return (segkmem_alloc(vmp, size, flags));
196 }
197 
198 /*
199  * Allocations from static_arena arena (or any other arena that uses
200  * segkmem_alloc_permanent()) require non-relocatable (permanently
201  * wired) memory pages, since these pages are referenced by physical
202  * as well as virtual address.
203  */
204 void *
205 segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags)
206 {
207 	return (segkmem_alloc(vmp, size, flags | VM_NORELOC));
208 }
209 
210 /*
211  * Initialize kernel heap boundaries.
212  */
213 void
214 kernelheap_init(
215 	void *heap_start,
216 	void *heap_end,
217 	char *first_avail,
218 	void *core_start,
219 	void *core_end)
220 {
221 	uintptr_t textbase;
222 	size_t core_size;
223 	size_t heap_size;
224 	vmem_t *heaptext_parent;
225 	size_t	heap_lp_size = 0;
226 
227 	kernelheap = heap_start;
228 	ekernelheap = heap_end;
229 
230 #ifdef __sparc
231 	heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4);
232 	heap_lp_base = ekernelheap - heap_lp_size;
233 	heap_lp_end = heap_lp_base + heap_lp_size;
234 #endif	/* __sparc */
235 
236 	/*
237 	 * If this platform has a 'core' heap area, then the space for
238 	 * overflow module text should be carved out of the end of that
239 	 * heap.  Otherwise, it gets carved out of the general purpose
240 	 * heap.
241 	 */
242 	core_size = (uintptr_t)core_end - (uintptr_t)core_start;
243 	if (core_size > 0) {
244 		ASSERT(core_size >= HEAPTEXT_SIZE);
245 		textbase = (uintptr_t)core_end - HEAPTEXT_SIZE;
246 		core_size -= HEAPTEXT_SIZE;
247 	}
248 #ifndef __sparc
249 	else {
250 		ekernelheap -= HEAPTEXT_SIZE;
251 		textbase = (uintptr_t)ekernelheap;
252 	}
253 #endif
254 
255 	heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap;
256 	heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE,
257 	    segkmem_alloc, segkmem_free);
258 
259 	if (core_size > 0) {
260 		heap_core_arena = vmem_create("heap_core", core_start,
261 		    core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
262 		heap_core_base = core_start;
263 	} else {
264 		heap_core_arena = heap_arena;
265 		heap_core_base = kernelheap;
266 	}
267 
268 	/*
269 	 * reserve space for the large page heap. If large pages for kernel
270 	 * heap is enabled large page heap arean will be created later in the
271 	 * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
272 	 * range will be returned back to the heap_arena.
273 	 */
274 	if (heap_lp_size) {
275 		(void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0,
276 		    heap_lp_base, heap_lp_end,
277 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
278 	}
279 
280 	/*
281 	 * Remove the already-spoken-for memory range [kernelheap, first_avail).
282 	 */
283 	(void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE,
284 	    0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
285 
286 #ifdef __sparc
287 	heap32_arena = vmem_create("heap32", (void *)SYSBASE32,
288 	    SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL,
289 	    NULL, NULL, 0, VM_SLEEP);
290 
291 	textbase = SYSLIMIT32 - HEAPTEXT_SIZE;
292 	heaptext_parent = NULL;
293 #else	/* __sparc */
294 	heap32_arena = heap_core_arena;
295 	heaptext_parent = heap_core_arena;
296 #endif	/* __sparc */
297 
298 	heaptext_arena = vmem_create("heaptext", (void *)textbase,
299 	    HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP);
300 
301 	/*
302 	 * Create a set of arenas for memory with static translations
303 	 * (e.g. VA -> PA translations cannot change).  Since using
304 	 * kernel pages by physical address implies it isn't safe to
305 	 * walk across page boundaries, the static_arena quantum must
306 	 * be PAGESIZE.  Any kmem caches that require static memory
307 	 * should source from static_arena, while direct allocations
308 	 * should only use static_alloc_arena.
309 	 */
310 	static_arena = vmem_create("static", NULL, 0, PAGESIZE,
311 	    segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP);
312 	static_alloc_arena = vmem_create("static_alloc", NULL, 0,
313 	    sizeof (uint64_t), vmem_alloc, vmem_free, static_arena,
314 	    0, VM_SLEEP);
315 
316 	/*
317 	 * Create an arena for translation data (ptes, hmes, or hblks).
318 	 * We need an arena for this because hat_memload() is essential
319 	 * to vmem_populate() (see comments in common/os/vmem.c).
320 	 *
321 	 * Note: any kmem cache that allocates from hat_memload_arena
322 	 * must be created as a KMC_NOHASH cache (i.e. no external slab
323 	 * and bufctl structures to allocate) so that slab creation doesn't
324 	 * require anything more than a single vmem_alloc().
325 	 */
326 	hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE,
327 	    hat_memload_alloc, segkmem_free, heap_arena, 0,
328 	    VM_SLEEP | VMC_POPULATOR);
329 }
330 
331 /*
332  * Grow kernel heap downward.
333  */
334 void
335 kernelheap_extend(void *range_start, void *range_end)
336 {
337 	size_t len = (uintptr_t)range_end - (uintptr_t)range_start;
338 
339 	ASSERT(range_start < range_end && range_end == kernelheap);
340 
341 	if (vmem_add(heap_arena, range_start, len, VM_NOSLEEP) == NULL) {
342 		cmn_err(CE_WARN, "Could not grow kernel heap below 0x%p",
343 		    (void *)kernelheap);
344 	} else {
345 		kernelheap = range_start;
346 	}
347 }
348 
349 void
350 boot_mapin(caddr_t addr, size_t size)
351 {
352 	caddr_t	 eaddr;
353 	page_t	*pp;
354 	pfn_t	 pfnum;
355 
356 	if (page_resv(btop(size), KM_NOSLEEP) == 0)
357 		panic("boot_mapin: page_resv failed");
358 
359 	for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
360 		pfnum = va_to_pfn(addr);
361 		if ((pp = page_numtopp_nolock(pfnum)) == NULL)
362 			panic("boot_mapin(): No pp for pfnum = %lx", pfnum);
363 
364 		/*
365 		 * must break up any large pages that may have constituent
366 		 * pages being utilized for BOP_ALLOC()'s before calling
367 		 * page_numtopp().The locking code (ie. page_reclaim())
368 		 * can't handle them
369 		 */
370 		if (pp->p_szc != 0)
371 			page_boot_demote(pp);
372 
373 		pp = page_numtopp(pfnum, SE_EXCL);
374 		if (pp == NULL || PP_ISFREE(pp))
375 			panic("boot_alloc: pp is NULL or free");
376 
377 		/*
378 		 * If the cage is on but doesn't yet contain this page,
379 		 * mark it as non-relocatable.
380 		 */
381 		if (kcage_on && !PP_ISNORELOC(pp))
382 			PP_SETNORELOC(pp);
383 
384 		(void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL);
385 		pp->p_lckcnt = 1;
386 #if defined(__x86)
387 		page_downgrade(pp);
388 #else
389 		page_unlock(pp);
390 #endif
391 	}
392 }
393 
394 /*
395  * Get pages from boot and hash them into the kernel's vp.
396  * Used after page structs have been allocated, but before segkmem is ready.
397  */
398 void *
399 boot_alloc(void *inaddr, size_t size, uint_t align)
400 {
401 	caddr_t addr = inaddr;
402 
403 	if (bootops == NULL)
404 		prom_panic("boot_alloc: attempt to allocate memory after "
405 		    "BOP_GONE");
406 
407 	size = ptob(btopr(size));
408 	if (BOP_ALLOC(bootops, addr, size, align) != addr)
409 		panic("boot_alloc: BOP_ALLOC failed");
410 	boot_mapin((caddr_t)addr, size);
411 	return (addr);
412 }
413 
414 static void
415 segkmem_badop()
416 {
417 	panic("segkmem_badop");
418 }
419 
420 #define	SEGKMEM_BADOP(t)	(t(*)())segkmem_badop
421 
422 /*ARGSUSED*/
423 static faultcode_t
424 segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size,
425 	enum fault_type type, enum seg_rw rw)
426 {
427 	pgcnt_t npages;
428 	spgcnt_t pg;
429 	page_t *pp;
430 
431 	ASSERT(RW_READ_HELD(&seg->s_as->a_lock));
432 
433 	if (seg->s_as != &kas || size > seg->s_size ||
434 	    addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
435 		panic("segkmem_fault: bad args");
436 
437 	if (segkp_bitmap && seg == &kvseg) {
438 		/*
439 		 * If it is one of segkp pages, call segkp_fault.
440 		 */
441 		if (BT_TEST(segkp_bitmap,
442 			btop((uintptr_t)(addr - seg->s_base))))
443 			return (SEGOP_FAULT(hat, segkp, addr, size, type, rw));
444 	}
445 
446 	if (rw != S_READ && rw != S_WRITE && rw != S_OTHER)
447 		return (FC_NOSUPPORT);
448 
449 	npages = btopr(size);
450 
451 	switch (type) {
452 	case F_SOFTLOCK:	/* lock down already-loaded translations */
453 		for (pg = 0; pg < npages; pg++) {
454 			pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
455 			    SE_SHARED);
456 			if (pp == NULL) {
457 				/*
458 				 * Hmm, no page. Does a kernel mapping
459 				 * exist for it?
460 				 */
461 				if (!hat_probe(kas.a_hat, addr)) {
462 					addr -= PAGESIZE;
463 					while (--pg >= 0) {
464 						pp = page_find(&kvp,
465 						(u_offset_t)(uintptr_t)addr);
466 						if (pp)
467 							page_unlock(pp);
468 						addr -= PAGESIZE;
469 					}
470 					return (FC_NOMAP);
471 				}
472 			}
473 			addr += PAGESIZE;
474 		}
475 		if (rw == S_OTHER)
476 			hat_reserve(seg->s_as, addr, size);
477 		return (0);
478 	case F_SOFTUNLOCK:
479 		while (npages--) {
480 			pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
481 			if (pp)
482 				page_unlock(pp);
483 			addr += PAGESIZE;
484 		}
485 		return (0);
486 	default:
487 		return (FC_NOSUPPORT);
488 	}
489 	/*NOTREACHED*/
490 }
491 
492 static int
493 segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
494 {
495 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
496 
497 	if (seg->s_as != &kas || size > seg->s_size ||
498 	    addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
499 		panic("segkmem_setprot: bad args");
500 
501 	if (segkp_bitmap && seg == &kvseg) {
502 
503 		/*
504 		 * If it is one of segkp pages, call segkp.
505 		 */
506 		if (BT_TEST(segkp_bitmap,
507 			btop((uintptr_t)(addr - seg->s_base))))
508 			return (SEGOP_SETPROT(segkp, addr, size, prot));
509 	}
510 
511 	if (prot == 0)
512 		hat_unload(kas.a_hat, addr, size, HAT_UNLOAD);
513 	else
514 		hat_chgprot(kas.a_hat, addr, size, prot);
515 	return (0);
516 }
517 
518 /*
519  * This is a dummy segkmem function overloaded to call segkp
520  * when segkp is under the heap.
521  */
522 /* ARGSUSED */
523 static int
524 segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
525 {
526 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
527 
528 	if (seg->s_as != &kas)
529 		segkmem_badop();
530 
531 	if (segkp_bitmap && seg == &kvseg) {
532 
533 		/*
534 		 * If it is one of segkp pages, call into segkp.
535 		 */
536 		if (BT_TEST(segkp_bitmap,
537 			btop((uintptr_t)(addr - seg->s_base))))
538 			return (SEGOP_CHECKPROT(segkp, addr, size, prot));
539 	}
540 	segkmem_badop();
541 	return (0);
542 }
543 
544 /*
545  * This is a dummy segkmem function overloaded to call segkp
546  * when segkp is under the heap.
547  */
548 /* ARGSUSED */
549 static int
550 segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
551 {
552 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
553 
554 	if (seg->s_as != &kas)
555 		segkmem_badop();
556 
557 	if (segkp_bitmap && seg == &kvseg) {
558 
559 		/*
560 		 * If it is one of segkp pages, call into segkp.
561 		 */
562 		if (BT_TEST(segkp_bitmap,
563 			btop((uintptr_t)(addr - seg->s_base))))
564 			return (SEGOP_KLUSTER(segkp, addr, delta));
565 	}
566 	segkmem_badop();
567 	return (0);
568 }
569 
570 static void
571 segkmem_xdump_range(void *arg, void *start, size_t size)
572 {
573 	struct as *as = arg;
574 	caddr_t addr = start;
575 	caddr_t addr_end = addr + size;
576 
577 	while (addr < addr_end) {
578 		pfn_t pfn = hat_getpfnum(kas.a_hat, addr);
579 		if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn))
580 			dump_addpage(as, addr, pfn);
581 		addr += PAGESIZE;
582 		dump_timeleft = dump_timeout;
583 	}
584 }
585 
586 static void
587 segkmem_dump_range(void *arg, void *start, size_t size)
588 {
589 	caddr_t addr = start;
590 	caddr_t addr_end = addr + size;
591 
592 	/*
593 	 * If we are about to start dumping the range of addresses we
594 	 * carved out of the kernel heap for the large page heap walk
595 	 * heap_lp_arena to find what segments are actually populated
596 	 */
597 	if (SEGKMEM_USE_LARGEPAGES &&
598 	    addr == heap_lp_base && addr_end == heap_lp_end &&
599 	    vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
600 		vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT,
601 		    segkmem_xdump_range, arg);
602 	} else {
603 		segkmem_xdump_range(arg, start, size);
604 	}
605 }
606 
607 static void
608 segkmem_dump(struct seg *seg)
609 {
610 	/*
611 	 * The kernel's heap_arena (represented by kvseg) is a very large
612 	 * VA space, most of which is typically unused.  To speed up dumping
613 	 * we use vmem_walk() to quickly find the pieces of heap_arena that
614 	 * are actually in use.  We do the same for heap32_arena and
615 	 * heap_core.
616 	 *
617 	 * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
618 	 * may ultimately need to allocate memory.  Reentrant walks are
619 	 * necessarily imperfect snapshots.  The kernel heap continues
620 	 * to change during a live crash dump, for example.  For a normal
621 	 * crash dump, however, we know that there won't be any other threads
622 	 * messing with the heap.  Therefore, at worst, we may fail to dump
623 	 * the pages that get allocated by the act of dumping; but we will
624 	 * always dump every page that was allocated when the walk began.
625 	 *
626 	 * The other segkmem segments are dense (fully populated), so there's
627 	 * no need to use this technique when dumping them.
628 	 *
629 	 * Note: when adding special dump handling for any new sparsely-
630 	 * populated segments, be sure to add similar handling to the ::kgrep
631 	 * code in mdb.
632 	 */
633 	if (seg == &kvseg) {
634 		vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT,
635 		    segkmem_dump_range, seg->s_as);
636 #ifndef __sparc
637 		vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
638 		    segkmem_dump_range, seg->s_as);
639 #endif
640 	} else if (seg == &kvseg_core) {
641 		vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT,
642 		    segkmem_dump_range, seg->s_as);
643 	} else if (seg == &kvseg32) {
644 		vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT,
645 		    segkmem_dump_range, seg->s_as);
646 		vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
647 		    segkmem_dump_range, seg->s_as);
648 	} else {
649 		segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size);
650 	}
651 }
652 
653 /*
654  * lock/unlock kmem pages over a given range [addr, addr+len).
655  * Returns a shadow list of pages in ppp. If there are holes
656  * in the range (e.g. some of the kernel mappings do not have
657  * underlying page_ts) returns ENOTSUP so that as_pagelock()
658  * will handle the range via as_fault(F_SOFTLOCK).
659  */
660 /*ARGSUSED*/
661 static int
662 segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len,
663 	page_t ***ppp, enum lock_type type, enum seg_rw rw)
664 {
665 	page_t **pplist, *pp;
666 	pgcnt_t npages;
667 	spgcnt_t pg;
668 	size_t nb;
669 
670 	ASSERT(ppp != NULL);
671 
672 	if (segkp_bitmap && seg == &kvseg) {
673 		/*
674 		 * If it is one of segkp pages, call into segkp.
675 		 */
676 		if (BT_TEST(segkp_bitmap,
677 			btop((uintptr_t)(addr - seg->s_base))))
678 			return (SEGOP_PAGELOCK(segkp, addr, len, ppp,
679 						type, rw));
680 	}
681 
682 	if (type == L_PAGERECLAIM)
683 		return (ENOTSUP);
684 
685 	npages = btopr(len);
686 	nb = sizeof (page_t *) * npages;
687 
688 	if (type == L_PAGEUNLOCK) {
689 		pplist = *ppp;
690 		ASSERT(pplist != NULL);
691 
692 		for (pg = 0; pg < npages; pg++) {
693 			pp = pplist[pg];
694 			page_unlock(pp);
695 		}
696 		kmem_free(pplist, nb);
697 		return (0);
698 	}
699 
700 	ASSERT(type == L_PAGELOCK);
701 
702 	pplist = kmem_alloc(nb, KM_NOSLEEP);
703 	if (pplist == NULL) {
704 		*ppp = NULL;
705 		return (ENOTSUP);	/* take the slow path */
706 	}
707 
708 	for (pg = 0; pg < npages; pg++) {
709 		pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_SHARED);
710 		if (pp == NULL) {
711 			while (--pg >= 0)
712 				page_unlock(pplist[pg]);
713 			kmem_free(pplist, nb);
714 			*ppp = NULL;
715 			return (ENOTSUP);
716 		}
717 		pplist[pg] = pp;
718 		addr += PAGESIZE;
719 	}
720 
721 	*ppp = pplist;
722 	return (0);
723 }
724 
725 /*
726  * This is a dummy segkmem function overloaded to call segkp
727  * when segkp is under the heap.
728  */
729 /* ARGSUSED */
730 static int
731 segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
732 {
733 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
734 
735 	if (seg->s_as != &kas)
736 		segkmem_badop();
737 
738 	if (segkp_bitmap && seg == &kvseg) {
739 
740 		/*
741 		 * If it is one of segkp pages, call into segkp.
742 		 */
743 		if (BT_TEST(segkp_bitmap,
744 			btop((uintptr_t)(addr - seg->s_base))))
745 			return (SEGOP_GETMEMID(segkp, addr, memidp));
746 	}
747 	segkmem_badop();
748 	return (0);
749 }
750 
751 /*ARGSUSED*/
752 static lgrp_mem_policy_info_t *
753 segkmem_getpolicy(struct seg *seg, caddr_t addr)
754 {
755 	return (NULL);
756 }
757 
758 /*ARGSUSED*/
759 static int
760 segkmem_capable(struct seg *seg, segcapability_t capability)
761 {
762 	if (capability == S_CAPABILITY_NOMINFLT)
763 		return (1);
764 	return (0);
765 }
766 
767 static struct seg_ops segkmem_ops = {
768 	SEGKMEM_BADOP(int),		/* dup */
769 	SEGKMEM_BADOP(int),		/* unmap */
770 	SEGKMEM_BADOP(void),		/* free */
771 	segkmem_fault,
772 	SEGKMEM_BADOP(faultcode_t),	/* faulta */
773 	segkmem_setprot,
774 	segkmem_checkprot,
775 	segkmem_kluster,
776 	SEGKMEM_BADOP(size_t),		/* swapout */
777 	SEGKMEM_BADOP(int),		/* sync */
778 	SEGKMEM_BADOP(size_t),		/* incore */
779 	SEGKMEM_BADOP(int),		/* lockop */
780 	SEGKMEM_BADOP(int),		/* getprot */
781 	SEGKMEM_BADOP(u_offset_t),	/* getoffset */
782 	SEGKMEM_BADOP(int),		/* gettype */
783 	SEGKMEM_BADOP(int),		/* getvp */
784 	SEGKMEM_BADOP(int),		/* advise */
785 	segkmem_dump,
786 	segkmem_pagelock,
787 	SEGKMEM_BADOP(int),		/* setpgsz */
788 	segkmem_getmemid,
789 	segkmem_getpolicy,		/* getpolicy */
790 	segkmem_capable,		/* capable */
791 };
792 
793 int
794 segkmem_create(struct seg *seg)
795 {
796 	ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
797 	seg->s_ops = &segkmem_ops;
798 	seg->s_data = NULL;
799 	kas.a_size += seg->s_size;
800 	return (0);
801 }
802 
803 /*ARGSUSED*/
804 page_t *
805 segkmem_page_create(void *addr, size_t size, int vmflag, void *arg)
806 {
807 	struct seg kseg;
808 	int pgflags;
809 
810 	kseg.s_as = &kas;
811 	pgflags = PG_EXCL;
812 
813 	if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
814 		pgflags |= PG_NORELOC;
815 	if ((vmflag & VM_NOSLEEP) == 0)
816 		pgflags |= PG_WAIT;
817 	if (vmflag & VM_PANIC)
818 		pgflags |= PG_PANIC;
819 	if (vmflag & VM_PUSHPAGE)
820 		pgflags |= PG_PUSHPAGE;
821 
822 	return (page_create_va(&kvp, (u_offset_t)(uintptr_t)addr, size,
823 	    pgflags, &kseg, addr));
824 }
825 
826 /*
827  * Allocate pages to back the virtual address range [addr, addr + size).
828  * If addr is NULL, allocate the virtual address space as well.
829  */
830 void *
831 segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr,
832 	page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg)
833 {
834 	page_t *ppl;
835 	caddr_t addr = inaddr;
836 	pgcnt_t npages = btopr(size);
837 	int allocflag;
838 
839 	if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
840 		return (NULL);
841 
842 	ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
843 
844 	if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
845 		if (inaddr == NULL)
846 			vmem_free(vmp, addr, size);
847 		return (NULL);
848 	}
849 
850 	ppl = page_create_func(addr, size, vmflag, pcarg);
851 	if (ppl == NULL) {
852 		if (inaddr == NULL)
853 			vmem_free(vmp, addr, size);
854 		page_unresv(npages);
855 		return (NULL);
856 	}
857 
858 	/*
859 	 * Under certain conditions, we need to let the HAT layer know
860 	 * that it cannot safely allocate memory.  Allocations from
861 	 * the hat_memload vmem arena always need this, to prevent
862 	 * infinite recursion.
863 	 *
864 	 * In addition, the x86 hat cannot safely do memory
865 	 * allocations while in vmem_populate(), because there
866 	 * is no simple bound on its usage.
867 	 */
868 	if (vmflag & VM_MEMLOAD)
869 		allocflag = HAT_NO_KALLOC;
870 #if defined(__x86)
871 	else if (vmem_is_populator())
872 		allocflag = HAT_NO_KALLOC;
873 #endif
874 	else
875 		allocflag = 0;
876 
877 	while (ppl != NULL) {
878 		page_t *pp = ppl;
879 		page_sub(&ppl, pp);
880 		ASSERT(page_iolock_assert(pp));
881 		ASSERT(PAGE_EXCL(pp));
882 		page_io_unlock(pp);
883 		hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp,
884 		    (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
885 		    HAT_LOAD_LOCK | allocflag);
886 		pp->p_lckcnt = 1;
887 #if defined(__x86)
888 		page_downgrade(pp);
889 #else
890 		if (vmflag & SEGKMEM_SHARELOCKED)
891 			page_downgrade(pp);
892 		else
893 			page_unlock(pp);
894 #endif
895 	}
896 
897 	return (addr);
898 }
899 
900 void *
901 segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
902 {
903 	void *addr;
904 	segkmem_gc_list_t *gcp, **prev_gcpp;
905 
906 	if (kvseg.s_base == NULL) {
907 #ifndef __sparc
908 		if (bootops->bsys_alloc == NULL)
909 			halt("Memory allocation between bop_alloc() and "
910 			    "kmem_alloc().\n");
911 #endif
912 
913 		/*
914 		 * There's not a lot of memory to go around during boot,
915 		 * so recycle it if we can.
916 		 */
917 		for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL;
918 		    prev_gcpp = &gcp->gc_next) {
919 			if (gcp->gc_arena == vmp && gcp->gc_size == size) {
920 				*prev_gcpp = gcp->gc_next;
921 				return (gcp);
922 			}
923 		}
924 
925 		addr = vmem_alloc(vmp, size, vmflag | VM_PANIC);
926 		if (boot_alloc(addr, size, BO_NO_ALIGN) != addr)
927 			panic("segkmem_alloc: boot_alloc failed");
928 		return (addr);
929 	}
930 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
931 	    segkmem_page_create, NULL));
932 }
933 
934 /*
935  * Any changes to this routine must also be carried over to
936  * devmap_free_pages() in the seg_dev driver. This is because
937  * we currently don't have a special kernel segment for non-paged
938  * kernel memory that is exported by drivers to user space.
939  */
940 void
941 segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
942 {
943 	page_t *pp;
944 	caddr_t addr = inaddr;
945 	caddr_t eaddr;
946 	pgcnt_t npages = btopr(size);
947 
948 	ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
949 
950 	if (kvseg.s_base == NULL) {
951 		segkmem_gc_list_t *gc = inaddr;
952 		gc->gc_arena = vmp;
953 		gc->gc_size = size;
954 		gc->gc_next = segkmem_gc_list;
955 		segkmem_gc_list = gc;
956 		return;
957 	}
958 
959 	hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
960 
961 	for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
962 #if defined(__x86)
963 		pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
964 		if (pp == NULL)
965 			panic("segkmem_free: page not found");
966 		if (!page_tryupgrade(pp)) {
967 			/*
968 			 * Some other thread has a sharelock. Wait for
969 			 * it to drop the lock so we can free this page.
970 			 */
971 			page_unlock(pp);
972 			pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
973 			    SE_EXCL);
974 		}
975 #else
976 		pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
977 #endif
978 		if (pp == NULL)
979 			panic("segkmem_free: page not found");
980 		/* Clear p_lckcnt so page_destroy() doesn't update availrmem */
981 		pp->p_lckcnt = 0;
982 		page_destroy(pp, 0);
983 	}
984 	page_unresv(npages);
985 
986 	if (vmp != NULL)
987 		vmem_free(vmp, inaddr, size);
988 }
989 
990 void
991 segkmem_gc(void)
992 {
993 	ASSERT(kvseg.s_base != NULL);
994 	while (segkmem_gc_list != NULL) {
995 		segkmem_gc_list_t *gc = segkmem_gc_list;
996 		segkmem_gc_list = gc->gc_next;
997 		segkmem_free(gc->gc_arena, gc, gc->gc_size);
998 	}
999 }
1000 
1001 /*
1002  * Legacy entry points from here to end of file.
1003  */
1004 void
1005 segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot,
1006     pfn_t pfn, uint_t flags)
1007 {
1008 	hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1009 	hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot,
1010 	    flags | HAT_LOAD_LOCK);
1011 }
1012 
1013 void
1014 segkmem_mapout(struct seg *seg, void *addr, size_t size)
1015 {
1016 	hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1017 }
1018 
1019 void *
1020 kmem_getpages(pgcnt_t npages, int kmflag)
1021 {
1022 	return (kmem_alloc(ptob(npages), kmflag));
1023 }
1024 
1025 void
1026 kmem_freepages(void *addr, pgcnt_t npages)
1027 {
1028 	kmem_free(addr, ptob(npages));
1029 }
1030 
1031 /*
1032  * segkmem_page_create_large() allocates a large page to be used for the kmem
1033  * caches. If kpr is enabled we ask for a relocatable page unless requested
1034  * otherwise. If kpr is disabled we have to ask for a non-reloc page
1035  */
1036 static page_t *
1037 segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg)
1038 {
1039 	int pgflags;
1040 
1041 	pgflags = PG_EXCL;
1042 
1043 	if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
1044 		pgflags |= PG_NORELOC;
1045 	if (!(vmflag & VM_NOSLEEP))
1046 		pgflags |= PG_WAIT;
1047 	if (vmflag & VM_PUSHPAGE)
1048 		pgflags |= PG_PUSHPAGE;
1049 
1050 	return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
1051 	    pgflags, &kvseg, addr, arg));
1052 }
1053 
1054 /*
1055  * Allocate a large page to back the virtual address range
1056  * [addr, addr + size).  If addr is NULL, allocate the virtual address
1057  * space as well.
1058  */
1059 static void *
1060 segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
1061     uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
1062     void *pcarg)
1063 {
1064 	caddr_t addr = inaddr, pa;
1065 	size_t  lpsize = segkmem_lpsize;
1066 	pgcnt_t npages = btopr(size);
1067 	pgcnt_t nbpages = btop(lpsize);
1068 	pgcnt_t nlpages = size >> segkmem_lpshift;
1069 	size_t  ppasize = nbpages * sizeof (page_t *);
1070 	page_t *pp, *rootpp, **ppa, *pplist = NULL;
1071 	int i;
1072 
1073 	if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
1074 		return (NULL);
1075 	}
1076 
1077 	/*
1078 	 * allocate an array we need for hat_memload_array.
1079 	 * we use a separate arena to avoid recursion.
1080 	 * we will not need this array when hat_memload_array learns pp++
1081 	 */
1082 	if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) {
1083 		goto fail_array_alloc;
1084 	}
1085 
1086 	if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
1087 		goto fail_vmem_alloc;
1088 
1089 	ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0);
1090 
1091 	/* create all the pages */
1092 	for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) {
1093 		if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL)
1094 			goto fail_page_create;
1095 		page_list_concat(&pplist, &pp);
1096 	}
1097 
1098 	/* at this point we have all the resource to complete the request */
1099 	while ((rootpp = pplist) != NULL) {
1100 		for (i = 0; i < nbpages; i++) {
1101 			ASSERT(pplist != NULL);
1102 			pp = pplist;
1103 			page_sub(&pplist, pp);
1104 			ASSERT(page_iolock_assert(pp));
1105 			page_io_unlock(pp);
1106 			ppa[i] = pp;
1107 		}
1108 		/*
1109 		 * Load the locked entry. It's OK to preload the entry into the
1110 		 * TSB since we now support large mappings in the kernel TSB.
1111 		 */
1112 		hat_memload_array(kas.a_hat,
1113 		    (caddr_t)(uintptr_t)rootpp->p_offset, lpsize,
1114 		    ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
1115 		    HAT_LOAD_LOCK);
1116 
1117 		for (--i; i >= 0; --i) {
1118 			ppa[i]->p_lckcnt = 1;
1119 			page_unlock(ppa[i]);
1120 		}
1121 	}
1122 
1123 	vmem_free(segkmem_ppa_arena, ppa, ppasize);
1124 	return (addr);
1125 
1126 fail_page_create:
1127 	while ((rootpp = pplist) != NULL) {
1128 		for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) {
1129 			ASSERT(pp != NULL);
1130 			page_sub(&pplist, pp);
1131 			ASSERT(page_iolock_assert(pp));
1132 			page_io_unlock(pp);
1133 		}
1134 		page_destroy_pages(rootpp);
1135 	}
1136 
1137 	if (inaddr == NULL)
1138 		vmem_free(vmp, addr, size);
1139 
1140 fail_vmem_alloc:
1141 	vmem_free(segkmem_ppa_arena, ppa, ppasize);
1142 
1143 fail_array_alloc:
1144 	page_unresv(npages);
1145 
1146 	return (NULL);
1147 }
1148 
1149 static void
1150 segkmem_free_one_lp(caddr_t addr, size_t size)
1151 {
1152 	page_t		*pp, *rootpp = NULL;
1153 	pgcnt_t 	pgs_left = btopr(size);
1154 
1155 	ASSERT(size == segkmem_lpsize);
1156 
1157 	hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1158 
1159 	for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) {
1160 		pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1161 		if (pp == NULL)
1162 			panic("segkmem_free_one_lp: page not found");
1163 		ASSERT(PAGE_EXCL(pp));
1164 		pp->p_lckcnt = 0;
1165 		if (rootpp == NULL)
1166 			rootpp = pp;
1167 	}
1168 	ASSERT(rootpp != NULL);
1169 	page_destroy_pages(rootpp);
1170 
1171 	/* page_unresv() is done by the caller */
1172 }
1173 
1174 /*
1175  * This function is called to import new spans into the vmem arenas like
1176  * kmem_default_arena and kmem_oversize_arena. It first tries to import
1177  * spans from large page arena - kmem_lp_arena. In order to do this it might
1178  * have to "upgrade the requested size" to kmem_lp_arena quantum. If
1179  * it was not able to satisfy the upgraded request it then calls regular
1180  * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
1181  */
1182 void *
1183 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, int vmflag)
1184 {
1185 	size_t size;
1186 	kthread_t *t = curthread;
1187 	segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1188 
1189 	ASSERT(sizep != NULL);
1190 
1191 	size = *sizep;
1192 
1193 	if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
1194 	    !(vmflag & SEGKMEM_SHARELOCKED)) {
1195 
1196 		size_t kmemlp_qnt = segkmem_kmemlp_quantum;
1197 		size_t asize = P2ROUNDUP(size, kmemlp_qnt);
1198 		void  *addr = NULL;
1199 		ulong_t *lpthrtp = &lpcb->lp_throttle;
1200 		ulong_t lpthrt = *lpthrtp;
1201 		int	dowakeup = 0;
1202 		int	doalloc = 1;
1203 
1204 		ASSERT(kmem_lp_arena != NULL);
1205 		ASSERT(asize >= size);
1206 
1207 		if (lpthrt != 0) {
1208 			/* try to update the throttle value */
1209 			lpthrt = atomic_add_long_nv(lpthrtp, 1);
1210 			if (lpthrt >= segkmem_lpthrottle_max) {
1211 				lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
1212 				    segkmem_lpthrottle_max / 4);
1213 			}
1214 
1215 			/*
1216 			 * when we get above throttle start do an exponential
1217 			 * backoff at trying large pages and reaping
1218 			 */
1219 			if (lpthrt > segkmem_lpthrottle_start &&
1220 			    (lpthrt & (lpthrt - 1))) {
1221 				lpcb->allocs_throttled++;
1222 				lpthrt--;
1223 				if ((lpthrt & (lpthrt - 1)) == 0)
1224 					kmem_reap();
1225 				return (segkmem_alloc(vmp, size, vmflag));
1226 			}
1227 		}
1228 
1229 		if (!(vmflag & VM_NOSLEEP) &&
1230 		    segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
1231 		    vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
1232 		    asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
1233 
1234 			/*
1235 			 * we are low on free memory in kmem_lp_arena
1236 			 * we let only one guy to allocate heap_lp
1237 			 * quantum size chunk that everybody is going to
1238 			 * share
1239 			 */
1240 			mutex_enter(&lpcb->lp_lock);
1241 
1242 			if (lpcb->lp_wait) {
1243 
1244 				/* we are not the first one - wait */
1245 				cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
1246 				if (vmem_size(kmem_lp_arena, VMEM_FREE) <
1247 				    kmemlp_qnt)  {
1248 					doalloc = 0;
1249 				}
1250 			} else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
1251 			    kmemlp_qnt) {
1252 
1253 				/*
1254 				 * we are the first one, make sure we import
1255 				 * a large page
1256 				 */
1257 				if (asize == kmemlp_qnt)
1258 					asize += kmemlp_qnt;
1259 				dowakeup = 1;
1260 				lpcb->lp_wait = 1;
1261 			}
1262 
1263 			mutex_exit(&lpcb->lp_lock);
1264 		}
1265 
1266 		/*
1267 		 * VM_ABORT flag prevents sleeps in vmem_xalloc when
1268 		 * large pages are not available. In that case this allocation
1269 		 * attempt will fail and we will retry allocation with small
1270 		 * pages. We also do not want to panic if this allocation fails
1271 		 * because we are going to retry.
1272 		 */
1273 		if (doalloc) {
1274 			addr = vmem_alloc(kmem_lp_arena, asize,
1275 			    (vmflag | VM_ABORT) & ~VM_PANIC);
1276 
1277 			if (dowakeup) {
1278 				mutex_enter(&lpcb->lp_lock);
1279 				ASSERT(lpcb->lp_wait != 0);
1280 				lpcb->lp_wait = 0;
1281 				cv_broadcast(&lpcb->lp_cv);
1282 				mutex_exit(&lpcb->lp_lock);
1283 			}
1284 		}
1285 
1286 		if (addr != NULL) {
1287 			*sizep = asize;
1288 			*lpthrtp = 0;
1289 			return (addr);
1290 		}
1291 
1292 		if (vmflag & VM_NOSLEEP)
1293 			lpcb->nosleep_allocs_failed++;
1294 		else
1295 			lpcb->sleep_allocs_failed++;
1296 		lpcb->alloc_bytes_failed += size;
1297 
1298 		/* if large page throttling is not started yet do it */
1299 		if (segkmem_use_lpthrottle && lpthrt == 0) {
1300 			lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
1301 		}
1302 	}
1303 	return (segkmem_alloc(vmp, size, vmflag));
1304 }
1305 
1306 void
1307 segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size)
1308 {
1309 	if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) {
1310 		segkmem_free(vmp, inaddr, size);
1311 	} else {
1312 		vmem_free(kmem_lp_arena, inaddr, size);
1313 	}
1314 }
1315 
1316 /*
1317  * segkmem_alloc_lpi() imports virtual memory from large page heap arena
1318  * into kmem_lp arena. In the process it maps the imported segment with
1319  * large pages
1320  */
1321 static void *
1322 segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag)
1323 {
1324 	segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1325 	void  *addr;
1326 
1327 	ASSERT(size != 0);
1328 	ASSERT(vmp == heap_lp_arena);
1329 
1330 	/* do not allow large page heap grow beyound limits */
1331 	if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) {
1332 		lpcb->allocs_limited++;
1333 		return (NULL);
1334 	}
1335 
1336 	addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0,
1337 	    segkmem_page_create_large, NULL);
1338 	return (addr);
1339 }
1340 
1341 /*
1342  * segkmem_free_lpi() returns virtual memory back into large page heap arena
1343  * from kmem_lp arena. Beore doing this it unmaps the segment and frees
1344  * large pages used to map it.
1345  */
1346 static void
1347 segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size)
1348 {
1349 	pgcnt_t		nlpages = size >> segkmem_lpshift;
1350 	size_t		lpsize = segkmem_lpsize;
1351 	caddr_t		addr = inaddr;
1352 	pgcnt_t 	npages = btopr(size);
1353 	int		i;
1354 
1355 	ASSERT(vmp == heap_lp_arena);
1356 	ASSERT(IS_KMEM_VA_LARGEPAGE(addr));
1357 	ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0);
1358 
1359 	for (i = 0; i < nlpages; i++) {
1360 		segkmem_free_one_lp(addr, lpsize);
1361 		addr += lpsize;
1362 	}
1363 
1364 	page_unresv(npages);
1365 
1366 	vmem_free(vmp, inaddr, size);
1367 }
1368 
1369 /*
1370  * This function is called at system boot time by kmem_init right after
1371  * /etc/system file has been read. It checks based on hardware configuration
1372  * and /etc/system settings if system is going to use large pages. The
1373  * initialiazation necessary to actually start using large pages
1374  * happens later in the process after segkmem_heap_lp_init() is called.
1375  */
1376 int
1377 segkmem_lpsetup()
1378 {
1379 	int use_large_pages = 0;
1380 
1381 #ifdef __sparc
1382 
1383 	size_t memtotal = physmem * PAGESIZE;
1384 
1385 	if (heap_lp_base == NULL) {
1386 		segkmem_lpsize = PAGESIZE;
1387 		return (0);
1388 	}
1389 
1390 	/* get a platform dependent value of large page size for kernel heap */
1391 	segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize);
1392 
1393 	if (segkmem_lpsize <= PAGESIZE) {
1394 		/*
1395 		 * put virtual space reserved for the large page kernel
1396 		 * back to the regular heap
1397 		 */
1398 		vmem_xfree(heap_arena, heap_lp_base,
1399 		    heap_lp_end - heap_lp_base);
1400 		heap_lp_base = NULL;
1401 		heap_lp_end = NULL;
1402 		segkmem_lpsize = PAGESIZE;
1403 		return (0);
1404 	}
1405 
1406 	/* set heap_lp quantum if necessary */
1407 	if (segkmem_heaplp_quantum == 0 ||
1408 	    (segkmem_heaplp_quantum & (segkmem_heaplp_quantum - 1)) ||
1409 	    P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) {
1410 		segkmem_heaplp_quantum = segkmem_lpsize;
1411 	}
1412 
1413 	/* set kmem_lp quantum if necessary */
1414 	if (segkmem_kmemlp_quantum == 0 ||
1415 	    (segkmem_kmemlp_quantum & (segkmem_kmemlp_quantum - 1)) ||
1416 	    segkmem_kmemlp_quantum > segkmem_heaplp_quantum) {
1417 		segkmem_kmemlp_quantum = segkmem_heaplp_quantum;
1418 	}
1419 
1420 	/* set total amount of memory allowed for large page kernel heap */
1421 	if (segkmem_kmemlp_max == 0) {
1422 		if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100)
1423 			segkmem_kmemlp_pcnt = 12;
1424 		segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100;
1425 	}
1426 	segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max,
1427 	    segkmem_heaplp_quantum);
1428 
1429 	/* fix lp kmem preallocation request if necesssary */
1430 	if (segkmem_kmemlp_min) {
1431 		segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min,
1432 		    segkmem_heaplp_quantum);
1433 		if (segkmem_kmemlp_min > segkmem_kmemlp_max)
1434 			segkmem_kmemlp_min = segkmem_kmemlp_max;
1435 	}
1436 
1437 	use_large_pages = 1;
1438 	segkmem_lpshift = page_get_shift(page_szc(segkmem_lpsize));
1439 
1440 #endif
1441 	return (use_large_pages);
1442 }
1443 
1444 #ifdef __sparc
1445 
1446 
1447 static void *
1448 segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag)
1449 {
1450 	size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1451 	void   *addr;
1452 
1453 	if (ppaquantum <= PAGESIZE)
1454 		return (segkmem_alloc(vmp, size, vmflag));
1455 
1456 	ASSERT((size & (ppaquantum - 1)) == 0);
1457 
1458 	addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag);
1459 	if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0,
1460 		segkmem_page_create, NULL) == NULL) {
1461 		vmem_xfree(vmp, addr, size);
1462 		addr = NULL;
1463 	}
1464 
1465 	return (addr);
1466 }
1467 
1468 static void
1469 segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size)
1470 {
1471 	size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1472 
1473 	ASSERT(addr != NULL);
1474 
1475 	if (ppaquantum <= PAGESIZE) {
1476 		segkmem_free(vmp, addr, size);
1477 	} else {
1478 		segkmem_free(NULL, addr, size);
1479 		vmem_xfree(vmp, addr, size);
1480 	}
1481 }
1482 
1483 void
1484 segkmem_heap_lp_init()
1485 {
1486 	segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1487 	size_t heap_lp_size = heap_lp_end - heap_lp_base;
1488 	size_t lpsize = segkmem_lpsize;
1489 	size_t ppaquantum;
1490 	void   *addr;
1491 
1492 	if (segkmem_lpsize <= PAGESIZE) {
1493 		ASSERT(heap_lp_base == NULL);
1494 		ASSERT(heap_lp_end == NULL);
1495 		return;
1496 	}
1497 
1498 	ASSERT(segkmem_heaplp_quantum >= lpsize);
1499 	ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0);
1500 	ASSERT(lpcb->lp_uselp == 0);
1501 	ASSERT(heap_lp_base != NULL);
1502 	ASSERT(heap_lp_end != NULL);
1503 	ASSERT(heap_lp_base < heap_lp_end);
1504 	ASSERT(heap_lp_arena == NULL);
1505 	ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0);
1506 	ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0);
1507 
1508 	/* create large page heap arena */
1509 	heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size,
1510 	    segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1511 
1512 	ASSERT(heap_lp_arena != NULL);
1513 
1514 	/* This arena caches memory already mapped by large pages */
1515 	kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum,
1516 	    segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP);
1517 
1518 	ASSERT(kmem_lp_arena != NULL);
1519 
1520 	mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL);
1521 	cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL);
1522 
1523 	/*
1524 	 * this arena is used for the array of page_t pointers necessary
1525 	 * to call hat_mem_load_array
1526 	 */
1527 	ppaquantum = btopr(lpsize) * sizeof (page_t *);
1528 	segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum,
1529 	    segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum,
1530 	    VM_SLEEP);
1531 
1532 	ASSERT(segkmem_ppa_arena != NULL);
1533 
1534 	/* prealloacate some memory for the lp kernel heap */
1535 	if (segkmem_kmemlp_min) {
1536 
1537 		ASSERT(P2PHASE(segkmem_kmemlp_min,
1538 		    segkmem_heaplp_quantum) == 0);
1539 
1540 		if ((addr = segkmem_alloc_lpi(heap_lp_arena,
1541 		    segkmem_kmemlp_min, VM_SLEEP)) != NULL) {
1542 
1543 			addr = vmem_add(kmem_lp_arena, addr,
1544 			    segkmem_kmemlp_min, VM_SLEEP);
1545 			ASSERT(addr != NULL);
1546 		}
1547 	}
1548 
1549 	lpcb->lp_uselp = 1;
1550 }
1551 
1552 #endif
1553