xref: /titanic_51/usr/src/uts/common/vm/seg_kmem.c (revision 17723245e5b42ca8abb9964a36c0b051c0b237fe)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <sys/types.h>
26 #include <sys/t_lock.h>
27 #include <sys/param.h>
28 #include <sys/sysmacros.h>
29 #include <sys/tuneable.h>
30 #include <sys/systm.h>
31 #include <sys/vm.h>
32 #include <sys/kmem.h>
33 #include <sys/vmem.h>
34 #include <sys/mman.h>
35 #include <sys/cmn_err.h>
36 #include <sys/debug.h>
37 #include <sys/dumphdr.h>
38 #include <sys/bootconf.h>
39 #include <sys/lgrp.h>
40 #include <vm/seg_kmem.h>
41 #include <vm/hat.h>
42 #include <vm/page.h>
43 #include <vm/vm_dep.h>
44 #include <vm/faultcode.h>
45 #include <sys/promif.h>
46 #include <vm/seg_kp.h>
47 #include <sys/bitmap.h>
48 #include <sys/mem_cage.h>
49 
50 #ifdef __sparc
51 #include <sys/ivintr.h>
52 #include <sys/panic.h>
53 #endif
54 
55 /*
56  * seg_kmem is the primary kernel memory segment driver.  It
57  * maps the kernel heap [kernelheap, ekernelheap), module text,
58  * and all memory which was allocated before the VM was initialized
59  * into kas.
60  *
61  * Pages which belong to seg_kmem are hashed into &kvp vnode at
62  * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1.
63  * They must never be paged out since segkmem_fault() is a no-op to
64  * prevent recursive faults.
65  *
66  * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
67  * __x86 and are unlocked (p_sharelock == 0) on __sparc.  Once __x86
68  * supports relocation the #ifdef kludges can be removed.
69  *
70  * seg_kmem pages may be subject to relocation by page_relocate(),
71  * provided that the HAT supports it; if this is so, segkmem_reloc
72  * will be set to a nonzero value. All boot time allocated memory as
73  * well as static memory is considered off limits to relocation.
74  * Pages are "relocatable" if p_state does not have P_NORELOC set, so
75  * we request P_NORELOC pages for memory that isn't safe to relocate.
76  *
77  * The kernel heap is logically divided up into four pieces:
78  *
79  *   heap32_arena is for allocations that require 32-bit absolute
80  *   virtual addresses (e.g. code that uses 32-bit pointers/offsets).
81  *
82  *   heap_core is for allocations that require 2GB *relative*
83  *   offsets; in other words all memory from heap_core is within
84  *   2GB of all other memory from the same arena. This is a requirement
85  *   of the addressing modes of some processors in supervisor code.
86  *
87  *   heap_arena is the general heap arena.
88  *
89  *   static_arena is the static memory arena.  Allocations from it
90  *   are not subject to relocation so it is safe to use the memory
91  *   physical address as well as the virtual address (e.g. the VA to
92  *   PA translations are static).  Caches may import from static_arena;
93  *   all other static memory allocations should use static_alloc_arena.
94  *
95  * On some platforms which have limited virtual address space, seg_kmem
96  * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
97  * segkp_bitmap is non-NULL, and each bit represents a page of virtual
98  * address space which is actually seg_kp mapped.
99  */
100 
101 extern ulong_t *segkp_bitmap;   /* Is set if segkp is from the kernel heap */
102 
103 char *kernelheap;		/* start of primary kernel heap */
104 char *ekernelheap;		/* end of primary kernel heap */
105 struct seg kvseg;		/* primary kernel heap segment */
106 struct seg kvseg_core;		/* "core" kernel heap segment */
107 struct seg kzioseg;		/* Segment for zio mappings */
108 vmem_t *heap_arena;		/* primary kernel heap arena */
109 vmem_t *heap_core_arena;	/* core kernel heap arena */
110 char *heap_core_base;		/* start of core kernel heap arena */
111 char *heap_lp_base;		/* start of kernel large page heap arena */
112 char *heap_lp_end;		/* end of kernel large page heap arena */
113 vmem_t *hat_memload_arena;	/* HAT translation data */
114 struct seg kvseg32;		/* 32-bit kernel heap segment */
115 vmem_t *heap32_arena;		/* 32-bit kernel heap arena */
116 vmem_t *heaptext_arena;		/* heaptext arena */
117 struct as kas;			/* kernel address space */
118 int segkmem_reloc;		/* enable/disable relocatable segkmem pages */
119 vmem_t *static_arena;		/* arena for caches to import static memory */
120 vmem_t *static_alloc_arena;	/* arena for allocating static memory */
121 vmem_t *zio_arena = NULL;	/* arena for allocating zio memory */
122 vmem_t *zio_alloc_arena = NULL;	/* arena for allocating zio memory */
123 
124 /*
125  * seg_kmem driver can map part of the kernel heap with large pages.
126  * Currently this functionality is implemented for sparc platforms only.
127  *
128  * The large page size "segkmem_lpsize" for kernel heap is selected in the
129  * platform specific code. It can also be modified via /etc/system file.
130  * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
131  * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
132  * match segkmem_lpsize.
133  *
134  * At boot time we carve from kernel heap arena a range of virtual addresses
135  * that will be used for large page mappings. This range [heap_lp_base,
136  * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
137  * create "kmem_lp_arena" that caches memory already backed up by large
138  * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
139  */
140 
141 size_t	segkmem_lpsize;
142 static  uint_t	segkmem_lpshift = PAGESHIFT;
143 int	segkmem_lpszc = 0;
144 
145 size_t  segkmem_kmemlp_quantum = 0x400000;	/* 4MB */
146 size_t  segkmem_heaplp_quantum;
147 vmem_t *heap_lp_arena;
148 static  vmem_t *kmem_lp_arena;
149 static  vmem_t *segkmem_ppa_arena;
150 static	segkmem_lpcb_t segkmem_lpcb;
151 
152 /*
153  * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
154  * consumed by the large page heap. By default this parameter is set to 1/8 of
155  * physmem but can be adjusted through /etc/system either directly or
156  * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
157  * we allow for large page heap.
158  */
159 size_t  segkmem_kmemlp_max;
160 static  uint_t  segkmem_kmemlp_pcnt;
161 
162 /*
163  * Getting large pages for kernel heap could be problematic due to
164  * physical memory fragmentation. That's why we allow to preallocate
165  * "segkmem_kmemlp_min" bytes at boot time.
166  */
167 static  size_t	segkmem_kmemlp_min;
168 
169 /*
170  * Throttling is used to avoid expensive tries to allocate large pages
171  * for kernel heap when a lot of succesive attempts to do so fail.
172  */
173 static  ulong_t segkmem_lpthrottle_max = 0x400000;
174 static  ulong_t segkmem_lpthrottle_start = 0x40;
175 static  ulong_t segkmem_use_lpthrottle = 1;
176 
177 /*
178  * Freed pages accumulate on a garbage list until segkmem is ready,
179  * at which point we call segkmem_gc() to free it all.
180  */
181 typedef struct segkmem_gc_list {
182 	struct segkmem_gc_list	*gc_next;
183 	vmem_t			*gc_arena;
184 	size_t			gc_size;
185 } segkmem_gc_list_t;
186 
187 static segkmem_gc_list_t *segkmem_gc_list;
188 
189 /*
190  * Allocations from the hat_memload arena add VM_MEMLOAD to their
191  * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
192  * to take steps to prevent infinite recursion.  HAT allocations also
193  * must be non-relocatable to prevent recursive page faults.
194  */
195 static void *
196 hat_memload_alloc(vmem_t *vmp, size_t size, int flags)
197 {
198 	flags |= (VM_MEMLOAD | VM_NORELOC);
199 	return (segkmem_alloc(vmp, size, flags));
200 }
201 
202 /*
203  * Allocations from static_arena arena (or any other arena that uses
204  * segkmem_alloc_permanent()) require non-relocatable (permanently
205  * wired) memory pages, since these pages are referenced by physical
206  * as well as virtual address.
207  */
208 void *
209 segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags)
210 {
211 	return (segkmem_alloc(vmp, size, flags | VM_NORELOC));
212 }
213 
214 /*
215  * Initialize kernel heap boundaries.
216  */
217 void
218 kernelheap_init(
219 	void *heap_start,
220 	void *heap_end,
221 	char *first_avail,
222 	void *core_start,
223 	void *core_end)
224 {
225 	uintptr_t textbase;
226 	size_t core_size;
227 	size_t heap_size;
228 	vmem_t *heaptext_parent;
229 	size_t	heap_lp_size = 0;
230 #ifdef __sparc
231 	size_t kmem64_sz = kmem64_aligned_end - kmem64_base;
232 #endif	/* __sparc */
233 
234 	kernelheap = heap_start;
235 	ekernelheap = heap_end;
236 
237 #ifdef __sparc
238 	heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4);
239 	/*
240 	 * Bias heap_lp start address by kmem64_sz to reduce collisions
241 	 * in 4M kernel TSB between kmem64 area and heap_lp
242 	 */
243 	kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M);
244 	if (kmem64_sz <= heap_lp_size / 2)
245 		heap_lp_size -= kmem64_sz;
246 	heap_lp_base = ekernelheap - heap_lp_size;
247 	heap_lp_end = heap_lp_base + heap_lp_size;
248 #endif	/* __sparc */
249 
250 	/*
251 	 * If this platform has a 'core' heap area, then the space for
252 	 * overflow module text should be carved out of the end of that
253 	 * heap.  Otherwise, it gets carved out of the general purpose
254 	 * heap.
255 	 */
256 	core_size = (uintptr_t)core_end - (uintptr_t)core_start;
257 	if (core_size > 0) {
258 		ASSERT(core_size >= HEAPTEXT_SIZE);
259 		textbase = (uintptr_t)core_end - HEAPTEXT_SIZE;
260 		core_size -= HEAPTEXT_SIZE;
261 	}
262 #ifndef __sparc
263 	else {
264 		ekernelheap -= HEAPTEXT_SIZE;
265 		textbase = (uintptr_t)ekernelheap;
266 	}
267 #endif
268 
269 	heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap;
270 	heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE,
271 	    segkmem_alloc, segkmem_free);
272 
273 	if (core_size > 0) {
274 		heap_core_arena = vmem_create("heap_core", core_start,
275 		    core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
276 		heap_core_base = core_start;
277 	} else {
278 		heap_core_arena = heap_arena;
279 		heap_core_base = kernelheap;
280 	}
281 
282 	/*
283 	 * reserve space for the large page heap. If large pages for kernel
284 	 * heap is enabled large page heap arean will be created later in the
285 	 * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
286 	 * range will be returned back to the heap_arena.
287 	 */
288 	if (heap_lp_size) {
289 		(void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0,
290 		    heap_lp_base, heap_lp_end,
291 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
292 	}
293 
294 	/*
295 	 * Remove the already-spoken-for memory range [kernelheap, first_avail).
296 	 */
297 	(void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE,
298 	    0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
299 
300 #ifdef __sparc
301 	heap32_arena = vmem_create("heap32", (void *)SYSBASE32,
302 	    SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL,
303 	    NULL, NULL, 0, VM_SLEEP);
304 	/*
305 	 * Prom claims the physical and virtual resources used by panicbuf
306 	 * and inter_vec_table. So reserve space for panicbuf, intr_vec_table,
307 	 * reserved interrupt vector data structures from 32-bit heap.
308 	 */
309 	(void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
310 	    panicbuf, panicbuf + PANICBUFSIZE,
311 	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
312 
313 	(void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
314 	    intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
315 	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
316 
317 	textbase = SYSLIMIT32 - HEAPTEXT_SIZE;
318 	heaptext_parent = NULL;
319 #else	/* __sparc */
320 	heap32_arena = heap_core_arena;
321 	heaptext_parent = heap_core_arena;
322 #endif	/* __sparc */
323 
324 	heaptext_arena = vmem_create("heaptext", (void *)textbase,
325 	    HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP);
326 
327 	/*
328 	 * Create a set of arenas for memory with static translations
329 	 * (e.g. VA -> PA translations cannot change).  Since using
330 	 * kernel pages by physical address implies it isn't safe to
331 	 * walk across page boundaries, the static_arena quantum must
332 	 * be PAGESIZE.  Any kmem caches that require static memory
333 	 * should source from static_arena, while direct allocations
334 	 * should only use static_alloc_arena.
335 	 */
336 	static_arena = vmem_create("static", NULL, 0, PAGESIZE,
337 	    segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP);
338 	static_alloc_arena = vmem_create("static_alloc", NULL, 0,
339 	    sizeof (uint64_t), vmem_alloc, vmem_free, static_arena,
340 	    0, VM_SLEEP);
341 
342 	/*
343 	 * Create an arena for translation data (ptes, hmes, or hblks).
344 	 * We need an arena for this because hat_memload() is essential
345 	 * to vmem_populate() (see comments in common/os/vmem.c).
346 	 *
347 	 * Note: any kmem cache that allocates from hat_memload_arena
348 	 * must be created as a KMC_NOHASH cache (i.e. no external slab
349 	 * and bufctl structures to allocate) so that slab creation doesn't
350 	 * require anything more than a single vmem_alloc().
351 	 */
352 	hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE,
353 	    hat_memload_alloc, segkmem_free, heap_arena, 0,
354 	    VM_SLEEP | VMC_POPULATOR | VMC_DUMPSAFE);
355 }
356 
357 void
358 boot_mapin(caddr_t addr, size_t size)
359 {
360 	caddr_t	 eaddr;
361 	page_t	*pp;
362 	pfn_t	 pfnum;
363 
364 	if (page_resv(btop(size), KM_NOSLEEP) == 0)
365 		panic("boot_mapin: page_resv failed");
366 
367 	for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
368 		pfnum = va_to_pfn(addr);
369 		if (pfnum == PFN_INVALID)
370 			continue;
371 		if ((pp = page_numtopp_nolock(pfnum)) == NULL)
372 			panic("boot_mapin(): No pp for pfnum = %lx", pfnum);
373 
374 		/*
375 		 * must break up any large pages that may have constituent
376 		 * pages being utilized for BOP_ALLOC()'s before calling
377 		 * page_numtopp().The locking code (ie. page_reclaim())
378 		 * can't handle them
379 		 */
380 		if (pp->p_szc != 0)
381 			page_boot_demote(pp);
382 
383 		pp = page_numtopp(pfnum, SE_EXCL);
384 		if (pp == NULL || PP_ISFREE(pp))
385 			panic("boot_alloc: pp is NULL or free");
386 
387 		/*
388 		 * If the cage is on but doesn't yet contain this page,
389 		 * mark it as non-relocatable.
390 		 */
391 		if (kcage_on && !PP_ISNORELOC(pp)) {
392 			PP_SETNORELOC(pp);
393 			PLCNT_XFER_NORELOC(pp);
394 		}
395 
396 		(void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL);
397 		pp->p_lckcnt = 1;
398 #if defined(__x86)
399 		page_downgrade(pp);
400 #else
401 		page_unlock(pp);
402 #endif
403 	}
404 }
405 
406 /*
407  * Get pages from boot and hash them into the kernel's vp.
408  * Used after page structs have been allocated, but before segkmem is ready.
409  */
410 void *
411 boot_alloc(void *inaddr, size_t size, uint_t align)
412 {
413 	caddr_t addr = inaddr;
414 
415 	if (bootops == NULL)
416 		prom_panic("boot_alloc: attempt to allocate memory after "
417 		    "BOP_GONE");
418 
419 	size = ptob(btopr(size));
420 #ifdef __sparc
421 	if (bop_alloc_chunk(addr, size, align) != (caddr_t)addr)
422 		panic("boot_alloc: bop_alloc_chunk failed");
423 #else
424 	if (BOP_ALLOC(bootops, addr, size, align) != addr)
425 		panic("boot_alloc: BOP_ALLOC failed");
426 #endif
427 	boot_mapin((caddr_t)addr, size);
428 	return (addr);
429 }
430 
431 static void
432 segkmem_badop()
433 {
434 	panic("segkmem_badop");
435 }
436 
437 #define	SEGKMEM_BADOP(t)	(t(*)())segkmem_badop
438 
439 /*ARGSUSED*/
440 static faultcode_t
441 segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size,
442 	enum fault_type type, enum seg_rw rw)
443 {
444 	pgcnt_t npages;
445 	spgcnt_t pg;
446 	page_t *pp;
447 	struct vnode *vp = seg->s_data;
448 
449 	ASSERT(RW_READ_HELD(&seg->s_as->a_lock));
450 
451 	if (seg->s_as != &kas || size > seg->s_size ||
452 	    addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
453 		panic("segkmem_fault: bad args");
454 
455 	/*
456 	 * If it is one of segkp pages, call segkp_fault.
457 	 */
458 	if (segkp_bitmap && seg == &kvseg &&
459 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
460 		return (SEGOP_FAULT(hat, segkp, addr, size, type, rw));
461 
462 	if (rw != S_READ && rw != S_WRITE && rw != S_OTHER)
463 		return (FC_NOSUPPORT);
464 
465 	npages = btopr(size);
466 
467 	switch (type) {
468 	case F_SOFTLOCK:	/* lock down already-loaded translations */
469 		for (pg = 0; pg < npages; pg++) {
470 			pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
471 			    SE_SHARED);
472 			if (pp == NULL) {
473 				/*
474 				 * Hmm, no page. Does a kernel mapping
475 				 * exist for it?
476 				 */
477 				if (!hat_probe(kas.a_hat, addr)) {
478 					addr -= PAGESIZE;
479 					while (--pg >= 0) {
480 						pp = page_find(vp, (u_offset_t)
481 						    (uintptr_t)addr);
482 						if (pp)
483 							page_unlock(pp);
484 						addr -= PAGESIZE;
485 					}
486 					return (FC_NOMAP);
487 				}
488 			}
489 			addr += PAGESIZE;
490 		}
491 		if (rw == S_OTHER)
492 			hat_reserve(seg->s_as, addr, size);
493 		return (0);
494 	case F_SOFTUNLOCK:
495 		while (npages--) {
496 			pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
497 			if (pp)
498 				page_unlock(pp);
499 			addr += PAGESIZE;
500 		}
501 		return (0);
502 	default:
503 		return (FC_NOSUPPORT);
504 	}
505 	/*NOTREACHED*/
506 }
507 
508 static int
509 segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
510 {
511 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
512 
513 	if (seg->s_as != &kas || size > seg->s_size ||
514 	    addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
515 		panic("segkmem_setprot: bad args");
516 
517 	/*
518 	 * If it is one of segkp pages, call segkp.
519 	 */
520 	if (segkp_bitmap && seg == &kvseg &&
521 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
522 		return (SEGOP_SETPROT(segkp, addr, size, prot));
523 
524 	if (prot == 0)
525 		hat_unload(kas.a_hat, addr, size, HAT_UNLOAD);
526 	else
527 		hat_chgprot(kas.a_hat, addr, size, prot);
528 	return (0);
529 }
530 
531 /*
532  * This is a dummy segkmem function overloaded to call segkp
533  * when segkp is under the heap.
534  */
535 /* ARGSUSED */
536 static int
537 segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
538 {
539 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
540 
541 	if (seg->s_as != &kas)
542 		segkmem_badop();
543 
544 	/*
545 	 * If it is one of segkp pages, call into segkp.
546 	 */
547 	if (segkp_bitmap && seg == &kvseg &&
548 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
549 		return (SEGOP_CHECKPROT(segkp, addr, size, prot));
550 
551 	segkmem_badop();
552 	return (0);
553 }
554 
555 /*
556  * This is a dummy segkmem function overloaded to call segkp
557  * when segkp is under the heap.
558  */
559 /* ARGSUSED */
560 static int
561 segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
562 {
563 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
564 
565 	if (seg->s_as != &kas)
566 		segkmem_badop();
567 
568 	/*
569 	 * If it is one of segkp pages, call into segkp.
570 	 */
571 	if (segkp_bitmap && seg == &kvseg &&
572 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
573 		return (SEGOP_KLUSTER(segkp, addr, delta));
574 
575 	segkmem_badop();
576 	return (0);
577 }
578 
579 static void
580 segkmem_xdump_range(void *arg, void *start, size_t size)
581 {
582 	struct as *as = arg;
583 	caddr_t addr = start;
584 	caddr_t addr_end = addr + size;
585 
586 	while (addr < addr_end) {
587 		pfn_t pfn = hat_getpfnum(kas.a_hat, addr);
588 		if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn))
589 			dump_addpage(as, addr, pfn);
590 		addr += PAGESIZE;
591 		dump_timeleft = dump_timeout;
592 	}
593 }
594 
595 static void
596 segkmem_dump_range(void *arg, void *start, size_t size)
597 {
598 	caddr_t addr = start;
599 	caddr_t addr_end = addr + size;
600 
601 	/*
602 	 * If we are about to start dumping the range of addresses we
603 	 * carved out of the kernel heap for the large page heap walk
604 	 * heap_lp_arena to find what segments are actually populated
605 	 */
606 	if (SEGKMEM_USE_LARGEPAGES &&
607 	    addr == heap_lp_base && addr_end == heap_lp_end &&
608 	    vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
609 		vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT,
610 		    segkmem_xdump_range, arg);
611 	} else {
612 		segkmem_xdump_range(arg, start, size);
613 	}
614 }
615 
616 static void
617 segkmem_dump(struct seg *seg)
618 {
619 	/*
620 	 * The kernel's heap_arena (represented by kvseg) is a very large
621 	 * VA space, most of which is typically unused.  To speed up dumping
622 	 * we use vmem_walk() to quickly find the pieces of heap_arena that
623 	 * are actually in use.  We do the same for heap32_arena and
624 	 * heap_core.
625 	 *
626 	 * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
627 	 * may ultimately need to allocate memory.  Reentrant walks are
628 	 * necessarily imperfect snapshots.  The kernel heap continues
629 	 * to change during a live crash dump, for example.  For a normal
630 	 * crash dump, however, we know that there won't be any other threads
631 	 * messing with the heap.  Therefore, at worst, we may fail to dump
632 	 * the pages that get allocated by the act of dumping; but we will
633 	 * always dump every page that was allocated when the walk began.
634 	 *
635 	 * The other segkmem segments are dense (fully populated), so there's
636 	 * no need to use this technique when dumping them.
637 	 *
638 	 * Note: when adding special dump handling for any new sparsely-
639 	 * populated segments, be sure to add similar handling to the ::kgrep
640 	 * code in mdb.
641 	 */
642 	if (seg == &kvseg) {
643 		vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT,
644 		    segkmem_dump_range, seg->s_as);
645 #ifndef __sparc
646 		vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
647 		    segkmem_dump_range, seg->s_as);
648 #endif
649 	} else if (seg == &kvseg_core) {
650 		vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT,
651 		    segkmem_dump_range, seg->s_as);
652 	} else if (seg == &kvseg32) {
653 		vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT,
654 		    segkmem_dump_range, seg->s_as);
655 		vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
656 		    segkmem_dump_range, seg->s_as);
657 	} else if (seg == &kzioseg) {
658 		/*
659 		 * We don't want to dump pages attached to kzioseg since they
660 		 * contain file data from ZFS.  If this page's segment is
661 		 * kzioseg return instead of writing it to the dump device.
662 		 */
663 		return;
664 	} else {
665 		segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size);
666 	}
667 }
668 
669 /*
670  * lock/unlock kmem pages over a given range [addr, addr+len).
671  * Returns a shadow list of pages in ppp. If there are holes
672  * in the range (e.g. some of the kernel mappings do not have
673  * underlying page_ts) returns ENOTSUP so that as_pagelock()
674  * will handle the range via as_fault(F_SOFTLOCK).
675  */
676 /*ARGSUSED*/
677 static int
678 segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len,
679 	page_t ***ppp, enum lock_type type, enum seg_rw rw)
680 {
681 	page_t **pplist, *pp;
682 	pgcnt_t npages;
683 	spgcnt_t pg;
684 	size_t nb;
685 	struct vnode *vp = seg->s_data;
686 
687 	ASSERT(ppp != NULL);
688 
689 	/*
690 	 * If it is one of segkp pages, call into segkp.
691 	 */
692 	if (segkp_bitmap && seg == &kvseg &&
693 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
694 		return (SEGOP_PAGELOCK(segkp, addr, len, ppp, type, rw));
695 
696 	npages = btopr(len);
697 	nb = sizeof (page_t *) * npages;
698 
699 	if (type == L_PAGEUNLOCK) {
700 		pplist = *ppp;
701 		ASSERT(pplist != NULL);
702 
703 		for (pg = 0; pg < npages; pg++) {
704 			pp = pplist[pg];
705 			page_unlock(pp);
706 		}
707 		kmem_free(pplist, nb);
708 		return (0);
709 	}
710 
711 	ASSERT(type == L_PAGELOCK);
712 
713 	pplist = kmem_alloc(nb, KM_NOSLEEP);
714 	if (pplist == NULL) {
715 		*ppp = NULL;
716 		return (ENOTSUP);	/* take the slow path */
717 	}
718 
719 	for (pg = 0; pg < npages; pg++) {
720 		pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED);
721 		if (pp == NULL) {
722 			while (--pg >= 0)
723 				page_unlock(pplist[pg]);
724 			kmem_free(pplist, nb);
725 			*ppp = NULL;
726 			return (ENOTSUP);
727 		}
728 		pplist[pg] = pp;
729 		addr += PAGESIZE;
730 	}
731 
732 	*ppp = pplist;
733 	return (0);
734 }
735 
736 /*
737  * This is a dummy segkmem function overloaded to call segkp
738  * when segkp is under the heap.
739  */
740 /* ARGSUSED */
741 static int
742 segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
743 {
744 	ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
745 
746 	if (seg->s_as != &kas)
747 		segkmem_badop();
748 
749 	/*
750 	 * If it is one of segkp pages, call into segkp.
751 	 */
752 	if (segkp_bitmap && seg == &kvseg &&
753 	    BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
754 		return (SEGOP_GETMEMID(segkp, addr, memidp));
755 
756 	segkmem_badop();
757 	return (0);
758 }
759 
760 /*ARGSUSED*/
761 static lgrp_mem_policy_info_t *
762 segkmem_getpolicy(struct seg *seg, caddr_t addr)
763 {
764 	return (NULL);
765 }
766 
767 /*ARGSUSED*/
768 static int
769 segkmem_capable(struct seg *seg, segcapability_t capability)
770 {
771 	if (capability == S_CAPABILITY_NOMINFLT)
772 		return (1);
773 	return (0);
774 }
775 
776 static struct seg_ops segkmem_ops = {
777 	SEGKMEM_BADOP(int),		/* dup */
778 	SEGKMEM_BADOP(int),		/* unmap */
779 	SEGKMEM_BADOP(void),		/* free */
780 	segkmem_fault,
781 	SEGKMEM_BADOP(faultcode_t),	/* faulta */
782 	segkmem_setprot,
783 	segkmem_checkprot,
784 	segkmem_kluster,
785 	SEGKMEM_BADOP(size_t),		/* swapout */
786 	SEGKMEM_BADOP(int),		/* sync */
787 	SEGKMEM_BADOP(size_t),		/* incore */
788 	SEGKMEM_BADOP(int),		/* lockop */
789 	SEGKMEM_BADOP(int),		/* getprot */
790 	SEGKMEM_BADOP(u_offset_t),	/* getoffset */
791 	SEGKMEM_BADOP(int),		/* gettype */
792 	SEGKMEM_BADOP(int),		/* getvp */
793 	SEGKMEM_BADOP(int),		/* advise */
794 	segkmem_dump,
795 	segkmem_pagelock,
796 	SEGKMEM_BADOP(int),		/* setpgsz */
797 	segkmem_getmemid,
798 	segkmem_getpolicy,		/* getpolicy */
799 	segkmem_capable,		/* capable */
800 	seg_inherit_notsup		/* inherit */
801 };
802 
803 int
804 segkmem_zio_create(struct seg *seg)
805 {
806 	ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
807 	seg->s_ops = &segkmem_ops;
808 	seg->s_data = &zvp;
809 	kas.a_size += seg->s_size;
810 	return (0);
811 }
812 
813 int
814 segkmem_create(struct seg *seg)
815 {
816 	ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
817 	seg->s_ops = &segkmem_ops;
818 	seg->s_data = &kvp;
819 	kas.a_size += seg->s_size;
820 	return (0);
821 }
822 
823 /*ARGSUSED*/
824 page_t *
825 segkmem_page_create(void *addr, size_t size, int vmflag, void *arg)
826 {
827 	struct seg kseg;
828 	int pgflags;
829 	struct vnode *vp = arg;
830 
831 	if (vp == NULL)
832 		vp = &kvp;
833 
834 	kseg.s_as = &kas;
835 	pgflags = PG_EXCL;
836 
837 	if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
838 		pgflags |= PG_NORELOC;
839 	if ((vmflag & VM_NOSLEEP) == 0)
840 		pgflags |= PG_WAIT;
841 	if (vmflag & VM_PANIC)
842 		pgflags |= PG_PANIC;
843 	if (vmflag & VM_PUSHPAGE)
844 		pgflags |= PG_PUSHPAGE;
845 	if (vmflag & VM_NORMALPRI) {
846 		ASSERT(vmflag & VM_NOSLEEP);
847 		pgflags |= PG_NORMALPRI;
848 	}
849 
850 	return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size,
851 	    pgflags, &kseg, addr));
852 }
853 
854 /*
855  * Allocate pages to back the virtual address range [addr, addr + size).
856  * If addr is NULL, allocate the virtual address space as well.
857  */
858 void *
859 segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr,
860 	page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg)
861 {
862 	page_t *ppl;
863 	caddr_t addr = inaddr;
864 	pgcnt_t npages = btopr(size);
865 	int allocflag;
866 
867 	if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
868 		return (NULL);
869 
870 	ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
871 
872 	if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
873 		if (inaddr == NULL)
874 			vmem_free(vmp, addr, size);
875 		return (NULL);
876 	}
877 
878 	ppl = page_create_func(addr, size, vmflag, pcarg);
879 	if (ppl == NULL) {
880 		if (inaddr == NULL)
881 			vmem_free(vmp, addr, size);
882 		page_unresv(npages);
883 		return (NULL);
884 	}
885 
886 	/*
887 	 * Under certain conditions, we need to let the HAT layer know
888 	 * that it cannot safely allocate memory.  Allocations from
889 	 * the hat_memload vmem arena always need this, to prevent
890 	 * infinite recursion.
891 	 *
892 	 * In addition, the x86 hat cannot safely do memory
893 	 * allocations while in vmem_populate(), because there
894 	 * is no simple bound on its usage.
895 	 */
896 	if (vmflag & VM_MEMLOAD)
897 		allocflag = HAT_NO_KALLOC;
898 #if defined(__x86)
899 	else if (vmem_is_populator())
900 		allocflag = HAT_NO_KALLOC;
901 #endif
902 	else
903 		allocflag = 0;
904 
905 	while (ppl != NULL) {
906 		page_t *pp = ppl;
907 		page_sub(&ppl, pp);
908 		ASSERT(page_iolock_assert(pp));
909 		ASSERT(PAGE_EXCL(pp));
910 		page_io_unlock(pp);
911 		hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp,
912 		    (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
913 		    HAT_LOAD_LOCK | allocflag);
914 		pp->p_lckcnt = 1;
915 #if defined(__x86)
916 		page_downgrade(pp);
917 #else
918 		if (vmflag & SEGKMEM_SHARELOCKED)
919 			page_downgrade(pp);
920 		else
921 			page_unlock(pp);
922 #endif
923 	}
924 
925 	return (addr);
926 }
927 
928 static void *
929 segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp)
930 {
931 	void *addr;
932 	segkmem_gc_list_t *gcp, **prev_gcpp;
933 
934 	ASSERT(vp != NULL);
935 
936 	if (kvseg.s_base == NULL) {
937 #ifndef __sparc
938 		if (bootops->bsys_alloc == NULL)
939 			halt("Memory allocation between bop_alloc() and "
940 			    "kmem_alloc().\n");
941 #endif
942 
943 		/*
944 		 * There's not a lot of memory to go around during boot,
945 		 * so recycle it if we can.
946 		 */
947 		for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL;
948 		    prev_gcpp = &gcp->gc_next) {
949 			if (gcp->gc_arena == vmp && gcp->gc_size == size) {
950 				*prev_gcpp = gcp->gc_next;
951 				return (gcp);
952 			}
953 		}
954 
955 		addr = vmem_alloc(vmp, size, vmflag | VM_PANIC);
956 		if (boot_alloc(addr, size, BO_NO_ALIGN) != addr)
957 			panic("segkmem_alloc: boot_alloc failed");
958 		return (addr);
959 	}
960 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
961 	    segkmem_page_create, vp));
962 }
963 
964 void *
965 segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
966 {
967 	return (segkmem_alloc_vn(vmp, size, vmflag, &kvp));
968 }
969 
970 void *
971 segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag)
972 {
973 	return (segkmem_alloc_vn(vmp, size, vmflag, &zvp));
974 }
975 
976 /*
977  * Any changes to this routine must also be carried over to
978  * devmap_free_pages() in the seg_dev driver. This is because
979  * we currently don't have a special kernel segment for non-paged
980  * kernel memory that is exported by drivers to user space.
981  */
982 static void
983 segkmem_free_vn(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp,
984     void (*func)(page_t *))
985 {
986 	page_t *pp;
987 	caddr_t addr = inaddr;
988 	caddr_t eaddr;
989 	pgcnt_t npages = btopr(size);
990 
991 	ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
992 	ASSERT(vp != NULL);
993 
994 	if (kvseg.s_base == NULL) {
995 		segkmem_gc_list_t *gc = inaddr;
996 		gc->gc_arena = vmp;
997 		gc->gc_size = size;
998 		gc->gc_next = segkmem_gc_list;
999 		segkmem_gc_list = gc;
1000 		return;
1001 	}
1002 
1003 	hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1004 
1005 	for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
1006 #if defined(__x86)
1007 		pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
1008 		if (pp == NULL)
1009 			panic("segkmem_free: page not found");
1010 		if (!page_tryupgrade(pp)) {
1011 			/*
1012 			 * Some other thread has a sharelock. Wait for
1013 			 * it to drop the lock so we can free this page.
1014 			 */
1015 			page_unlock(pp);
1016 			pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
1017 			    SE_EXCL);
1018 		}
1019 #else
1020 		pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1021 #endif
1022 		if (pp == NULL)
1023 			panic("segkmem_free: page not found");
1024 		/* Clear p_lckcnt so page_destroy() doesn't update availrmem */
1025 		pp->p_lckcnt = 0;
1026 		if (func)
1027 			func(pp);
1028 		else
1029 			page_destroy(pp, 0);
1030 	}
1031 	if (func == NULL)
1032 		page_unresv(npages);
1033 
1034 	if (vmp != NULL)
1035 		vmem_free(vmp, inaddr, size);
1036 
1037 }
1038 
1039 void
1040 segkmem_xfree(vmem_t *vmp, void *inaddr, size_t size, void (*func)(page_t *))
1041 {
1042 	segkmem_free_vn(vmp, inaddr, size, &kvp, func);
1043 }
1044 
1045 void
1046 segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
1047 {
1048 	segkmem_free_vn(vmp, inaddr, size, &kvp, NULL);
1049 }
1050 
1051 void
1052 segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size)
1053 {
1054 	segkmem_free_vn(vmp, inaddr, size, &zvp, NULL);
1055 }
1056 
1057 void
1058 segkmem_gc(void)
1059 {
1060 	ASSERT(kvseg.s_base != NULL);
1061 	while (segkmem_gc_list != NULL) {
1062 		segkmem_gc_list_t *gc = segkmem_gc_list;
1063 		segkmem_gc_list = gc->gc_next;
1064 		segkmem_free(gc->gc_arena, gc, gc->gc_size);
1065 	}
1066 }
1067 
1068 /*
1069  * Legacy entry points from here to end of file.
1070  */
1071 void
1072 segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot,
1073     pfn_t pfn, uint_t flags)
1074 {
1075 	hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1076 	hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot,
1077 	    flags | HAT_LOAD_LOCK);
1078 }
1079 
1080 void
1081 segkmem_mapout(struct seg *seg, void *addr, size_t size)
1082 {
1083 	hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1084 }
1085 
1086 void *
1087 kmem_getpages(pgcnt_t npages, int kmflag)
1088 {
1089 	return (kmem_alloc(ptob(npages), kmflag));
1090 }
1091 
1092 void
1093 kmem_freepages(void *addr, pgcnt_t npages)
1094 {
1095 	kmem_free(addr, ptob(npages));
1096 }
1097 
1098 /*
1099  * segkmem_page_create_large() allocates a large page to be used for the kmem
1100  * caches. If kpr is enabled we ask for a relocatable page unless requested
1101  * otherwise. If kpr is disabled we have to ask for a non-reloc page
1102  */
1103 static page_t *
1104 segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg)
1105 {
1106 	int pgflags;
1107 
1108 	pgflags = PG_EXCL;
1109 
1110 	if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
1111 		pgflags |= PG_NORELOC;
1112 	if (!(vmflag & VM_NOSLEEP))
1113 		pgflags |= PG_WAIT;
1114 	if (vmflag & VM_PUSHPAGE)
1115 		pgflags |= PG_PUSHPAGE;
1116 	if (vmflag & VM_NORMALPRI)
1117 		pgflags |= PG_NORMALPRI;
1118 
1119 	return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
1120 	    pgflags, &kvseg, addr, arg));
1121 }
1122 
1123 /*
1124  * Allocate a large page to back the virtual address range
1125  * [addr, addr + size).  If addr is NULL, allocate the virtual address
1126  * space as well.
1127  */
1128 static void *
1129 segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
1130     uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
1131     void *pcarg)
1132 {
1133 	caddr_t addr = inaddr, pa;
1134 	size_t  lpsize = segkmem_lpsize;
1135 	pgcnt_t npages = btopr(size);
1136 	pgcnt_t nbpages = btop(lpsize);
1137 	pgcnt_t nlpages = size >> segkmem_lpshift;
1138 	size_t  ppasize = nbpages * sizeof (page_t *);
1139 	page_t *pp, *rootpp, **ppa, *pplist = NULL;
1140 	int i;
1141 
1142 	vmflag |= VM_NOSLEEP;
1143 
1144 	if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
1145 		return (NULL);
1146 	}
1147 
1148 	/*
1149 	 * allocate an array we need for hat_memload_array.
1150 	 * we use a separate arena to avoid recursion.
1151 	 * we will not need this array when hat_memload_array learns pp++
1152 	 */
1153 	if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) {
1154 		goto fail_array_alloc;
1155 	}
1156 
1157 	if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
1158 		goto fail_vmem_alloc;
1159 
1160 	ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0);
1161 
1162 	/* create all the pages */
1163 	for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) {
1164 		if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL)
1165 			goto fail_page_create;
1166 		page_list_concat(&pplist, &pp);
1167 	}
1168 
1169 	/* at this point we have all the resource to complete the request */
1170 	while ((rootpp = pplist) != NULL) {
1171 		for (i = 0; i < nbpages; i++) {
1172 			ASSERT(pplist != NULL);
1173 			pp = pplist;
1174 			page_sub(&pplist, pp);
1175 			ASSERT(page_iolock_assert(pp));
1176 			page_io_unlock(pp);
1177 			ppa[i] = pp;
1178 		}
1179 		/*
1180 		 * Load the locked entry. It's OK to preload the entry into the
1181 		 * TSB since we now support large mappings in the kernel TSB.
1182 		 */
1183 		hat_memload_array(kas.a_hat,
1184 		    (caddr_t)(uintptr_t)rootpp->p_offset, lpsize,
1185 		    ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
1186 		    HAT_LOAD_LOCK);
1187 
1188 		for (--i; i >= 0; --i) {
1189 			ppa[i]->p_lckcnt = 1;
1190 			page_unlock(ppa[i]);
1191 		}
1192 	}
1193 
1194 	vmem_free(segkmem_ppa_arena, ppa, ppasize);
1195 	return (addr);
1196 
1197 fail_page_create:
1198 	while ((rootpp = pplist) != NULL) {
1199 		for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) {
1200 			ASSERT(pp != NULL);
1201 			page_sub(&pplist, pp);
1202 			ASSERT(page_iolock_assert(pp));
1203 			page_io_unlock(pp);
1204 		}
1205 		page_destroy_pages(rootpp);
1206 	}
1207 
1208 	if (inaddr == NULL)
1209 		vmem_free(vmp, addr, size);
1210 
1211 fail_vmem_alloc:
1212 	vmem_free(segkmem_ppa_arena, ppa, ppasize);
1213 
1214 fail_array_alloc:
1215 	page_unresv(npages);
1216 
1217 	return (NULL);
1218 }
1219 
1220 static void
1221 segkmem_free_one_lp(caddr_t addr, size_t size)
1222 {
1223 	page_t		*pp, *rootpp = NULL;
1224 	pgcnt_t 	pgs_left = btopr(size);
1225 
1226 	ASSERT(size == segkmem_lpsize);
1227 
1228 	hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1229 
1230 	for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) {
1231 		pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1232 		if (pp == NULL)
1233 			panic("segkmem_free_one_lp: page not found");
1234 		ASSERT(PAGE_EXCL(pp));
1235 		pp->p_lckcnt = 0;
1236 		if (rootpp == NULL)
1237 			rootpp = pp;
1238 	}
1239 	ASSERT(rootpp != NULL);
1240 	page_destroy_pages(rootpp);
1241 
1242 	/* page_unresv() is done by the caller */
1243 }
1244 
1245 /*
1246  * This function is called to import new spans into the vmem arenas like
1247  * kmem_default_arena and kmem_oversize_arena. It first tries to import
1248  * spans from large page arena - kmem_lp_arena. In order to do this it might
1249  * have to "upgrade the requested size" to kmem_lp_arena quantum. If
1250  * it was not able to satisfy the upgraded request it then calls regular
1251  * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
1252  */
1253 /*ARGSUSED*/
1254 void *
1255 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
1256 {
1257 	size_t size;
1258 	kthread_t *t = curthread;
1259 	segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1260 
1261 	ASSERT(sizep != NULL);
1262 
1263 	size = *sizep;
1264 
1265 	if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
1266 	    !(vmflag & SEGKMEM_SHARELOCKED)) {
1267 
1268 		size_t kmemlp_qnt = segkmem_kmemlp_quantum;
1269 		size_t asize = P2ROUNDUP(size, kmemlp_qnt);
1270 		void  *addr = NULL;
1271 		ulong_t *lpthrtp = &lpcb->lp_throttle;
1272 		ulong_t lpthrt = *lpthrtp;
1273 		int	dowakeup = 0;
1274 		int	doalloc = 1;
1275 
1276 		ASSERT(kmem_lp_arena != NULL);
1277 		ASSERT(asize >= size);
1278 
1279 		if (lpthrt != 0) {
1280 			/* try to update the throttle value */
1281 			lpthrt = atomic_inc_ulong_nv(lpthrtp);
1282 			if (lpthrt >= segkmem_lpthrottle_max) {
1283 				lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
1284 				    segkmem_lpthrottle_max / 4);
1285 			}
1286 
1287 			/*
1288 			 * when we get above throttle start do an exponential
1289 			 * backoff at trying large pages and reaping
1290 			 */
1291 			if (lpthrt > segkmem_lpthrottle_start &&
1292 			    !ISP2(lpthrt)) {
1293 				lpcb->allocs_throttled++;
1294 				lpthrt--;
1295 				if (ISP2(lpthrt))
1296 					kmem_reap();
1297 				return (segkmem_alloc(vmp, size, vmflag));
1298 			}
1299 		}
1300 
1301 		if (!(vmflag & VM_NOSLEEP) &&
1302 		    segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
1303 		    vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
1304 		    asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
1305 
1306 			/*
1307 			 * we are low on free memory in kmem_lp_arena
1308 			 * we let only one guy to allocate heap_lp
1309 			 * quantum size chunk that everybody is going to
1310 			 * share
1311 			 */
1312 			mutex_enter(&lpcb->lp_lock);
1313 
1314 			if (lpcb->lp_wait) {
1315 
1316 				/* we are not the first one - wait */
1317 				cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
1318 				if (vmem_size(kmem_lp_arena, VMEM_FREE) <
1319 				    kmemlp_qnt)  {
1320 					doalloc = 0;
1321 				}
1322 			} else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
1323 			    kmemlp_qnt) {
1324 
1325 				/*
1326 				 * we are the first one, make sure we import
1327 				 * a large page
1328 				 */
1329 				if (asize == kmemlp_qnt)
1330 					asize += kmemlp_qnt;
1331 				dowakeup = 1;
1332 				lpcb->lp_wait = 1;
1333 			}
1334 
1335 			mutex_exit(&lpcb->lp_lock);
1336 		}
1337 
1338 		/*
1339 		 * VM_ABORT flag prevents sleeps in vmem_xalloc when
1340 		 * large pages are not available. In that case this allocation
1341 		 * attempt will fail and we will retry allocation with small
1342 		 * pages. We also do not want to panic if this allocation fails
1343 		 * because we are going to retry.
1344 		 */
1345 		if (doalloc) {
1346 			addr = vmem_alloc(kmem_lp_arena, asize,
1347 			    (vmflag | VM_ABORT) & ~VM_PANIC);
1348 
1349 			if (dowakeup) {
1350 				mutex_enter(&lpcb->lp_lock);
1351 				ASSERT(lpcb->lp_wait != 0);
1352 				lpcb->lp_wait = 0;
1353 				cv_broadcast(&lpcb->lp_cv);
1354 				mutex_exit(&lpcb->lp_lock);
1355 			}
1356 		}
1357 
1358 		if (addr != NULL) {
1359 			*sizep = asize;
1360 			*lpthrtp = 0;
1361 			return (addr);
1362 		}
1363 
1364 		if (vmflag & VM_NOSLEEP)
1365 			lpcb->nosleep_allocs_failed++;
1366 		else
1367 			lpcb->sleep_allocs_failed++;
1368 		lpcb->alloc_bytes_failed += size;
1369 
1370 		/* if large page throttling is not started yet do it */
1371 		if (segkmem_use_lpthrottle && lpthrt == 0) {
1372 			lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
1373 		}
1374 	}
1375 	return (segkmem_alloc(vmp, size, vmflag));
1376 }
1377 
1378 void
1379 segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size)
1380 {
1381 	if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) {
1382 		segkmem_free(vmp, inaddr, size);
1383 	} else {
1384 		vmem_free(kmem_lp_arena, inaddr, size);
1385 	}
1386 }
1387 
1388 /*
1389  * segkmem_alloc_lpi() imports virtual memory from large page heap arena
1390  * into kmem_lp arena. In the process it maps the imported segment with
1391  * large pages
1392  */
1393 static void *
1394 segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag)
1395 {
1396 	segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1397 	void  *addr;
1398 
1399 	ASSERT(size != 0);
1400 	ASSERT(vmp == heap_lp_arena);
1401 
1402 	/* do not allow large page heap grow beyound limits */
1403 	if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) {
1404 		lpcb->allocs_limited++;
1405 		return (NULL);
1406 	}
1407 
1408 	addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0,
1409 	    segkmem_page_create_large, NULL);
1410 	return (addr);
1411 }
1412 
1413 /*
1414  * segkmem_free_lpi() returns virtual memory back into large page heap arena
1415  * from kmem_lp arena. Beore doing this it unmaps the segment and frees
1416  * large pages used to map it.
1417  */
1418 static void
1419 segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size)
1420 {
1421 	pgcnt_t		nlpages = size >> segkmem_lpshift;
1422 	size_t		lpsize = segkmem_lpsize;
1423 	caddr_t		addr = inaddr;
1424 	pgcnt_t 	npages = btopr(size);
1425 	int		i;
1426 
1427 	ASSERT(vmp == heap_lp_arena);
1428 	ASSERT(IS_KMEM_VA_LARGEPAGE(addr));
1429 	ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0);
1430 
1431 	for (i = 0; i < nlpages; i++) {
1432 		segkmem_free_one_lp(addr, lpsize);
1433 		addr += lpsize;
1434 	}
1435 
1436 	page_unresv(npages);
1437 
1438 	vmem_free(vmp, inaddr, size);
1439 }
1440 
1441 /*
1442  * This function is called at system boot time by kmem_init right after
1443  * /etc/system file has been read. It checks based on hardware configuration
1444  * and /etc/system settings if system is going to use large pages. The
1445  * initialiazation necessary to actually start using large pages
1446  * happens later in the process after segkmem_heap_lp_init() is called.
1447  */
1448 int
1449 segkmem_lpsetup()
1450 {
1451 	int use_large_pages = 0;
1452 
1453 #ifdef __sparc
1454 
1455 	size_t memtotal = physmem * PAGESIZE;
1456 
1457 	if (heap_lp_base == NULL) {
1458 		segkmem_lpsize = PAGESIZE;
1459 		return (0);
1460 	}
1461 
1462 	/* get a platform dependent value of large page size for kernel heap */
1463 	segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize);
1464 
1465 	if (segkmem_lpsize <= PAGESIZE) {
1466 		/*
1467 		 * put virtual space reserved for the large page kernel
1468 		 * back to the regular heap
1469 		 */
1470 		vmem_xfree(heap_arena, heap_lp_base,
1471 		    heap_lp_end - heap_lp_base);
1472 		heap_lp_base = NULL;
1473 		heap_lp_end = NULL;
1474 		segkmem_lpsize = PAGESIZE;
1475 		return (0);
1476 	}
1477 
1478 	/* set heap_lp quantum if necessary */
1479 	if (segkmem_heaplp_quantum == 0 || !ISP2(segkmem_heaplp_quantum) ||
1480 	    P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) {
1481 		segkmem_heaplp_quantum = segkmem_lpsize;
1482 	}
1483 
1484 	/* set kmem_lp quantum if necessary */
1485 	if (segkmem_kmemlp_quantum == 0 || !ISP2(segkmem_kmemlp_quantum) ||
1486 	    segkmem_kmemlp_quantum > segkmem_heaplp_quantum) {
1487 		segkmem_kmemlp_quantum = segkmem_heaplp_quantum;
1488 	}
1489 
1490 	/* set total amount of memory allowed for large page kernel heap */
1491 	if (segkmem_kmemlp_max == 0) {
1492 		if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100)
1493 			segkmem_kmemlp_pcnt = 12;
1494 		segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100;
1495 	}
1496 	segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max,
1497 	    segkmem_heaplp_quantum);
1498 
1499 	/* fix lp kmem preallocation request if necesssary */
1500 	if (segkmem_kmemlp_min) {
1501 		segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min,
1502 		    segkmem_heaplp_quantum);
1503 		if (segkmem_kmemlp_min > segkmem_kmemlp_max)
1504 			segkmem_kmemlp_min = segkmem_kmemlp_max;
1505 	}
1506 
1507 	use_large_pages = 1;
1508 	segkmem_lpszc = page_szc(segkmem_lpsize);
1509 	segkmem_lpshift = page_get_shift(segkmem_lpszc);
1510 
1511 #endif
1512 	return (use_large_pages);
1513 }
1514 
1515 void
1516 segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size)
1517 {
1518 	ASSERT(zio_mem_base != NULL);
1519 	ASSERT(zio_mem_size != 0);
1520 
1521 	/*
1522 	 * To reduce VA space fragmentation, we set up quantum caches for the
1523 	 * smaller sizes;  we chose 32k because that translates to 128k VA
1524 	 * slabs, which matches nicely with the common 128k zio_data bufs.
1525 	 */
1526 	zio_arena = vmem_create("zfs_file_data", zio_mem_base, zio_mem_size,
1527 	    PAGESIZE, NULL, NULL, NULL, 32 * 1024, VM_SLEEP);
1528 
1529 	zio_alloc_arena = vmem_create("zfs_file_data_buf", NULL, 0, PAGESIZE,
1530 	    segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP);
1531 
1532 	ASSERT(zio_arena != NULL);
1533 	ASSERT(zio_alloc_arena != NULL);
1534 }
1535 
1536 #ifdef __sparc
1537 
1538 
1539 static void *
1540 segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag)
1541 {
1542 	size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1543 	void   *addr;
1544 
1545 	if (ppaquantum <= PAGESIZE)
1546 		return (segkmem_alloc(vmp, size, vmflag));
1547 
1548 	ASSERT((size & (ppaquantum - 1)) == 0);
1549 
1550 	addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag);
1551 	if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0,
1552 	    segkmem_page_create, NULL) == NULL) {
1553 		vmem_xfree(vmp, addr, size);
1554 		addr = NULL;
1555 	}
1556 
1557 	return (addr);
1558 }
1559 
1560 static void
1561 segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size)
1562 {
1563 	size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1564 
1565 	ASSERT(addr != NULL);
1566 
1567 	if (ppaquantum <= PAGESIZE) {
1568 		segkmem_free(vmp, addr, size);
1569 	} else {
1570 		segkmem_free(NULL, addr, size);
1571 		vmem_xfree(vmp, addr, size);
1572 	}
1573 }
1574 
1575 void
1576 segkmem_heap_lp_init()
1577 {
1578 	segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1579 	size_t heap_lp_size = heap_lp_end - heap_lp_base;
1580 	size_t lpsize = segkmem_lpsize;
1581 	size_t ppaquantum;
1582 	void   *addr;
1583 
1584 	if (segkmem_lpsize <= PAGESIZE) {
1585 		ASSERT(heap_lp_base == NULL);
1586 		ASSERT(heap_lp_end == NULL);
1587 		return;
1588 	}
1589 
1590 	ASSERT(segkmem_heaplp_quantum >= lpsize);
1591 	ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0);
1592 	ASSERT(lpcb->lp_uselp == 0);
1593 	ASSERT(heap_lp_base != NULL);
1594 	ASSERT(heap_lp_end != NULL);
1595 	ASSERT(heap_lp_base < heap_lp_end);
1596 	ASSERT(heap_lp_arena == NULL);
1597 	ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0);
1598 	ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0);
1599 
1600 	/* create large page heap arena */
1601 	heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size,
1602 	    segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1603 
1604 	ASSERT(heap_lp_arena != NULL);
1605 
1606 	/* This arena caches memory already mapped by large pages */
1607 	kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum,
1608 	    segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP);
1609 
1610 	ASSERT(kmem_lp_arena != NULL);
1611 
1612 	mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL);
1613 	cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL);
1614 
1615 	/*
1616 	 * this arena is used for the array of page_t pointers necessary
1617 	 * to call hat_mem_load_array
1618 	 */
1619 	ppaquantum = btopr(lpsize) * sizeof (page_t *);
1620 	segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum,
1621 	    segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum,
1622 	    VM_SLEEP);
1623 
1624 	ASSERT(segkmem_ppa_arena != NULL);
1625 
1626 	/* prealloacate some memory for the lp kernel heap */
1627 	if (segkmem_kmemlp_min) {
1628 
1629 		ASSERT(P2PHASE(segkmem_kmemlp_min,
1630 		    segkmem_heaplp_quantum) == 0);
1631 
1632 		if ((addr = segkmem_alloc_lpi(heap_lp_arena,
1633 		    segkmem_kmemlp_min, VM_SLEEP)) != NULL) {
1634 
1635 			addr = vmem_add(kmem_lp_arena, addr,
1636 			    segkmem_kmemlp_min, VM_SLEEP);
1637 			ASSERT(addr != NULL);
1638 		}
1639 	}
1640 
1641 	lpcb->lp_uselp = 1;
1642 }
1643 
1644 #endif
1645